From a770c77a23ceacd1631991be6f30b6991754c76c Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 24 Sep 2026 17:55:37 +0000 Subject: [PATCH] Document resilient reports and remove obsolete warning material --- README.md | 5 + docs/cli.md | 26 +- docs/config.md | 10 +- docs/integrations/openai-compatible-llm.md | 17 +- docs/integrations/output-files.md | 253 ++++++------------ docs/integrations/subprocess.md | 23 +- docs/internal/diagnostics-reporting.md | 119 +++----- docs/internal/llm-runtime.md | 15 +- docs/internal/modules.md | 11 +- docs/internal/overview.md | 7 +- docs/internal/pipeline.md | 98 +++---- docs/internal/validators.md | 15 +- docs/migrations/v2.md | 25 +- docs/operations.md | 154 ++++------- docs/policy/architecture.md | 12 +- docs/roadmap/configkit.md | 4 +- docs/roadmap/implementation.md | 11 + docs/roadmap/library-adoption.md | 5 +- docs/roadmap/library-boundary-findings.md | 6 +- docs/roadmap/library-boundary.md | 2 +- docs/roadmap/remediation.md | 2 +- docs/roadmap/runtime-behavior.md | 12 +- examples/production-config.yml | 5 +- .../processreport/correction_ledger.go | 12 - .../processreport/correction_ledger_test.go | 15 -- internal/framework/warnings/warnings.go | 18 -- 26 files changed, 346 insertions(+), 536 deletions(-) delete mode 100644 internal/framework/warnings/warnings.go diff --git a/README.md b/README.md index f7069a7..a63599d 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,11 @@ Run the included small input with the minimal config: This makes LLM requests. Explicit setting flags override the selected file and optional overlay; general environment setting overrides are no longer read. +By default the command writes `errors.yml`, `recoveries.yml`, and `report.md` +beside `corrected.json`. Use `--reporting=false` to suppress those files, or +`--debug` to retain detailed run artifacts under the configured debug work +directory. Model failures affecting some operations can still yield a completed +transcript with applied corrections; the summary and reports identify them. See the [v2 migration guide](docs/migrations/v2.md) for old configurations. ## Documentation diff --git a/docs/cli.md b/docs/cli.md index e8376d2..cc70d5c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -41,9 +41,17 @@ validation. The shared setting flags are: | Normalization (numeric seconds) | `--normalize-max-segment-gap`, `--normalize-ellipsis-gap`, `--normalize-max-segment-duration`, `--normalize-max-segment-tokens` | | Confidence thresholds | `--glossary-confidence-threshold`, `--homophones-confidence-threshold`, `--spoken-word-confidence-threshold`, `--grammar-confidence-threshold` | | Context | `--transcript-description` | -| Diagnostics | `--work-dir`, `--work-dir-retention` | +| Reporting | `--reporting[=true\|false]`, `--report-dir DIR` | +| Debug | `--debug[=true\|false]`, `--work-dir DIR` | -`process` alone accepts `--glossary` (required), `--output`, and `--report-json`. +`process` alone accepts `--glossary` (required) and `--output`. +Reporting defaults to enabled. `--reporting=false` disables all three ordinary +reports; an explicit false overrides a true file or overlay setting. With file +output, reports go beside the transcript and `--report-dir` has no effect. With +stdout output, `--report-dir` selects the report directory, defaulting to the +invocation working directory. `--debug` separately enables retained detailed +artifacts under `--work-dir`. Configuration inspection does not create or probe +these output directories. Per-role model, base URL, key, timeout, retry, and concurrency flags have been removed. Models and generation controls belong to PromptKit profiles, backend credentials and capacity to configuration, and retry/deadline settings to the @@ -65,13 +73,19 @@ settings YAML is for inspection and is not guaranteed reloadable. For `process`, `--output` writes the corrected transcript to a file and leaves stdout empty on success. Without it, stdout carries transcript JSON only on -success. `--report-json` writes a separate report file. Stderr carries readable -errors and diagnostic locations. A successful run can contain skipped corrections -or warnings; see the [report contract](integrations/output-files.md). +success. Enabled reporting creates `errors.yml`, `recoveries.yml`, and +`report.md` in the selected directory. Successful stderr begins with +`Completed: N corrections applied; E work units with errors; R recoveries.` +Report and debug locations follow only when created. Model-operation failures +can coexist with a completed run and applied corrections; read the YAML reports +for details. Debug artifacts are created only with `--debug`. See the +[report contract](integrations/output-files.md). Exit codes are `0` for success, `1` for runtime or output failure, and `2` for usage or configuration input errors. Invalid settings and profiles return `2`; -broken embedded assets and serialization or write failures return `1`. +broken embedded assets, caller cancellation, fatal processing, and required +output failures return `1`. A late report-write failure can follow a published +transcript; exit status and report status remain authoritative. Configuration commands inspect profiles without contacting a provider. See the [input files](integrations/transcript-glossary-files.md), diff --git a/docs/config.md b/docs/config.md index 91a8c11..27a89fd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -61,12 +61,18 @@ field provenance separately from ConfigKit's redacted settings YAML. | `normalization.max_segment_tokens` | `2048` | Positive token estimate. | | `thresholds.glossary`, `.homophones`, `.spoken_word`, `.grammar` | `0.8` each | Confidence thresholds in `[0, 1]`. | | `context.description` | Empty | Trimmed prompt context, at most 500 bytes. | -| `diagnostics.work_dir` | `/tmp/audita` | Nonempty artifact parent directory. | -| `diagnostics.retention` | `auto` | `auto`, `always`, or `never`; see [operations](operations.md). | +| `reporting.enabled` | `true` | Write the ordinary three-file report bundle when processing. | +| `reporting.stdout_dir` | `.` | Report directory for stdout transcript output; ignored for file output or disabled reporting. | +| `diagnostics.debug` | `false` | Persist detailed debug artifacts when processing. | +| `diagnostics.work_dir` | `/tmp/audita` | Parent of the retained debug run directory; required only when debug is enabled. | Module keys select correction modules; validator chains remain built into them. The loader preserves module order and repetitions. Normalization time settings accept numeric seconds only. The operation timeout uses a duration string. +`reporting.stdout_dir` must be nonblank only for reporting-enabled stdout runs; +`diagnostics.work_dir` must be nonblank only with debug enabled. These path +requirements are checked when choosing process destinations, not during +configuration inspection. ## Credentials diff --git a/docs/integrations/openai-compatible-llm.md b/docs/integrations/openai-compatible-llm.md index 4c1a774..6931398 100644 --- a/docs/integrations/openai-compatible-llm.md +++ b/docs/integrations/openai-compatible-llm.md @@ -22,10 +22,13 @@ cancellation or an earlier deadline also applies. See [configuration](../config. for defaults and [PromptKit's consumer contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/consumers/pkg-promptkit.md) for the pinned protocol and retry details. -A failed JSON Schema result after repair is classified as malformed generated -output; proposal and validator callers can turn it into a warning. Provider, -transport, malformed HTTP envelope, retry, admission, and deadline failures stay -runtime failures. Audita decodes into domain types only after PromptKit validates -the JSON. Errors exposed to users carry safe messages while retaining typed -causes for internal classification. Diagnostic request/response content is -sensitive and follows [operations guidance](../operations.md#artifact-handling). +A model operation that exhausts structural repair, a live-parent local operation +timeout, capacity rejection, transport failure, malformed provider response, +or provider HTTP failure is recoverable for that operation. Proposal generation +then contributes no candidates from that response; a failed validation batch +marks its candidates unavailable while other batches and later modules can +continue. Caller cancellation, broken assets or configuration, and unknown +internal failures remain fatal and cancel peers. Audita decodes into domain +types only after PromptKit validates the JSON. Ordinary reports use safe reason +codes and counts rather than provider messages. Full request/response content +is written only with `--debug`; see [operations](../operations.md#diagnostics-directory). diff --git a/docs/integrations/output-files.md b/docs/integrations/output-files.md index ac50c11..cc033cd 100644 --- a/docs/integrations/output-files.md +++ b/docs/integrations/output-files.md @@ -1,189 +1,90 @@ -# Output Files and Report Formats - -This document owns externally consumed corrected-transcript and report formats, -and describes the correction ledger and utilization diagnostics. It documents -current serialized shapes without making every debugging artifact a versioned -compatibility promise. Physical filenames, placement, retention, and permissions -belong to [operations](../operations.md); invocation and output channels belong to -the [CLI](../cli.md#streams-and-exit-status). +# Output Files and Reports ## Corrected Transcript -The [output setting](../config.md#settings) selects the envelope: +The corrected transcript uses the selected `output.schema`: `bare-segments` +or `audita-v1`. Its schema and text rules are unchanged by resilience +reporting. See [configuration](../config.md#settings) for selection and +[operations](../operations.md#required-output-files) for placement and writes. -| Encoder | JSON shape | +With reporting enabled, Audita produces three ordinary files: + +- `errors.yml`: affected work units and run-level failures; +- `recoveries.yml`: successful logical model operations with repair, service + retry, or defective-proposal filtering; +- `report.md`: a concise overview and complete per-chunk matrices. + +The YAML documents have Audita-owned schemas `audita-errors` and +`audita-recoveries`, each with integer `version: 1`. Both contain, in order, +`schema`, `version`, `run_id`, `status`, `started_at`, `completed_at`, `summary`, +and an always-present `errors` or `recoveries` sequence. Status is +`in_progress`, `completed`, or `failed`. Times are UTC RFC3339 strings; +`completed_at` is null until finalization. A recoverable model error can appear +with `status: completed`. + +## Summary Counts + +Both documents carry the same summary, in this order: + +| Count | Meaning | | --- | --- | -| `bare-segments` | Top-level array of segment objects | -| `audita-v1` | Object with `schema: "audita-v1"`, `version: "v1"`, and `segments` containing the segment array | +| `work_units_total` | Configured module instances × initial chunks. | +| `work_units_started`, `work_units_completed`, `work_units_interrupted`, `work_units_not_started` | Explicit processing states. | +| `work_units_with_errors` | Distinct units with recoverable operation failures. | +| `run_errors` | Fatal processing or output failures outside that unit count. | +| `corrections_applied` | Approved proposals applied, not replacement occurrences. | +| `llm_operations` | Structured-client invocations, including pre-dispatch failures. | +| `proposal_responses` | Structurally usable proposal responses. | +| `recoveries` | Successful operations with one or more recovery reasons. | +| `proposal_responses_with_filtering` | Proposal responses that filtered at least one defective proposal. | -Both emit these segment fields: +Unknown counts before processing are `null`; completed empty counts are `0`. +Retries and repair generations are details of one operation, not additional +operations. Provider token usage, execution timing, and attempt-level evidence +remain in opt-in debug files. -| Field | Type | Meaning | -| --- | --- | --- | -| `id` | Integer | Identifier in the normalized transcript | -| `speaker` | String | Speaker label | -| `start`, `end` | Numbers | Start/end times in seconds | -| `text` | String | Text after normalization and accepted corrections | -| `categories` | Array of strings, optional | Segment categories; omitted when empty | +## Error and Recovery Records -Input IDs and segment boundaries need not survive normalization unchanged; -normalization can merge segments and reassign IDs. The output describes the -resulting transcript, not a lossless copy of the source JSON. Input acceptance -rules belong to the [input-file contract](transcript-glossary-files.md). +An `errors` record with `scope: work_unit` identifies zero-based `stage_index` +and `chunk_index`, module key/instance, initial segment range, applied-correction +count, and a `failures` sequence. Each failure identifies the operation +(`proposal`, `validation`, or `validation_input`), optional validator and +zero-based batch index, safe reason/message, sorted proposal indices when +known, and execution counts when available. Multiple failed batches in one +work unit form one error record. `scope: run` instead records phase, safe reason +and message, optional origin, and output path where relevant. It can describe +a fatal error or secondary output-write failure without inventing a failed +model work unit. Raw provider errors and transcript text are absent. -## Process Report +Each recovery identifies its work unit and successful operation, with sorted +`reasons`, a safe message, available execution counts, and sorted +`filtered_proposals` details. One recovery can contain `output_repaired`, +`service_recovered`, and `proposals_filtered` together. Filtering covers +defective response proposals; low confidence, protected-term rejection, +ordinary editorial rejection, and an apply-time conflict caused by an earlier +accepted correction are not filtering recoveries. Failed operations are errors, +not recoveries. The initial chunk identity remains stable even if earlier +modules change its text. -The report is a JSON object. `report_metadata` identifies its schema separately -from the transcript output envelope: +## Markdown Overview -| Metadata field | Type / current value | -| --- | --- | -| `report_schema_name` | String: `audita-process-report` | -| `report_schema_version` | String: `v2` | -| `output_schema` | String identifying the selected transcript encoder | -| `config_version` | Optional integer, present when file configuration was loaded | +`report.md` includes run ID, status, corrections, work coverage, error and +recovery totals, input/output paths, and full work-unit error and recovery +matrices. Rows show every initial chunk with 1-based chunk number and initial +segment range; columns follow configured module-instance order, including +repeats. Completed error cells are `0` or `1`; recovery cells are operation +counts. `—` means not started and `n*` means interrupted with n facts recorded. +An empty stage or chunk set says “No work units.” Short reason breakdowns count +records containing each reason; a recovery can have several reasons, so those +counts need not sum to the recovery total. The report refers to both YAML +files. It does not include transcript excerpts or correction lists. -The remaining top-level fields are: +## Debug-Only Files -| Field | Type and meaning | -| --- | --- | -| `phase` | String; currently `default_pipeline` | -| `status` | String: `success` or `failed` | -| `operation` | String; currently `process` | -| `transcript_path`, `glossary_path` | Input path strings | -| `output_path` | Optional transcript destination string | -| `modules` | Array of configured module keys, including repeats | -| `started_at`, `completed_at` | RFC 3339 timestamps, possibly fractional; completion is an optional field populated by completed process runs | -| `error_phase`, `error_message` | Optional failure classification and human-readable detail | -| `input_segment_count`, `normalized_segment_count` | Optional integer normalization counts | -| `normalization_merges`, `normalization_id_reassignments` | Optional integer transformation counts | -| `normalization_skipped` | Object of optional integer counters: `different_speakers`, `gap_too_large`, `duration_exceeded`, `token_limit_exceeded`; can be empty | -| `chunking` | Optional section summary described below | -| `diagnostics` | Optional artifact-path metadata described below | -| `llm_summary` | Optional logical LLM execution summary, present when observations were initialized | -| `modules_summary` | Optional aggregate module outcome | -| `module_results` | Optional array of module outcomes | - -Failures can preserve partial module results. Missing summaries indicate unavailable -information, not necessarily zero work. Early configuration failures occur before -report construction. Report success describes processing results; a later file-write -or cleanup error can still cause a nonzero CLI exit. Use the exit result and report -together as explained in [subprocess guidance](subprocess.md). - -Error phases identify where processing failed, for example `transcript_read`, -`glossary_read`, `transcript_schema`, `glossary_schema`, `chunking`, `runner_setup`, -`runner_execution`, `output_schema`, `serialization`, `output_write`, and -`stdout_write`. This is not an exhaustive enumeration or a promise that every -late failure is represented in a successfully written report. Error messages are -human-readable, not structured fields to parse for control flow. - -### Chunking and Diagnostics Metadata - -`chunking` contains integer fields `chunk_count`, `min_estimated_chunk_tokens`, -`max_estimated_chunk_tokens`, `total_estimated_transcript_tokens`, -`max_section_tokens`, and `min_section_tokens`, plus optional integer -`target_sections`. Token counts are estimates. - -`diagnostics` contains optional path strings: - -- `directory_path` -- `source_transcript_path`, `parsed_source_transcript_path`, `normalized_transcript_path` -- `normalization_summary_path`, `chunking_summary_path` -- `utilization_summary_path`, `llm_execution_path`, `selected_profiles_path`, `correction_ledger_path` -- `invocation_metadata_path`, `redacted_effective_config_path`, `error_log_path` - -The [operations guide](../operations.md#diagnostics-directory) defines the physical -artifacts and path-lifetime caveats. - -### Module Outcomes - -`modules_summary` contains integer `module_count`, `total_applied_changes`, and -`total_skipped_changes`, plus optional string `failed_module_instance`. -`total_skipped_changes` counts both apply-time skipped changes and validator -rejections. Their detailed records are stored in separate arrays. - -Each `module_results` object contains: - -| Field | Type and meaning | -| --- | --- | -| `module_key`, `module_instance` | Strings; the instance distinguishes repeated occurrences of a configured module | -| `replacement_policy` | Optional string describing the module's replacement policy | -| `status` | String describing module outcome (`success` or `failed`) | -| `proposal_count` | Integer | -| `started_at`, `completed_at` | Optional timestamps | -| `error_message` | Optional failure detail | -| `warnings` | Optional array of warning records | -| `validator_decisions` | Optional array of decision records | -| `validator_rejected` | Optional array of rejected-correction records | -| `applied_changes`, `skipped_changes` | Optional arrays of apply outcomes | - -A validator decision has `validator_name` (string), `proposal_index` (integer), -`approved` (boolean), `reason_code` and `message` (strings), and optional -`diagnostic_artifact_path` (string). - -Rejected corrections include `validator_name`, `proposal_index`, `module_key`, -`module_instance`, `target_segment_id`, `original_text`, `corrected_text`, -`reason_code`, and `message`. Applied changes include the same correction identity -and text fields without validator/reason/message fields, plus integer -`replacement_count`. Skipped changes include correction identity/text fields, -`skip_reason`, and `message`. - -A warning has string `scope`, `reason_code`, and `message`; optional integer -`section_index` and `batch_index`; and optional string `validator_name` and -`diagnostic_artifact_path`. Warnings, skipped changes, and rejected corrections -can occur without a failed run. Proposal indexes associate outcomes with their -proposal within a module instance; they are not transcript segment IDs. - -## Correction Ledger - -The ledger is an array of correction entries; it can be `null` when no module -results are available and empty when modules produced no entries. Unlike the -process report, it has no separate schema-version envelope. - -Each entry has required `module_key`, `module_instance`, integer `proposal_index`, -and `disposition`. The current dispositions are: - -| Disposition | Meaning | -| --- | --- | -| `applied` | The proposed change was applied | -| `skipped` | The proposal reached application but was skipped | -| `rejected` | A validator rejected the proposal | -| `failed` | Module-level failure record, not necessarily an individual proposal | - -Optional fields are `run_id`, integer `segment_id`, `original_text`, -`proposed_corrected_text`, `applied_corrected_text`, `replacement_policy`, -`disposition_reason_code`, and `disposition_message`. Optional validator arrays -are `deterministic_validator_decisions` and `llm_validator_decisions`; each record -contains string `validator_key`, boolean `approved`, string `reason_code`, and -optional string `message`. - -Module failure entries can have the zero-value proposal index and no correction -text or segment. Do not interpret them as evidence that proposal zero failed. -Empty optional values are omitted. The ledger records disposition, not a replay -or resume protocol. - -## Utilization Diagnostics - -`llm_summary` contains integer `logical_calls`, `provider_attempts`, -`transient_retries`, and `structural_repairs`. Its `usage` contains `prompt`, -`completion`, `total`, `cached`, and `cache_write`, each with integer `value` -and boolean `known` and `complete`. A call counts once even if preparation fails. -If no call dispatches, usage is known-zero and complete. Missing `llm_summary` -means observations were unavailable. - -The unversioned utilization JSON contains `backends`, keyed by backend ID. Each -entry has configured `concurrency` and `queue_capacity`, the same execution -summary fields, and integer `capacity_wait_ms`, `generation_ms`, -`retry_delay_ms`, and `logical_wall_ms`. It also has `run_wall_time_ms` and -optional `modules` and `validators` arrays. Module records contain the module -identity, logical proposal/validation call counts, and `module_wall_time_ms`. -Validator records contain module/validator identity, `llm_backed`, `calls`, and -`elapsed_ms`. Timings can overlap; they do not decompose total wall time. - -`llm-execution.json` has `calls` and `events` arrays joined by `call_id`. Calls -include logical attribution, final classification, summary, elapsed milliseconds, -and available prompt/profile/model metadata. Events contain safe execution -metadata, including attempt and terminal records, with millisecond durations. -Preparation failures have a call and no fabricated events. This artifact has no -provider messages, endpoints, credentials, or transcript content. Other request -and response artifacts can contain redacted prompt and transcript content. +With `--debug`, `correction-ledger.json` contains proposed candidates and +their application, rejection, or interruption details. `llm-execution.json` +holds logical calls and PromptKit execution events; `utilization-diagnostics.json` +groups backend capacity and timing observations while preserving known and +complete usage semantics. Interaction request/response artifacts, selected +profiles, configuration provenance, and input snapshots are also debug-only. +These files are diagnostic evidence, not an alternate ordinary report schema. diff --git a/docs/integrations/subprocess.md b/docs/integrations/subprocess.md index 32bb313..29ac78b 100644 --- a/docs/integrations/subprocess.md +++ b/docs/integrations/subprocess.md @@ -8,8 +8,10 @@ The [CLI reference](../cli.md) owns command syntax, streams, and exit semantics; Use an argument array with the transcript path and required glossary argument, following the [process examples](../cli.md#commands-and-help). For machine workflows, -select explicit transcript and report destinations rather than depending on the -working directory or run-directory retention. Supplying a configuration path also +select an explicit transcript file destination and a configuration path. Reports +then go beside the transcript. For stdout output, select `--report-dir` or use +the invocation working directory. Debug is independent and off by default. +Supplying a configuration path also makes file selection predictable; only explicitly referenced credential variables are read from the environment. @@ -24,15 +26,20 @@ Interpret the exit result according to the [CLI output and exit contract](../cli.md#streams-and-exit-status). Consume transcript/report data through the [output-file contract](output-files.md), and treat stderr as human-readable detail. -The report supplements the exit result: it may be absent after early failures, -and output may have been written before a later reporting or cleanup failure. +The three reports may be absent after early configuration or destination +failures. After initialization, failed reports can describe invalid input or +partial processing. Exit `0` can include recoverable model errors and applied +corrections; inspect `errors.yml` and `recoveries.yml` for those outcomes. A +transcript may have been written before a later required report failure, so +the file alone does not prove completion. If processing fails, preserve available evidence and follow [operations recovery](../operations.md#failure-inspection-and-recovery). -For reports retained outside the run directory, account for the -[artifact lifecycle](../operations.md#retention-behavior) before dereferencing -reported paths. Neither the existence of an output file nor a diagnostics path -alone establishes successful completion. +Ordinary reports are independent of optional retained debug evidence. Parent +cancellation reaches the child through its context; Audita joins workers and +attempts failed-status report finalization before exiting. Uncatchable +termination can leave `in_progress` reports. Neither the existence of an output +file nor a debug directory alone establishes successful completion. ## Related Contracts diff --git a/docs/internal/diagnostics-reporting.md b/docs/internal/diagnostics-reporting.md index 3c128c0..177452d 100644 --- a/docs/internal/diagnostics-reporting.md +++ b/docs/internal/diagnostics-reporting.md @@ -1,91 +1,42 @@ -# Diagnostics and Report Assembly +# Diagnostics and Reporting Internals -## Scope +The [output-file contract](../integrations/output-files.md) owns public +schemas and the [operations guide](../operations.md) owns physical placement. -This document owns implementation flow and collaborators. Physical artifacts and -retention belong to [operations](../operations.md). Serialized reports, ledger -records, and utilization shapes belong to the -[output-file contract](../integrations/output-files.md). +`internal/framework/outcomes` contains processing-owned operation references, +counts, failure reasons, filtering details, and work states. It imports no +reporting, CLI, or provider package. The runner records those facts per initial +section and module instance. `internal/framework/processreport` maps them once +into `internal/core/reporting.Snapshot`, including nullable counts, grouped +work-unit errors, separate run errors, and one recovery per successful logical +operation. Reports order stages, sections, validators, and batches by semantic +identity, never concurrent completion time. The core reporting package owns +Audita's versioned YAML DTOs, fixed Markdown rendering, and brief CLI +completion formatter. YAML redaction walks serialized string nodes so numeric +and boolean values retain their types. Ordinary reports omit raw causes, +transcript text, complete decisions, and token/timing payloads. -## Run Directory and Artifact Writers +`internal/core/diagnostics.OutputSession` owns active destination checks, +transcript preflight, report initialization, and per-file final replacement. +CLI composes it after configuration/profile inspection and before reading +inputs or dispatching model work. It does not create ordinary report parents. +On a final write error it attempts other enabled reports and one failed-status +pass; a published transcript is never erased by a later report failure. -`internal/core/diagnostics` creates run directories, writes snapshots and metadata, -and evaluates retention. CLI orchestration supplies the selected configuration, -paths, run outcome, and correction-presence information. Artifact names are -centralized in diagnostics; their operator-facing inventory is in -[operations](../operations.md#diagnostics-directory). - -Snapshot writes are attempted at the relevant processing phases. Several diagnostic -writes are best-effort, so report metadata can identify a location without proving -that a file was written. The CLI distinguishes primary processing errors from -secondary diagnostic/report write errors. - -## Process Report Mapping - -`internal/framework/processreport.Build` maps invocation facts, normalization and -chunking summaries, and runner output into `reporting.ProcessReport`. Core reporting -owns the Go data structures and JSON writer. The builder attaches diagnostic -metadata when a run-directory path is available. - -Module mapping copies outcomes from runner results, collects summary counts, and -preserves available partial module results on failure. The CLI failure path does -not pass normalization or chunking summaries to report construction even when -some earlier processing completed. Consumer interpretation belongs to the -[process-report contract](../integrations/output-files.md#process-report). - -## Correction Ledger and Utilization - -`BuildCorrectionLedger` flattens module changes, rejections, and failures into -ledger entries. It associates validator records by proposal index within each -module, separates execution classes through validator metadata, and sorts entries -by module instance, proposal index, and disposition. - -Runner instrumentation and PromptKit observations collect utilization; CLI -orchestration writes the backend summary and safe call/event artifact when run -output is available. Report assembly maps the same observations to `llm_summary` -without independently measuring calls. A selected backend remains attributable -when preparation fails before dispatch. Domain module/validator timing stays -separate from provider timing, and those durations can overlap. -For the serialized forms see [ledger](../integrations/output-files.md#correction-ledger) -and [utilization](../integrations/output-files.md#utilization-diagnostics). - -## CLI Coordination and Retention - -CLI orchestration writes caller-selected reports and run-local artifacts before -asking diagnostics to apply retention. `HasSkippedCorrections` derives the -retention input from runner outcomes. Keep the exact lifecycle definition in -[operations](../operations.md#retention-behavior), not in another internal table. - -On a processing failure, secondary report-write errors do not replace the original -error. On the success path, an external report-write failure returns before the -run-local report write; retention errors occur later. This ordering explains why -artifacts and process exit status must be assessed together. - -## Redaction - -Effective configuration uses ConfigKit redaction. The CLI supplies all resolved -backend credentials, including unselected backends, to report and diagnostic -writers. `core/redaction` sanitizes the serialized representation rather than -mutating processing results. JSON string values and keys are redacted after -parsing with number preservation; YAML redaction preserves scalar types. -Serialization failure returns an error without writing the original object. -Interaction artifacts, correction ledgers, execution/utilization diagnostics, -selected profiles, both report copies, and surfaced LLM errors share this -string-redaction implementation. Longer overlapping values take precedence, -and replacement text is not processed again within a redaction pass. - -Execution reports index events by call ID. Utilization groups calls directly by -backend and actual module instance, keeping module result order and retaining -usage knownness/completeness. Section and batch indices remain separate from -module identity; section-qualified interaction filenames avoid collisions. -These boundaries implement the [architecture requirement](../policy/architecture.md#errors-and-secret-handling). -Operational handling of transcript-bearing artifacts belongs to -[operations](../operations.md#artifact-handling). +Debug is independent. Only `diagnostics.debug` creates a unique retained child +under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers +when debug is off. With debug on, source/normalized snapshots, invocation, +effective settings, origins, profiles, initial chunk summary, correction ledger, +indexed execution/utilization, and model interactions are written. Runner +candidate capture includes proposals later rejected or interrupted. Debug +write failures produce safe stderr messages and do not abort processing. +Every debug writer uses all configured backend secrets for redaction, including +unselected backend credentials and source snapshots. Debug can still contain +sensitive transcript and prompt content. ## Key Tests -- `internal/core/redaction/redaction_test.go` -- `internal/core/diagnostics/*_test.go` -- `internal/framework/processreport/*_test.go` -- `internal/core/reporting/report_test.go` -- `cmd/audita/main_integration_test.go` +Useful tests: `internal/core/reporting/*_test.go`, +`internal/core/diagnostics/*_test.go`, +`internal/framework/processreport/*_test.go`, and +`cmd/audita/main_integration_test.go`. diff --git a/docs/internal/llm-runtime.md b/docs/internal/llm-runtime.md index 2240cc9..327676d 100644 --- a/docs/internal/llm-runtime.md +++ b/docs/internal/llm-runtime.md @@ -6,7 +6,7 @@ prompt and schema assets, registers application backends, and inspects the selected proposal and validation profiles offline. A role-bound `llm.RoleClient` implements `structuredoutput.Client`; proposal and validator contracts alias that -interface during migration. Both roles share engine-local backend capacity. +interface. Both roles share engine-local backend capacity. For each logical call, the adapter starts an operation context and a diagnostic record before preparation. It supplies the reserved hardening text, the selected @@ -15,12 +15,13 @@ contract. `PrepareExecution` produces the exact messages and hashes retained in an Audita snapshot. The adapter checks the prepared backend and endpoint before `RunPrepared`. It records available final content, usage, model, and profile metadata and decodes only a validated result. Call hooks expose logical -start/finish records and PromptKit execution events for the report collector. +start/finish records and PromptKit execution events for debug utilization. The [integration contract](../integrations/openai-compatible-llm.md) describes wire ownership and failure behavior. `structuredoutput.Failure` has a safe -message and a typed cause; only exhausted structural repair is a malformed -output failure. Secrets from all configured backends, including unselected +message, fixed reason code, typed cause, and recoverable/fatal/canceled class. +Counts distinguish provider attempts, transient retries, and structural +repairs, including on errors. Secrets from all configured backends, including unselected ones, are available to the diagnostics redaction boundary. ## Orchestration @@ -29,10 +30,10 @@ CLI processing supplies two role clients from the same engine to the runner. PromptKit's per-backend admission applies to both roles and every generation, retry, and repair. The runner still processes module instances in order, collects concurrent section work deterministically, and cancels remaining -section work on a runtime failure. `llm.DiagnosticsWriter` remains the protected +section work only on fatal failures. `llm.DiagnosticsWriter` remains the protected request/response artifact writer. `llm.ObservationCollector` captures metadata events under a short mutex and freezes them after runner work joins. Its logical records include preparation failures; summary usage comes from terminal events once per call and retains knownness and completeness for each field. Runner wall -and validator timings remain separate from provider attempts. The report and -artifact mapping is completed by the following roadmap work. +and validator timings remain separate from provider attempts. Ordinary reports +derive compact runner outcomes; full observations are debug-only. diff --git a/docs/internal/modules.md b/docs/internal/modules.md index 9a417a6..01fc639 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -49,12 +49,13 @@ policy. Shared context encoding lives in `internal/framework/promptcontext`. Built-in chains are resolved in `internal/validators` per module key. Module packages call the built-in chain resolver at construction. -## Failure and Warning Behavior +## Failure and Outcome Behavior -Module construction failures propagate to orchestration; execution failures can -preserve partial module results. Exhausted structural repair becomes a warning -and section-level rejection; provider and protocol failures propagate. Consumer-visible outcomes are defined by -the [report contract](../integrations/output-files.md#module-outcomes). +Module construction failures are fatal. Recoverable proposal model failures +record an unsuccessful operation and yield no candidates for that section; +other sections and module instances continue. Fatal adapter or application +failures cancel peers and retain partial observations. Consumer-visible +outcomes are defined by the [report contract](../integrations/output-files.md). ## Changing Modules diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 7a666ba..ae5ac12 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -47,7 +47,8 @@ Supporting areas: ## Current Scope -Runtime state is held in memory plus per-run diagnostics artifacts. Audita does +Runtime state is held in memory; optional debug artifacts are retained only +when requested. Audita does not currently implement a plugin framework, generic workflow engine, resume/checkpoint/manifest system, remote storage, or multi-process service mode. Its role-bound PromptKit adapter implements the structured LLM client contract. @@ -55,7 +56,9 @@ These are current implementation boundaries, not bans on future work. ## Main Execution Path (`audita process`) -The CLI prepares input and configuration, invokes the runner, and writes results. +The CLI inspects configuration and profiles, preflights and initializes active +outputs, optionally creates debug evidence, invokes the runner, and finalizes +transcript and reports. The [pipeline](pipeline.md) owns detailed state transitions; the [operations guide](../operations.md) owns the operator workflow and physical artifact lifecycle. diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index 4a6e3ee..5db45db 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -1,66 +1,44 @@ # Audita Internal Pipeline -## Scope +The [input contract](../integrations/transcript-glossary-files.md) owns input +formats; [configuration](../config.md) owns settings; the +[output contract](../integrations/output-files.md) owns serialized results. -This document describes processing collaborators and state transitions. External -input definitions belong to the [input-file contract](../integrations/transcript-glossary-files.md), -settings to [configuration](../config.md), and serialized results to the -[output-file contract](../integrations/output-files.md). +1. CLI filesystem adapters read transcript and glossary files. Schema parsing + produces source objects; transcript segments enter canonical normalization. +2. Normalization sorts and merges segments using configured time and token + limits, reassigning IDs where needed. +3. The runner partitions the normalized transcript exactly once before its + module loop. Initial section identity, segment membership, and token + estimates stay fixed while text evolves. +4. Module instances execute in configured order. Sections may propose + concurrently, but proposal indices follow section order and response order. +5. Deterministic validators precede LLM-backed validators. Every submitted + candidate receives one decision per validator. Recoverable failed batches + mark their candidates unavailable; successful survivors continue through + all required validators. +6. Only fully approved proposals reach application. Corrections apply in + proposal-index order within their owning section. Existing replacement + policies and apply-time safety checks resolve stale conflicts. +7. CLI encodes the final transcript and publishes active files through the + output session. `framework/processreport` builds one snapshot for YAML, + Markdown, and CLI totals. Debug evidence is captured separately when enabled. -## Inputs +`runner.Run` returns the final working transcript, initial section inventory, +explicit `not_started`/`interrupted`/`completed` work units, compact model +operation outcomes, and per-module results. Candidate payloads are retained +in the returned output only when debug capture is requested. A work unit can +complete with model errors and applied corrections. Recoverable model failures +never trigger shared cancellation. Fatal setup, internal, application, or +parent-cancellation errors cancel peers, join workers, and preserve earlier +committed module text for partial reports; no completed transcript is published. -Pipeline inputs are: -- effective config (`internal/core/config`); -- transcript JSON (`internal/core/schema`); -- glossary YAML (`internal/core/schema`). +Required transcript/report write failures are fatal. Optional debug writes +report safely to stderr without changing decisions or exit status. See +[operations](../operations.md) for file placement and failure handling, and +[architecture policy](../policy/architecture.md#correction-and-concurrency-invariants) +for invariants. -## Pipeline Phases - -1. **Input loading:** CLI filesystem adapters read inputs; schema parsing produces - source transcript and glossary objects. Source segments are converted to the - canonical transcript shape before normalization. -2. **Normalization:** the normalizer sorts and merges segments using its configured - time/token rules, reassigns IDs, and produces summary counters. -3. **Chunking:** the chunker partitions the normalized transcript and returns - sections and token estimates, or fails when constraints are impossible. -4. **Proposal generation:** the runner executes configured module instances in - sequence. Sections can generate proposals concurrently; results are collected - for deterministic indexing. -5. **Validation:** deterministic validators run before LLM-backed validators. - LLM validation is batched by prompt budget; candidate decisions feed application. -6. **Application:** `internal/framework/proposals` clones transcript state and - applies approved proposals in index order through replacement-policy checks. -7. **Serialization and reporting:** the CLI encodes the final transcript, maps - runner output into reports, writes artifacts, and delegates retention to - diagnostics. See [report assembly](diagnostics-reporting.md). - -## Runner Outputs - -`runner.Run` returns: -- final transcript; -- per-module results. - -CLI/reporting then map these results into process reports and diagnostics -artifacts. PromptKit execution observations are collected at the role-client -boundary for the new report summary. - -## Failure Behavior - -CLI orchestration labels failures at the phase boundary. Runner execution failures -can return completed or partial module results, which report assembly preserves. -[Report fields](../integrations/output-files.md#process-report) define how consumers -see these outcomes; [CLI semantics](../cli.md#streams-and-exit-status) -define streams and exits. Diagnostic write failures do not always abort processing. - -## Ordering and State - -The runner mutates its working transcript between module instances, after collecting -section results and validating proposals. Proposal indexes follow section order; -reports are derived from actual run results. Global requirements are owned by the -[architecture policy](../policy/architecture.md#correction-and-concurrency-invariants). - -## Key Tests - -- `internal/framework/runner/runner_test.go` -- `internal/framework/proposal_generation/*_test.go` -- `cmd/audita/main_integration_test.go` +Key tests: `internal/framework/runner/*_test.go`, +`internal/framework/validators/*_test.go`, and +`cmd/audita/main_integration_test.go`. diff --git a/docs/internal/validators.md b/docs/internal/validators.md index 78f3db4..58a1072 100644 --- a/docs/internal/validators.md +++ b/docs/internal/validators.md @@ -42,7 +42,7 @@ Glossary, homophones, spoken_word, and grammar each resolve a fixed ordered chai For each module section: 1. run deterministic validators; 2. run LLM-backed validators; -3. record decisions and warnings; +3. record decisions and compact operation outcomes; 4. carry only approved proposals forward. Decision cardinality is enforced: each candidate proposal must receive exactly one decision per validator. @@ -55,15 +55,16 @@ LLM validators: - select an embedded prompt ID and pass payload/description variables to the role-bound structured client with the validator-decision schema. -Oversized single proposals are rejected with `validator_input_too_large`. -Exhausted structural repair becomes a warning and rejected batch decisions. -Provider, malformed HTTP envelope, deadline, and admission failures propagate -as runtime errors. Schema-valid decisions still undergo exact index/cardinality -checks and domain acceptance without a semantic retry. +Oversized single inputs are rejected with a consolidated +`validation_input_too_large` operation, without a provider call. Recoverable +model failures make only their batch candidates unavailable while other +batches continue. Missing, duplicate, or unknown decision indices invalidate +that batch as a whole. Fatal errors stop the chain without returning unchecked +candidates as approved. Domain acceptance has no semantic retry. ## Decision and Rejection Reporting -Runner collects decisions, rejected candidates, and warning records. Ledger +Runner collects decisions, rejected candidates, and operation outcomes. Ledger assembly groups decisions by deterministic versus LLM-backed execution metadata. The serialized names and fields belong to the [report and ledger contract](../integrations/output-files.md); implementation mapping diff --git a/docs/migrations/v2.md b/docs/migrations/v2.md index 494198b..656e9ca 100644 --- a/docs/migrations/v2.md +++ b/docs/migrations/v2.md @@ -19,8 +19,11 @@ Settings resolve from defaults, main file, optional `--overlay ` in environment overlays no longer apply. Unknown fields and version-1 files fail; there is no automatic converter. -Keep transcript as the positional path and `--glossary`, `--output`, and -`--report-json` as invocation paths. Keep `pipeline.modules` / `--modules` and +Keep transcript as the positional path and `--glossary` and `--output` as +invocation paths. The former `--report-json` output has been replaced by the +default `errors.yml`, `recoveries.yml`, and `report.md` bundle; use +`--reporting=false` to disable it or `--report-dir` for stdout output. Keep +`pipeline.modules` / `--modules` and `output.schema` / `--output-schema`. An explicit `pipeline.modules: []` or `--modules ''` runs no correction modules; omission uses the default sequence. Ordered repetitions still matter. @@ -73,14 +76,16 @@ The following domain settings and explicit flags retain their meaning: | `normalization.max_segment_gap`, `.ellipsis_gap`, `.max_segment_duration`, `.max_segment_tokens` | `--normalize-max-segment-gap`, `--normalize-ellipsis-gap`, `--normalize-max-segment-duration`, `--normalize-max-segment-tokens` | | `thresholds.glossary`, `.homophones`, `.spoken_word`, `.grammar` | `--glossary-confidence-threshold`, `--homophones-confidence-threshold`, `--spoken-word-confidence-threshold`, `--grammar-confidence-threshold` | | `context.description` | `--transcript-description` | -| `diagnostics.work_dir`, `.retention` | `--work-dir`, `--work-dir-retention` | +| `reporting.enabled`, `.stdout_dir` | `--reporting[=true\|false]`, `--report-dir` | +| `diagnostics.debug`, `.work_dir` | `--debug[=true\|false]`, `--work-dir` | Normalization gaps and durations are **numeric seconds**, so replace `4s`, `3.5s`, and `60s` with `4.0`, `3.5`, and `60.0`. The operation timeout remains a Go duration string. `chunking.target_sections` may be omitted or null for automatic chunking; zero is invalid. Null is not a general synonym for omission: nonoptional scalar settings reject it. Empty `context.description` clears -context. Retention behavior is unchanged. +context. Legacy `diagnostics.retention` and `--work-dir-retention` are removed. +Debug is off by default and its directory is retained whenever enabled. Remove old implicit variables `AUDITA_MODULES`, `AUDITA_MODEL`, `AUDITA_VALIDATION_MODEL`, `AUDITA_BASE_URL`, `AUDITA_VALIDATION_BASE_URL`, @@ -109,14 +114,14 @@ calling a provider. `--overlay testing` selects the same overlay for validation, inspection, and processing. The old effective-config JSON format is replaced by YAML; settings YAML alone does not show profile-owned model controls. -The [process report](../integrations/output-files.md#process-report) now uses -schema `v2` and includes an `llm_summary` when execution observations are -available. Logical calls are distinct from provider attempts; usage fields carry -known and complete flags. Utilization is grouped by backend capacity rather than -old scheduler and active-concurrency fields. Retained runs write +The [ordinary output contract](../integrations/output-files.md) now uses +version-1 `audita-errors` and `audita-recoveries` YAML documents plus a fixed +Markdown overview. Compact outcomes distinguish logical operations from +provider attempts. With debug enabled, utilization remains grouped by backend +capacity and usage fields retain known/complete flags. Debug runs write `effective-config.redacted.yaml`, `selected-profiles.yaml`, `config-origins.yaml`, `utilization-diagnostics.json`, and `llm-execution.json`. Request and response artifacts remain redacted debugging -material. See [operations](../operations.md#diagnostics-directory) for retention +material. See [operations](../operations.md#diagnostics-directory) for placement and paths, [configuration](../config.md) for all settings, and the [CLI reference](../cli.md) for current flags. diff --git a/docs/operations.md b/docs/operations.md index 166da31..a5ec4bd 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,113 +1,73 @@ # Audita Operations -This document owns runtime workflow, physical artifact placement, retention, and -recovery. Use the [CLI reference](cli.md) for invocation and stream/exit semantics, -[configuration](config.md) for settings, and [output-file contracts](integrations/output-files.md) -for logical formats. +This guide owns physical output placement and failure handling. The +[CLI reference](cli.md) owns flags and streams, [configuration](config.md) owns +settings, and the [output contract](integrations/output-files.md) owns schemas. ## Normal Workflow -1. Choose a configuration and validate it using the - [configuration commands](cli.md#commands-and-help). -2. Prepare input files conforming to the [input contract](integrations/transcript-glossary-files.md). -3. Run the [processing command](cli.md#commands-and-help), choosing explicit output - and report destinations when those files must survive run-directory cleanup. -4. Inspect the corrected transcript and process report. Review rejected or skipped - corrections when the report indicates them; successful execution does not mean - every proposed correction was applied. +1. Validate a version-2 configuration and its selected profiles with + `audita config validate --config config.yml`. +2. Prepare a transcript and glossary following the [input contract](integrations/transcript-glossary-files.md). +3. Run `audita process input.json --config config.yml --glossary glossary.yaml --output corrected.json`. +4. Check the exit status, stderr summary, corrected transcript, and reports. + A completed run can contain recoverable model errors and still apply + corrections that passed every required validator. -Processing creates a run directory, loads and normalizes the inputs, chunks the -transcript, executes the correction pipeline, writes results, and applies -retention. Early configuration failures happen before run-directory creation. -Failures after initialization attempt to preserve diagnostic evidence; individual -artifact writes can fail, so a run directory need not contain every listed file. +## Required Output Files -## Output Files +Reporting is enabled by default. For file transcript output, `errors.yml`, +`recoveries.yml`, and `report.md` go in the transcript output's parent directory. +For stdout transcript output, they go in `reporting.stdout_dir` (default `.`), +overridden by `--report-dir`. `--reporting=false` suppresses all three reports +and removes their path requirements. It does not suppress the stderr summary or +fatal errors. Stdout contains only transcript JSON; file output leaves it empty. -The caller chooses transcript and optional external report destinations using the -[CLI](cli.md#streams-and-exit-status). These are separate from the per-run -diagnostic directory unless the caller deliberately places them inside it. -Choose existing, writable parent directories. Output/report files can be overwritten -by subsequent runs using the same destinations. - -A later report or cleanup failure can occur after transcript output was written. -Inspect the process result and available report before treating the files as a -completed run. Retention of the diagnostic directory does not archive caller-selected -output files elsewhere. +Existing report directories and transcript-output parents must be writable. +Audita checks active destinations against input, configuration, and selected +profile files before model dispatch. It preflights file transcript output +without truncating an existing file. With reporting enabled, all three reports +are initialized as `in_progress` before input processing. Output is encoded +fully before publication. Each file is replaced through a temporary file in +its destination directory: transcript first, then `errors.yml`, +`recoveries.yml`, and `report.md`. This is per-file replacement, not a +multi-file transaction. A late report error may leave the transcript published; +the process exits nonzero and attempts failed-status reports. On fatal processing +or cancellation, no completed transcript is published, but partial reports are +finalized where possible. An uncatchable termination can leave `in_progress`. ## Diagnostics Directory -The [diagnostics configuration](config.md#settings) -selects the parent work directory. Runs create children named like -`run-`. The runtime creates the work directory if needed. -Report paths may be relative when the supplied work directory is relative; -resolve them using the invocation's working directory. +`--debug` enables detailed evidence independently of reporting. One directory +named `run-` is created under `diagnostics.work_dir` (default `/tmp/audita`) +and retained on success or failure. Without debug, no directory or payload +files are created. The parent is created when needed only for debug. Failure to +write optional debug evidence prints a redacted stderr diagnostic but does not +change correction decisions or the exit status. -| Artifact in the run directory | Purpose | -| --- | --- | -| `source-transcript.json` | Original transcript input | -| `source-transcript-parsed.json` | Parsed source representation | -| `normalized-transcript.json` | Transcript after normalization | -| `normalization-summary.json` | Normalization counters | -| `chunking-summary.json` | Section/token summary | -| `invocation.json` | Invocation metadata | -| `effective-config.redacted.yaml` | Redacted ConfigKit configuration snapshot | -| `config-origins.yaml` | Safe configuration origin and resolution metadata | -| `selected-profiles.yaml` | Safe selected role profiles and model controls | -| `report.json` | Run's [process report](integrations/output-files.md#process-report) | -| `correction-ledger.json` | [Correction dispositions](integrations/output-files.md#correction-ledger) | -| `utilization-diagnostics.json` | [Backend execution and domain timing observations](integrations/output-files.md#utilization-diagnostics) | -| `llm-execution.json` | Safe logical-call and execution event records | -| `error.log` | Failure details when written | - -LLM request, response, and error diagnostics are also written within the run -directory. Their nested naming and payloads are debugging details. Prefer paths -provided by the report and its decision/warning records over reconstructing those -paths. The report describes intended artifact locations, not proof that every -write succeeded or that the files still exist after retention. - -## Retention Behavior - -[Configuration](config.md#settings) owns accepted -retention values and the default. Current lifecycle behavior is: - -| Run outcome / selection | Directory outcome | -| --- | --- | -| Failed run | Retained regardless of selection | -| Successful run, `always` | Retained | -| Successful run, `auto` | Retained if corrections were skipped or rejected; otherwise removed | -| Successful run, `never` | Currently retained, the same outcome as `always` | - -The name `never` does not currently mean deletion. Warnings alone do not trigger -retention under `auto` unless they also produce skipped or rejected corrections. -An external report outside the run directory survives automatic cleanup, but its -artifact paths can then point to removed files. - -## Artifact Handling - -Run artifacts can contain full transcripts, glossary-related context, and LLM -prompts/responses. Configured-key redaction does not anonymize those contents. -Choose a work directory whose ownership and permissions fit the transcript data, -and retain or share artifacts intentionally. Credential-supply mechanisms are -owned by [configuration](config.md#credentials). - -For a debugging run, select retained diagnostics using the -[retention flag](cli.md#setting-overrides). Copy needed artifacts before -removing completed run directories manually. Avoid deleting a directory while -its run is active. Audita does not provide a separate archival/backup or remote -artifact storage facility. +Debug evidence includes invocation and effective settings, configuration +origins, selected profiles, redacted source and normalized snapshots, +normalization and initial chunk summaries, all captured proposed candidates +and validator decisions in `correction-ledger.json`, execution/utilization +records, and model interaction files. Individual artifact writes can fail; a +directory's existence does not prove every file was written. These files can +contain transcript text and prompt/response payloads. Configured backend +credentials are redacted, but the transcript itself is not anonymized. Protect +and retain the directory according to the sensitivity of the input. ## Failure Inspection and Recovery -1. Read stderr and the exit classification defined by the [CLI](cli.md#streams-and-exit-status). -2. Locate the external report, or the run directory identified by the error when - available. If a late error omits the path, inspect the selected work directory. -3. Read the available report and error log. Use - [report status and error fields](integrations/output-files.md#process-report) - to identify the failed phase, then inspect referenced artifacts. -4. Correct the concrete input, configuration, endpoint, or filesystem issue and - rerun the command with diagnostics retained if further investigation is needed. +Exit `0` means processing and every enabled required output completed. Exit +`1` means cancellation, fatal processing, or required-output failure. Exit `2` +identifies usage or configuration/destination selection errors. Recoverable +model-operation failures appear in `errors.yml` as one record per affected +work unit; they do not automatically make the run fail. `recoveries.yml` +records successful operations that needed retry, repair, or proposal filtering. +`report.md` shows every initial chunk and configured module instance, including +interrupted and unattempted work. Ordinary reports omit transcript excerpts and +full decision ledgers; enable debug when those are needed. -Each invocation starts a new run. There is no implemented resume/checkpoint API; -recovery is a rerun, not continuation from an artifact. Failures during early -configuration or directory creation may leave no report or diagnostic files. +Read stderr and report status first. If a required output failed after a +transcript was published, do not treat the file alone as proof of a completed +run. Fix the input, service, or destination problem and rerun. Audita has no +resume/checkpoint API. diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index a526735..f0568c4 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -2,7 +2,7 @@ **Base Policy Version:** 1.0 -**Last Updated**: 2026-09-23 +**Last Updated**: 2026-09-24 ## Purpose and System Shape @@ -48,7 +48,9 @@ hidden global state. - Keep LLM concurrency bounded and context-aware through PromptKit's engine-local per-backend admission. Proposal and validation calls selecting the same backend share that capacity, including retries and repairs. Preserve deterministic - collection order and cancellation of remaining work on runtime failure. + collection order. Recoverable model-operation failures stay local to their + proposal response or validation batch; fatal invariant/application failure + and parent cancellation stop peers and join them. - Keep module and validator identities, reason codes, and execution-class distinctions stable where exposed as contracts. @@ -61,12 +63,14 @@ describe how these requirements are implemented. Keep transcript mutation and output handling deterministic around external calls. Reports and correction ledgers are derived from processing results. Preserve the separation between transient processing state and filesystem -adapters; physical artifact placement and retention belong to +adapters; physical report placement and optional debug retention belong to [operations](../operations.md). Public CLI, configuration, output-schema, diagnostics, report, prompt, module, and validator contracts should remain stable unless a change is deliberate and -tested. Prefer additive, compatible report changes. Current implementation +tested. The resilience work intentionally replaced the earlier JSON process +report with versioned errors/recoveries YAML and a fixed Markdown overview; +future changes should treat those current contracts deliberately. Current implementation limits are documented in the [internal overview](../internal/overview.md#current-scope); they are not additional prohibitions on future development. diff --git a/docs/roadmap/configkit.md b/docs/roadmap/configkit.md index f929e54..bd96a90 100644 --- a/docs/roadmap/configkit.md +++ b/docs/roadmap/configkit.md @@ -3,7 +3,7 @@ **Last updated:** 2026-09-24 **Status:** Production implementation complete. The selected configuration and -overlay model is implemented; [implementation evidence](implementation.md) +overlay model is implemented; [implementation evidence](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) records verification. No release has been published. ## Purpose and Direction @@ -137,7 +137,7 @@ ConfigKit and PromptKit form one intentional breaking refactor. Intermediate commits need not build or run. The target contains no temporary old-runtime adapter, compatibility shim, processing gate, dual loader, or automatic legacy conversion. The [combined feature roadmap](library-adoption.md) owns execution/reporting intent; -[implementation.md](implementation.md) alone owns stage order and implementation tasks. +[implementation.md](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) alone owns stage order and implementation tasks. The final application loads all retained domain settings, assembles both selected profiles, and processes through PromptKit. Controlled HTTP evidence must connect diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 7749709..eec3e31 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -782,6 +782,17 @@ has no obsolete report/retention path keeping debug enabled implicitly. ## Stage 13 — Update Canonical Guidance, Examples, and Remove Obsolete Material +**Completed 2026-09-24.** Canonical CLI, configuration, operations, integration, +internal, architecture, README, example, and migration guidance now describes +three ordinary reports, independent debug selection, partial outcomes, fixed +matrices, and finalization/exit behavior. The obsolete retention follow-up is +resolved while timestamp policy remains open. Historical library-adoption +links point to the prior plan's completion evidence. The unreferenced warning +package and retention-only helper/tests were removed. Maintained config and +profile inspection examples passed offline, relative file and heading links +resolved, and `go test ./internal/framework/processreport +./internal/core/config ./internal/cli` plus `git diff --check` passed. + **Work:** Update `docs/cli.md`, `config.md`, `operations.md`, integration output/subprocess/ LLM contracts, internal pipeline/validators/LLM/diagnostics/overview guidance, README, and maintained examples/migration guidance. Architecture policy must distinguish diff --git a/docs/roadmap/library-adoption.md b/docs/roadmap/library-adoption.md index a653250..744c5f6 100644 --- a/docs/roadmap/library-adoption.md +++ b/docs/roadmap/library-adoption.md @@ -3,8 +3,9 @@ **Last updated:** 2026-09-24 **Status:** Production implementation complete. ConfigKit v0.4.0 and PromptKit -v0.11.0 are adopted; [implementation evidence](implementation.md) records the -completed checks. Release compliance and publication remain separate work. +v0.11.0 are adopted; [historical implementation evidence](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) records the +completed checks. The current `implementation.md` is a separate resilience +roadmap. Release compliance and publication remain separate work. ## Goal and Ownership diff --git a/docs/roadmap/library-boundary-findings.md b/docs/roadmap/library-boundary-findings.md index d3718c0..4f9e797 100644 --- a/docs/roadmap/library-boundary-findings.md +++ b/docs/roadmap/library-boundary-findings.md @@ -312,12 +312,12 @@ claims of completion for the prototype. The new profile/credential/capacity and deadline choices are explicit above; no unresolved upstream requirement prevents starting the ConfigKit implementation plan. Production report field names and final runtime wiring are selected in the -[implementation plan](implementation.md), following the outcome and observation +[implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md), following the outcome and observation contract here. The coordinated refactor targets both libraries directly; intermediate commits do not require a working legacy runtime. ### Production Implementation Reference -The [implementation plan](implementation.md) owns ordered production work, +The [implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) owns ordered production work, component changes, tests, and acceptance criteria. The [combined roadmap](library-adoption.md) and [configuration roadmap](configkit.md) own the final target. This evidence does not require an interim runtime bridge or separately runnable configuration migration. @@ -388,5 +388,5 @@ The tests use a controlled built-in HTTP transport and synthetic credentials; there were no live provider calls. The prototype still covers one representative proposal/assessment pair, not all production modules, prompts, CLI paths, or diagnostic artifacts. Root `go test ./...` does not run this nested module. No -upstream API gap appeared in this extension. The [production implementation plan](implementation.md) +upstream API gap appeared in this extension. The [production implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) keeps implementation and release pending. diff --git a/docs/roadmap/library-boundary.md b/docs/roadmap/library-boundary.md index 5611f34..d468bac 100644 --- a/docs/roadmap/library-boundary.md +++ b/docs/roadmap/library-boundary.md @@ -12,7 +12,7 @@ changes were made in that historical unit. ## Next Unit: Testing Overlays and Production Migration Plan -**Status:** Complete. The [production implementation plan](implementation.md) +**Status:** Complete. The [production implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) owns subsequent work; the [overlay findings](library-boundary-findings.md#selected-overlay-extension-2026-09-24) record this unit's checks. The original work packages below remain historical scope for the completed first unit. diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 43fb160..d18c73c 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -2,7 +2,7 @@ **Reviewed:** 2026-09-24, source commit `7cefb3f`. **Scope:** The twelve implementation commits after `1ad2119`, compared with the -[implementation plan](implementation.md), [combined target](library-adoption.md), +[implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md), [combined target](library-adoption.md), and [configuration target](configkit.md). **Status:** R1–R5 addressed and verified on 2026-09-24. The evidence in the original findings below describes source commit `7cefb3f`; line references are diff --git a/docs/roadmap/runtime-behavior.md b/docs/roadmap/runtime-behavior.md index 350efb9..976a35a 100644 --- a/docs/roadmap/runtime-behavior.md +++ b/docs/roadmap/runtime-behavior.md @@ -2,13 +2,13 @@ **Last updated:** 2026-09-24 -The documentation migration identified these existing behaviors for separate -product consideration. Implementation status and any selected direction are noted -below; this document does not claim that runtime changes have landed. +The documentation migration identified existing behaviors for separate product +consideration. The resilience implementation resolved the diagnostics-retention +question; the input-timestamp question remains open. -- Decide whether the `never` retention selection should delete successful-run - diagnostics; current behavior is documented in - [operations](../operations.md#retention-behavior). +- Resolved: legacy `auto`/`always`/`never` retention settings were removed. + Debug is explicitly enabled with `diagnostics.debug` or `--debug`, creates no + directory when disabled, and retains its directory on every outcome when enabled. - Resolved: [configuration](../config.md) now treats explicit zero as disabling transient retries or structural repair independently; PromptKit performs both. - Decide whether missing or null input timestamps should be rejected; current diff --git a/examples/production-config.yml b/examples/production-config.yml index 82d3705..2e57be5 100644 --- a/examples/production-config.yml +++ b/examples/production-config.yml @@ -32,6 +32,9 @@ thresholds: grammar: 0.8 context: description: "General context for domain vocabulary and speaker names." +reporting: + enabled: true + stdout_dir: . diagnostics: + debug: false work_dir: /tmp/audita - retention: auto diff --git a/internal/framework/processreport/correction_ledger.go b/internal/framework/processreport/correction_ledger.go index cc5e433..31a035f 100644 --- a/internal/framework/processreport/correction_ledger.go +++ b/internal/framework/processreport/correction_ledger.go @@ -171,18 +171,6 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry return entries } -func HasSkippedCorrections(runOutput *runner.RunOutput) bool { - if runOutput == nil { - return false - } - for _, mr := range runOutput.ModuleResults { - if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 { - return true - } - } - return false -} - func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []LedgerValidatorDecisionRecord { if len(in) == 0 { return nil diff --git a/internal/framework/processreport/correction_ledger_test.go b/internal/framework/processreport/correction_ledger_test.go index 386006d..f421f86 100644 --- a/internal/framework/processreport/correction_ledger_test.go +++ b/internal/framework/processreport/correction_ledger_test.go @@ -112,21 +112,6 @@ func TestBuildCorrectionLedgerPreservesDispositionPolicy(t *testing.T) { } } -func TestHasSkippedCorrectionsIncludesApplicationSkipsAndValidatorRejections(t *testing.T) { - if HasSkippedCorrections(nil) { - t.Fatal("nil output should not have skipped corrections") - } - if HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 1}}}}}) { - t.Fatal("applied-only output should not have skipped corrections") - } - if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{SkippedChanges: []proposals.SkippedChange{{ProposalIndex: 1}}}}}) { - t.Fatal("application skips should count as skipped corrections") - } - if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 1}}}}}) { - t.Fatal("validator rejections should count as skipped corrections") - } -} - func TestCorrectionLedgerIncludesUnappliedCandidateAfterFatalValidation(t *testing.T) { output := &runner.RunOutput{ Candidates: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"}, ProposalMetadata: proposals.ProposalMetadata{ModuleKey: "grammar", ModuleInstance: "grammar", ProposalIndex: 2}}}, diff --git a/internal/framework/warnings/warnings.go b/internal/framework/warnings/warnings.go deleted file mode 100644 index 8f75478..0000000 --- a/internal/framework/warnings/warnings.go +++ /dev/null @@ -1,18 +0,0 @@ -package warnings - -type Scope string - -const ( - ScopeProposalGeneration Scope = "proposal_generation" - ScopeValidator Scope = "validator" -) - -type StageWarning struct { - Scope Scope `json:"scope"` - ReasonCode string `json:"reason_code"` - Message string `json:"message"` - SectionIndex *int `json:"section_index,omitempty"` - ValidatorName string `json:"validator_name,omitempty"` - BatchIndex *int `json:"batch_index,omitempty"` - DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` -}