Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3b4933256e | |||
| 6f7136724f | |||
| 9bcf18331e | |||
| 5525fe875e | |||
| b81bd28865 | |||
| 9b82eb7da2 | |||
| f8d00e2f20 | |||
| f5fa607c35 | |||
| 43a68e97ee | |||
| b1b0fc7455 | |||
| afcf58331f | |||
| b31955fc97 | |||
| 1777821e8b | |||
| 79d38606fe | |||
| 8cba751835 | |||
| 8175ed87d0 | |||
| b64010e729 | |||
| 1cfe0c5a01 | |||
| 0e2c0b0b82 | |||
| cd9ee78359 | |||
| 23e1df4cfb | |||
| 2023c91ec3 | |||
| 67519051f1 | |||
| 6ae1d7cd5f | |||
| ddfc3dd4c7 | |||
| 990ca0b71a | |||
| 61bc467a03 | |||
| 0bd116b6b7 |
@@ -15,7 +15,7 @@ go build -o ./bin/audita ./cmd/audita
|
||||
Processing requires a reachable [compatible LLM endpoint](docs/integrations/openai-compatible-llm.md)
|
||||
and access to a model you select. The [minimal config](examples/minimal-config.yml)
|
||||
targets an unauthenticated local endpoint; edit its endpoint and
|
||||
[local profiles](examples/profiles/) to match your service and models. The
|
||||
[local profiles](examples/profiles/local/) to match your service and models. The
|
||||
[service example](examples/production-config.yml) demonstrates a named credential
|
||||
reference and a shared backend. Set its referenced environment variable before
|
||||
validating it. The [testing overlay](examples/overlays/testing.yml) selects both
|
||||
|
||||
@@ -237,7 +237,7 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range []string{"proposal", "validation"} {
|
||||
profile := fmt.Sprintf("id: %s\nbackend: local\nmodel: local\ntimeout_seconds: 5\n", id)
|
||||
profile := fmt.Sprintf("profiles:\n %s:\n backend: local\n model: local\n timeout_seconds: 5\n", id)
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -293,6 +293,25 @@ func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (
|
||||
return code, stdout.String(), stderr.String()
|
||||
}
|
||||
|
||||
func (f processFixture) runConfigView(t *testing.T, view string) (int, string, string) {
|
||||
t.Helper()
|
||||
command := exec.Command(os.Args[0], "-test.run=^TestProcessSubprocessHelper$", "--", "config", "print-effective", "--config", f.config, "--view", view)
|
||||
command.Dir = f.dir
|
||||
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
||||
var stdout, stderr bytes.Buffer
|
||||
command.Stdout, command.Stderr = &stdout, &stderr
|
||||
err := command.Run()
|
||||
code := 0
|
||||
if err != nil {
|
||||
if exit, ok := err.(*exec.ExitError); ok {
|
||||
code = exit.ExitCode()
|
||||
} else {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return code, stdout.String(), stderr.String()
|
||||
}
|
||||
|
||||
func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml"))
|
||||
@@ -799,15 +818,70 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
transcriptDir := filepath.Join(f.dir, "transcripts")
|
||||
reportDir := filepath.Join(f.dir, "reports")
|
||||
debugDir := filepath.Join(f.dir, "debug")
|
||||
for _, dir := range []string{transcriptDir, reportDir} {
|
||||
if err := os.Mkdir(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
f.output = filepath.Join(transcriptDir, "corrected.json")
|
||||
code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir)
|
||||
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
||||
t.Fatalf("transcript: %s %v", output, err)
|
||||
}
|
||||
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
||||
if _, err := os.Stat(filepath.Join(reportDir, name)); err != nil {
|
||||
t.Fatalf("report %s: %v", name, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(transcriptDir, name)); !os.IsNotExist(err) {
|
||||
t.Fatalf("report %s appeared beside transcript: %v", name, err)
|
||||
}
|
||||
}
|
||||
dirs, err := filepath.Glob(filepath.Join(debugDir, "run-*"))
|
||||
if err != nil || len(dirs) != 1 {
|
||||
t.Fatalf("debug runs: %v %v", dirs, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dirs[0], "correction-ledger.json")); err != nil {
|
||||
t.Fatalf("debug capture: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
||||
var expectedOutput, expectedSummary string
|
||||
for _, selection := range []struct{ reporting, debug bool }{
|
||||
{false, false}, {false, true}, {true, false}, {true, true},
|
||||
} {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
extra := []string{"--reporting=false", "--report-dir", filepath.Join(f.dir, "missing")}
|
||||
if selection.reporting && selection.debug {
|
||||
original, err := os.ReadFile(f.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := fmt.Sprintf("backends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY", f.server.URL)
|
||||
inherited := fmt.Sprintf("backends:\n local-base:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\n queue_capacity: 0\n local:\n extends: local-base\n concurrency: 1", f.server.URL)
|
||||
updated := strings.Replace(string(original), old, inherited, 1)
|
||||
if updated == string(original) {
|
||||
t.Fatal("process fixture backend configuration was not found")
|
||||
}
|
||||
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
reportDir := filepath.Join(f.dir, "missing")
|
||||
extra := []string{"--reporting=false", "--report-dir", reportDir}
|
||||
if selection.reporting {
|
||||
extra = []string{"--reporting=true", "--report-dir", filepath.Join(f.dir, "missing")}
|
||||
reportDir = filepath.Join(f.dir, "selected-reports")
|
||||
if err := os.Mkdir(reportDir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extra = []string{"--reporting=true", "--report-dir", reportDir}
|
||||
}
|
||||
if selection.debug {
|
||||
extra = append(extra, "--debug")
|
||||
@@ -827,7 +901,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
||||
t.Fatalf("selection changed processing: %+v summary=%q output=%s", selection, summary, output)
|
||||
}
|
||||
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
||||
_, err := os.Stat(filepath.Join(f.dir, name))
|
||||
_, err := os.Stat(filepath.Join(reportDir, name))
|
||||
if selection.reporting && err != nil || !selection.reporting && !os.IsNotExist(err) {
|
||||
t.Fatalf("selection=%+v report %s: %v", selection, name, err)
|
||||
}
|
||||
@@ -842,6 +916,31 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if selection.reporting {
|
||||
effective, err := os.ReadFile(filepath.Join(dirs[0], "effective-config.redacted.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(effective), f.server.URL+"/v1") || !strings.Contains(string(effective), "concurrency: 1") || !strings.Contains(string(effective), "queue_capacity: 0") || strings.Contains(string(effective), f.provider.secret) {
|
||||
t.Fatalf("effective inherited settings: %s", effective)
|
||||
}
|
||||
origins, err := os.ReadFile(filepath.Join(dirs[0], "config-origins.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"from: /backends/local-base/endpoint", "to: /backends/local/endpoint", "reference: /backends/local/extends"} {
|
||||
if !strings.Contains(string(origins), want) {
|
||||
t.Fatalf("configuration origins missing %q: %s", want, origins)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(origins), f.provider.secret) {
|
||||
t.Fatal("configuration origins exposed the resolved inherited credential")
|
||||
}
|
||||
code, explain, stderr := f.runConfigView(t, "explain")
|
||||
if code != 0 || stderr != "" || !strings.Contains(explain, "from: /backends/local-base/endpoint") || strings.Contains(explain, f.provider.secret) {
|
||||
t.Fatalf("inheritance explanation: code=%d stdout=%s stderr=%s", code, explain, stderr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1011,6 +1110,63 @@ func TestSubprocessInvalidInputLeavesFailedReports(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
original, err := os.ReadFile(f.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
updated := strings.Replace(string(original), " local:\n endpoint:", " local:\n extends: missing\n endpoint:", 1)
|
||||
if updated == string(original) {
|
||||
t.Fatal("process fixture backend configuration was not found")
|
||||
}
|
||||
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr := f.run(t, true)
|
||||
if code != 2 || stdout != "" || f.provider.callCount() != 0 || !strings.Contains(stderr, "referenced sibling does not exist") {
|
||||
t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportDir := filepath.Join(f.dir, "failed-reports")
|
||||
if err := os.Mkdir(reportDir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, _ := f.run(t, true, "--report-dir", reportDir)
|
||||
if code != 1 || f.provider.callCount() != 0 {
|
||||
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(reportDir, "errors.yml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var document reporting.ErrorsDocument
|
||||
if err := yaml.Unmarshal(raw, &document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
|
||||
t.Fatalf("failed report: %+v", document)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(reportDir, "recoveries.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if raw, err := os.ReadFile(filepath.Join(reportDir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
|
||||
t.Fatalf("Markdown status: %s %v", raw, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
|
||||
t.Fatalf("failed report appeared in default directory: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||
t.Fatalf("failed transcript published: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
f.output = filepath.Join(f.dir, "missing", "output.json")
|
||||
@@ -1243,4 +1399,47 @@ func TestSubprocessProfileSources(t *testing.T) {
|
||||
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
|
||||
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
|
||||
}
|
||||
|
||||
parentDir := filepath.Join(f.dir, "profiles", "catalog")
|
||||
childDir := filepath.Join(f.dir, "profiles", "nested", "child")
|
||||
for _, directory := range []string{parentDir, childDir} {
|
||||
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
parent := filepath.Join(parentDir, "arbitrary-name.yaml")
|
||||
child := filepath.Join(childDir, "override.yaml")
|
||||
resource := filepath.Join(parentDir, "report.md")
|
||||
for path, content := range map[string]string{
|
||||
parent: "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./report.md\n output:\n format: text\n validation_mode: basic\n",
|
||||
child: "prompts:\n literal/~child:\n extends: source\n",
|
||||
resource: "Captured operator message.\n",
|
||||
} {
|
||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
resourceBefore, err := os.ReadFile(resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.output = resource
|
||||
code, _, stderr = f.run(t, true, "--reporting=false")
|
||||
resourceAfter, err := os.ReadFile(resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
|
||||
t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr)
|
||||
}
|
||||
|
||||
f.output = filepath.Join(f.dir, "output.json")
|
||||
code, _, stderr = f.run(t, true, "--report-dir", parentDir)
|
||||
resourceAfter, err = os.ReadFile(resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
|
||||
t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
28
docs/cli.md
28
docs/cli.md
@@ -48,12 +48,17 @@ validation. The shared setting flags are:
|
||||
An omitted or empty output value selects stdout. Nonempty output filenames are
|
||||
used literally, including whitespace.
|
||||
Reporting defaults to enabled. `--reporting=false` disables all three ordinary
|
||||
reports; an explicit false overrides a true file or overlay setting. With file
|
||||
output, reports go beside the transcript and `--report-dir` has no effect. With
|
||||
stdout output, `--report-dir` selects the report directory, defaulting to the
|
||||
invocation working directory. `--debug` separately enables retained detailed
|
||||
artifacts under `--work-dir`. Configuration inspection does not create or probe
|
||||
these output directories.
|
||||
reports; an explicit false overrides a true file or overlay setting. For stdout
|
||||
output, reports use the effective `reporting.stdout_dir`, which defaults to the
|
||||
invocation working directory. For file output, they go beside the transcript
|
||||
unless `--report-dir` is explicitly supplied. The flag selects the report
|
||||
directory for either transcript destination; a config-file or overlay value for
|
||||
`reporting.stdout_dir` alone affects stdout output only. A selected report
|
||||
directory must be nonblank when reporting is enabled. With reporting disabled,
|
||||
the report destination is inactive and is not probed. `--debug` separately
|
||||
enables retained detailed artifacts under `--work-dir`; selecting `--work-dir`
|
||||
alone does not enable capture. Configuration inspection does not create or
|
||||
probe these output directories.
|
||||
Per-role model, base URL, key, timeout, retry, and concurrency flags have been
|
||||
removed. Models and generation controls belong to PromptKit profiles, backend
|
||||
credentials and capacity to configuration, and retry/deadline settings to the
|
||||
@@ -69,9 +74,14 @@ YAML summary of selected ID, backend, model, sampling controls, per-attempt
|
||||
timeout, service tier, and reasoning effort for each role. It omits credentials,
|
||||
endpoints, and arbitrary extra provider parameters. Explain writes a YAML
|
||||
sequence of field paths, source origins, and recorded derivations without
|
||||
resolved setting values. These views share the same settings and profile
|
||||
validation path as `process`. Unknown views are usage errors. The effective
|
||||
settings YAML is for inspection and is not guaranteed reloadable.
|
||||
resolved setting values. An inherited field may also include an `inheritance`
|
||||
sequence; each step identifies `from`, `to`, and `reference` paths as JSON
|
||||
pointers. These fields describe the effective inheritance chain without
|
||||
including configuration values. A direct child override reports its own source
|
||||
instead. Debug configuration origins use the same explanation shape. These views
|
||||
share the same settings and profile validation path as `process`. Unknown views
|
||||
are usage errors. The effective settings YAML is for inspection and is not
|
||||
guaranteed reloadable.
|
||||
|
||||
For `process`, `--output` writes the corrected transcript to a file and leaves
|
||||
stdout empty on success. Without it, stdout carries transcript JSON only on
|
||||
|
||||
136
docs/config.md
136
docs/config.md
@@ -2,8 +2,12 @@
|
||||
|
||||
This reference describes the version-2 configuration loader in `internal/core/config`.
|
||||
Use the maintained [local](../examples/minimal-config.yml) and
|
||||
[service](../examples/production-config.yml) examples with their PromptKit
|
||||
[profiles](../examples/profiles/) and optional [testing overlay](../examples/overlays/testing.yml).
|
||||
[service](../examples/production-config.yml) examples with their separately
|
||||
selected [local profiles](../examples/profiles/local/) and
|
||||
[service profiles](../examples/profiles/service/), plus the optional
|
||||
[testing overlay](../examples/overlays/testing.yml).
|
||||
The [ConfigKit inheritance example](../examples/configkit-inheritance.yml)
|
||||
shows a local child backend reusing and overriding sibling settings.
|
||||
Choose real models for your endpoint; a testing profile makes no cost or quality
|
||||
guarantee. See the [migration guide](migrations/v2.md) for retired inputs.
|
||||
|
||||
@@ -20,26 +24,123 @@ and secret-directory bootstrap are disabled. Unknown fields and missing, null,
|
||||
or non-integer `version: 2` are errors. Version-1 files require migration; no
|
||||
compatibility loader or implicit environment configuration remains.
|
||||
|
||||
After overlay and explicit CLI override precedence is applied, ConfigKit resolves
|
||||
native sibling inheritance before typed decoding. Any input mapping may use
|
||||
`extends: <sibling-name>`; the child keeps its key, inherits parent settings, and
|
||||
overrides them with its own settings. Selected overlay and CLI values therefore
|
||||
participate in inheritance at their normal precedence. The generic syntax and
|
||||
resolution rules are defined by ConfigKit v0.5.0's
|
||||
[sibling inheritance contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.5.0/docs/config.md#sibling-inheritance).
|
||||
|
||||
`extends` is reserved structural input in base files, imports, overlays, and
|
||||
other application input mappings, including mapping values supplied by explicit
|
||||
CLI setting overrides. ConfigKit consumes it before typed decoding. A previously
|
||||
valid backend whose ID is literally `extends` cannot be represented under this
|
||||
rule. Rename that backend to a non-conflicting ID and update every PromptKit
|
||||
profile that selects it. This compatibility impact means the upgrade does not
|
||||
promise that every previously valid configuration remains valid.
|
||||
|
||||
Inherited values count as explicit input, including zero, false, empty, and
|
||||
null. Existing field rules still apply: for example, inherited backend
|
||||
concurrency must be positive, while an inherited queue capacity of zero is
|
||||
valid. Parent backends undergo the same decoding and application validation as
|
||||
other backends, whether or not a selected profile uses them. A child
|
||||
`credential: null` removes the child's inherited credential; it does not remove
|
||||
the parent's credential reference, which remains subject to ordinary secret
|
||||
resolution.
|
||||
|
||||
Validation runs after overlays, explicit overrides, and derived settings have
|
||||
resolved. An override can therefore repair an invalid lower-precedence setting
|
||||
when that setting can be decoded. ConfigKit retains field origins and derivations
|
||||
for effective-configuration inspection. Its effective YAML redacts resolved
|
||||
credentials.
|
||||
|
||||
Profile files are resolved against the selected main file's directory, or may
|
||||
use an absolute `profiles_dir`. Both selected profiles must exist and select a
|
||||
registered backend. Their effective endpoints must match that backend's endpoint
|
||||
after surrounding whitespace and trailing slashes are removed. Profiles cannot
|
||||
supply a competing `api_key_env` lookup. The backend credential supplies the
|
||||
runtime key when required; inspection makes no provider call. The
|
||||
[CLI inspection views](cli.md#streams-and-exit-status) show selected controls and
|
||||
field provenance separately from ConfigKit's redacted settings YAML.
|
||||
### PromptKit profiles and operator prompts
|
||||
|
||||
`profiles_dir` selects a PromptKit configuration root, resolved against the
|
||||
selected main file's directory unless it is absolute. The root may contain
|
||||
keyed `profiles`, `backends`, and `prompts` definitions; it is not limited to
|
||||
profile-only files. IDs come from literal YAML mapping keys, never filenames.
|
||||
Multiple definitions may share a document, and a document may contribute to any
|
||||
of the supported mappings. Directory discovery is recursive, but discovery
|
||||
order does not establish field precedence. Contributions within each source
|
||||
group must compose without conflicting values.
|
||||
|
||||
PromptKit assembles sources in this order: its selected catalogs and owned
|
||||
defaults, Audita's embedded prompt/schema inputs, operator documents under
|
||||
`profiles_dir`, then the application backend mapping. The application mapping
|
||||
owns each configured endpoint, concurrency limit, queue capacity, and credential
|
||||
environment lookup. Other supported provider fields can be contributed by the
|
||||
operator documents. Imports remain disabled. The current maintained examples
|
||||
select separate roots: [local](../examples/profiles/local/) and
|
||||
[service](../examples/profiles/service/), so definitions for one backend do not
|
||||
enter the other example's final configuration.
|
||||
|
||||
Both selected roles must resolve to an existing profile, registered backend,
|
||||
nonblank effective model, and nonblank endpoint. The normalized effective
|
||||
endpoint must match the application backend endpoint. A selected profile cannot
|
||||
introduce a competing `api_key_env` lookup; the application backend credential
|
||||
supplies the runtime key when required. Other unselected profiles may be
|
||||
partial, but every final backend, profile, and prompt definition is validated
|
||||
when the engine is constructed. A malformed or conflicting unused definition
|
||||
can therefore make the selected root invalid. A missing selected root is an
|
||||
operator configuration error; an empty root may contribute no definitions.
|
||||
|
||||
Profile IDs and parent IDs are exact, literal keys. PromptKit profiles can use
|
||||
native `extends` inheritance; child fields override inherited fields, while
|
||||
unmentioned controls survive changes to model or backend. For example:
|
||||
|
||||
```yaml
|
||||
profiles:
|
||||
"shared~base":
|
||||
backend: service
|
||||
model: example-model
|
||||
temperature: 0.4
|
||||
top_p: 0.8
|
||||
"proposal fast":
|
||||
extends: "shared~base"
|
||||
model: faster-model
|
||||
temperature: 0
|
||||
```
|
||||
|
||||
Presence matters. Explicit `temperature: 0` is sent as zero; omission leaves it
|
||||
unspecified; null suppresses an inherited temperature. Other fields have their
|
||||
own native null and omission rules. Provider `extra_params` maps merge
|
||||
recursively, and a null provider value is sent as JSON null. See the pinned
|
||||
[PromptKit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
|
||||
for the complete field semantics.
|
||||
|
||||
Operator documents can also override an embedded prompt/version, including its
|
||||
messages and definition-level output settings. Shared variables supplied by the
|
||||
adapter are rendered only when an effective message references them; Audita does
|
||||
not append omitted shared text. Audita's per-call schemas, structural repair
|
||||
budget, and downstream domain validators remain in force. The engine captures
|
||||
effective definitions and message-file contents when constructed, so edits are
|
||||
observed only by a newly constructed runtime.
|
||||
|
||||
### Migrating PromptKit profile documents
|
||||
|
||||
For the PromptKit v0.11 format, wrap each profile's fields under
|
||||
`profiles: {<literal-id>: ...}`, remove inner `id`, and replace
|
||||
`base_profile` with `extends`. Profile and prompt IDs are mapping keys, so they
|
||||
need not match source filenames. Review zeros, omitted values, and nulls against
|
||||
the native presence rules before deploying. Select a root whose final
|
||||
definitions have valid backend references; eager validation applies even to
|
||||
unselected definitions. See the complete maintained [local](../examples/minimal-config.yml)
|
||||
and [service](../examples/production-config.yml) examples and the pinned
|
||||
[consumer migration guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api).
|
||||
This incompatible profile-format migration is intended for an Audita major
|
||||
release; no automatic file conversion is provided.
|
||||
|
||||
The [CLI inspection views](cli.md#streams-and-exit-status) show selected
|
||||
controls and field provenance separately from ConfigKit's redacted settings
|
||||
YAML. Inspection resolves both roles without provider calls.
|
||||
|
||||
## Settings
|
||||
|
||||
| Field | Default | Rule |
|
||||
| --- | --- | --- |
|
||||
| `profiles_dir` | `profiles` | Nonempty; profile files are resolved relative to the selected main file. |
|
||||
| `profiles_dir` | `profiles` | Nonempty PromptKit document root; resolved relative to the selected main file, or absolute. |
|
||||
| `backends.<id>.endpoint` | None | Required nonempty endpoint for each backend. At least one backend is required. |
|
||||
| `backends.<id>.concurrency` | `1` | Positive finite capacity; shared by both roles using that backend. |
|
||||
| `backends.<id>.queue_capacity` | `1024` | Nonnegative; zero disables waiting admission. |
|
||||
@@ -62,17 +163,20 @@ field provenance separately from ConfigKit's redacted settings YAML.
|
||||
| `thresholds.glossary`, `.homophones`, `.spoken_word`, `.grammar` | `0.8` each | Confidence thresholds in `[0, 1]`. |
|
||||
| `context.description` | Empty | Trimmed prompt context, at most 500 bytes. |
|
||||
| `reporting.enabled` | `true` | Write the ordinary three-file report bundle when processing. |
|
||||
| `reporting.stdout_dir` | `.` | Report directory for stdout transcript output; ignored for file output or disabled reporting. |
|
||||
| `reporting.stdout_dir` | `.` | Report directory for stdout transcript output; a `--report-dir` flag can select reports for either transcript destination. |
|
||||
| `diagnostics.debug` | `false` | Persist detailed debug artifacts when processing. |
|
||||
| `diagnostics.work_dir` | `/tmp/audita` | Parent of the retained debug run directory; required only when debug is enabled. |
|
||||
|
||||
Module keys select correction modules; validator chains remain built into them.
|
||||
The loader preserves module order and repetitions. Normalization time settings
|
||||
accept numeric seconds only. The operation timeout uses a duration string.
|
||||
`reporting.stdout_dir` must be nonblank only for reporting-enabled stdout runs;
|
||||
`diagnostics.work_dir` must be nonblank only with debug enabled. These path
|
||||
requirements are checked when choosing process destinations, not during
|
||||
configuration inspection.
|
||||
`reporting.stdout_dir` must be nonblank when it supplies the active report
|
||||
destination: for stdout output, or for file output when `--report-dir` is
|
||||
explicitly supplied. File output without that flag uses the transcript's parent,
|
||||
so the configured value is inactive. Reporting-disabled runs do not select a
|
||||
report directory. `diagnostics.work_dir` must be nonblank only with debug
|
||||
enabled. These path requirements are checked when choosing process destinations,
|
||||
not during configuration inspection.
|
||||
|
||||
## Credentials
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ failures; explicit zero makes one attempt. `llm.repair_attempts` counts
|
||||
additional generations for schema-invalid model output. The operation timeout
|
||||
covers preparation, capacity waiting, retries, generation, and repair. Caller
|
||||
cancellation or an earlier deadline also applies. See [configuration](../config.md#settings)
|
||||
for defaults and [PromptKit's consumer contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/consumers/pkg-promptkit.md)
|
||||
for defaults and [PromptKit v0.12.0's consumer contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md)
|
||||
for the pinned protocol and retry details.
|
||||
|
||||
A model operation that exhausts structural repair, a live-parent local operation
|
||||
|
||||
@@ -9,11 +9,14 @@ The [CLI reference](../cli.md) owns command syntax, streams, and exit semantics;
|
||||
Use an argument array with the transcript path and required glossary argument,
|
||||
following the [process examples](../cli.md#commands-and-help). For machine workflows,
|
||||
select an explicit transcript file destination and a configuration path. Reports
|
||||
then go beside the transcript. For stdout output, select `--report-dir` or use
|
||||
the invocation working directory. Debug is independent and off by default.
|
||||
Supplying a configuration path also
|
||||
makes file selection predictable; only explicitly referenced credential variables
|
||||
are read from the environment.
|
||||
go beside the transcript unless an explicit `--report-dir` selects another
|
||||
directory. For stdout output, reports use the effective `reporting.stdout_dir`
|
||||
(defaulting to the invocation working directory), including a `--report-dir`
|
||||
override. A config-file or overlay value for `reporting.stdout_dir` alone does
|
||||
not redirect file-output reports. `--work-dir` selects a debug root, while
|
||||
configuration or `--debug` enables capture; the path flag alone does not.
|
||||
Supplying a configuration path also makes file selection predictable; only
|
||||
explicitly referenced credential variables are read from the environment.
|
||||
|
||||
Resolve the executable deliberately and set the child's working directory so
|
||||
relative input/output paths and reported artifact paths have a known base.
|
||||
@@ -28,7 +31,8 @@ Consume transcript/report data through the
|
||||
[output-file contract](output-files.md), and treat stderr as human-readable detail.
|
||||
The three reports may be absent after early configuration or destination
|
||||
failures. After initialization, failed reports can describe invalid input or
|
||||
partial processing. Exit `0` can include recoverable model errors and applied
|
||||
partial processing and use the same selected report directory as successful
|
||||
runs. Exit `0` can include recoverable model errors and applied
|
||||
corrections; inspect `errors.yml` and `recoveries.yml` for those outcomes. A
|
||||
transcript may have been written before a later required report failure, so
|
||||
the file alone does not prove completion.
|
||||
|
||||
@@ -35,11 +35,18 @@ From the repository root, validate the maintained configuration examples:
|
||||
|
||||
```sh
|
||||
go run ./cmd/audita config validate --config examples/minimal-config.yml
|
||||
go run ./cmd/audita config validate --config examples/configkit-inheritance.yml
|
||||
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml
|
||||
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml --overlay testing
|
||||
go run ./cmd/audita config print-effective --config examples/minimal-config.yml --view profiles
|
||||
go run ./cmd/audita config print-effective --config examples/configkit-inheritance.yml --view explain
|
||||
```
|
||||
|
||||
The local and ConfigKit inheritance files select `profiles/local`; the service
|
||||
file selects `profiles/service`, including the service testing profiles. Keep
|
||||
these roots separate because PromptKit validates every final definition in a
|
||||
selected root, including definitions that are not selected for a role.
|
||||
|
||||
Only referenced credentials read environment values; use a controlled synthetic
|
||||
value when validating the service example. Check affected CLI examples against the
|
||||
parser and use schema tests for maintained transcript/glossary inputs. Example
|
||||
|
||||
@@ -2,11 +2,28 @@
|
||||
|
||||
## Client Boundary
|
||||
|
||||
`llm.NewRuntime` assembles one PromptKit engine for both roles. It loads embedded
|
||||
prompt and schema assets, registers application backends, and inspects the
|
||||
selected proposal and validation profiles offline. A role-bound `llm.RoleClient`
|
||||
implements `structuredoutput.Client`; proposal and validator contracts alias that
|
||||
interface. Both roles share engine-local backend capacity.
|
||||
`llm.NewRuntime` assembles one PromptKit engine for both roles. PromptKit's
|
||||
catalogs and defaults are followed by Audita's embedded prompt and schema
|
||||
inputs, operator documents from `profiles_dir`, and the highest-priority
|
||||
application backend mapping. Audita owns backend endpoints, capacity settings,
|
||||
and resolved credentials; native PromptKit layering owns profile inheritance,
|
||||
prompt overrides, and provider-specific fields. Imports are disabled. The
|
||||
runtime inspects both selected roles offline, requiring complete model and
|
||||
endpoint values and rejecting competing profile credential lookups. A
|
||||
role-bound `llm.RoleClient` implements `structuredoutput.Client`; proposal and
|
||||
validator contracts alias that interface. Both roles share engine-local backend
|
||||
capacity.
|
||||
|
||||
Successful construction captures the final definitions and referenced message
|
||||
contents. The runtime does not reload operator documents or message files during
|
||||
calls. If final engine construction fails, an embedded-only diagnostic
|
||||
construction distinguishes shipped asset failures from invalid operator input.
|
||||
An asset failure remains an asset error; when embedded construction succeeds,
|
||||
the original failure remains a configuration error. The diagnostic engine
|
||||
never replaces or executes the requested configuration, and caller cancellation
|
||||
skips this path. The captured inspection also supplies an internal source
|
||||
ledger: all loaded host YAML plus effective, source-relative host
|
||||
`content_file` resources are tracked for output-collision checks.
|
||||
|
||||
For each logical call, the adapter starts an operation context and a diagnostic
|
||||
record before preparation. It supplies the reserved hardening text, the selected
|
||||
@@ -24,6 +41,10 @@ Counts distinguish provider attempts, transient retries, and structural
|
||||
repairs, including on errors. Secrets from all configured backends, including unselected
|
||||
ones, are available to the diagnostics redaction boundary.
|
||||
|
||||
CLI explain output and debug configuration origins share a value-free projection
|
||||
of effective field origins. The [CLI contract](../cli.md) defines its optional
|
||||
inheritance evidence fields.
|
||||
|
||||
## Orchestration
|
||||
|
||||
CLI processing supplies two role clients from the same engine to the runner.
|
||||
|
||||
@@ -11,18 +11,27 @@
|
||||
- `validators.meaning_reversal_review` (`v1`)
|
||||
- `validators.editorial_review` (`v1`)
|
||||
|
||||
PromptKit reads the embedded definitions and renders their system and user
|
||||
messages from named string variables. The role-bound adapter supplies
|
||||
PromptKit reads the embedded definitions as shipped defaults and renders their
|
||||
messages from named string variables. Operator documents in `profiles_dir` may
|
||||
contribute native keyed prompt definitions and override an embedded
|
||||
prompt/version, including its messages and definition-level output settings.
|
||||
The effective operator text is captured when the runtime is constructed; the
|
||||
embedded assets remain the source of Audita's shipped task instructions. The
|
||||
role-bound adapter supplies
|
||||
`HardeningPolicy` alongside `ProposalContract` and `ValidationContract` from
|
||||
embedded fragments under `assets/shared`, after caller variables so callers cannot replace those
|
||||
instructions. Common literal-span and identity guidance lives in the proposal
|
||||
fragment; shared candidate-reference, context, and confidence guidance lives in
|
||||
the validation fragment. Module-specific policies and examples remain in YAML.
|
||||
Proposal prompts receive the section JSON, glossary JSON where used, and transcript description block.
|
||||
Validator prompts receive the correction payload and description block.
|
||||
The prompt assets retain the prior task instructions and output-schema
|
||||
association. The role-bound runtime supplies the complete JSON Schema output contract
|
||||
and repair count; the definition does not own an operator-facing repair default.
|
||||
embedded fragments under `assets/shared`, after caller variables so callers
|
||||
cannot replace those instructions. Common literal-span and identity guidance
|
||||
lives in the proposal fragment; shared candidate-reference, context, and
|
||||
confidence guidance lives in the validation fragment. Module-specific policies
|
||||
and examples remain in YAML. Proposal prompts receive the section JSON, glossary
|
||||
JSON where used, and transcript description block. Validator prompts receive
|
||||
the correction payload and description block.
|
||||
The embedded prompt assets retain the shipped task instructions and
|
||||
output-schema association. An operator override controls which template
|
||||
variables its messages reference: supplying a shared variable does not append
|
||||
its text when the effective message omits that reference. The role-bound runtime
|
||||
supplies the complete JSON Schema output contract and repair count; the
|
||||
definition does not own an operator-facing repair default.
|
||||
The correction-set schema requires a string replacement and accepts an empty
|
||||
string. Proposal guidance uses it for eligible span deletion and permits literal
|
||||
spacing edits; validator guidance evaluates the full before/after segment under
|
||||
|
||||
@@ -17,23 +17,32 @@ settings, and the [output contract](integrations/output-files.md) owns schemas.
|
||||
## Required Output Files
|
||||
|
||||
Reporting is enabled by default. For file transcript output, `errors.yml`,
|
||||
`recoveries.yml`, and `report.md` go in the transcript output's parent directory.
|
||||
For stdout transcript output, they go in `reporting.stdout_dir` (default `.`),
|
||||
overridden by `--report-dir`. `--reporting=false` suppresses all three reports
|
||||
and removes their path requirements. It does not suppress the stderr summary or
|
||||
`recoveries.yml`, and `report.md` go in the transcript output's parent directory
|
||||
unless `--report-dir` is explicitly supplied. For stdout transcript output, they
|
||||
go in the effective `reporting.stdout_dir` (default `.`), including a
|
||||
`--report-dir` override. A config-file or overlay value for
|
||||
`reporting.stdout_dir` alone does not redirect file-output reports. An active
|
||||
selected report directory must be nonblank. `--reporting=false` suppresses all
|
||||
three reports and leaves their destination inactive, so it is not probed merely
|
||||
because `--report-dir` was supplied. It does not suppress the stderr summary or
|
||||
fatal errors. Stdout contains only transcript JSON; file output leaves it empty.
|
||||
|
||||
Existing report directories and parents for regular transcript files must be
|
||||
writable. Audita prevents report writes from overwriting transcript, glossary,
|
||||
loaded configuration, or loaded profile sources, and rejects conflicting output
|
||||
destinations. The final transcript may intentionally replace its own input after
|
||||
successful processing; other input/configuration/profile sources remain protected.
|
||||
loaded application configuration, host YAML discovered under `profiles_dir`,
|
||||
or host message files referenced by effective PromptKit `content_file` fields,
|
||||
and rejects conflicting output destinations. Source paths come from the
|
||||
runtime's captured configuration; unrelated files in the selected directory
|
||||
remain eligible output destinations. The final transcript may intentionally
|
||||
replace its own input after successful processing; other input/configuration/
|
||||
profile sources remain protected.
|
||||
Preparation never truncates an existing transcript. Device and pipe destinations
|
||||
(such as `/dev/null` and named pipes) receive ordinary writes at publication time;
|
||||
they are not opened during preflight or replaced with regular files. A pipe write
|
||||
can wait for its reader, and stream write errors are fatal. With reporting enabled,
|
||||
reports still use the output path's parent; disable reporting when that placement
|
||||
is unsuitable for a stream destination.
|
||||
reports still use the output path's parent unless an explicit `--report-dir`
|
||||
selects another directory; disable reporting when that placement is unsuitable
|
||||
for a stream destination.
|
||||
|
||||
With reporting enabled, all three reports are initialized as `in_progress` before
|
||||
input processing. Output is encoded fully before publication. Each regular output
|
||||
@@ -48,10 +57,12 @@ finalized where possible. An uncatchable termination can leave `in_progress`.
|
||||
|
||||
## Diagnostics Directory
|
||||
|
||||
`--debug` enables detailed evidence independently of reporting. One directory
|
||||
named `run-<id>` is created under `diagnostics.work_dir` (default `/tmp/audita`)
|
||||
and retained on success or failure. Without debug, no directory or payload
|
||||
files are created. The parent is created when needed only for debug. Failure to
|
||||
`--debug` or `diagnostics.debug` enables detailed evidence independently of
|
||||
reporting. One directory named `run-<id>` is created under
|
||||
`diagnostics.work_dir` (default `/tmp/audita`), which `--work-dir` can override,
|
||||
and retained on success or failure. Selecting `--work-dir` alone does not enable
|
||||
capture. Without debug, no directory or payload files are created. The parent
|
||||
is created when needed only for debug. Failure to
|
||||
write optional debug evidence prints a redacted stderr diagnostic but does not
|
||||
change correction decisions or the exit status.
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||
|
||||
## ConfigKit
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.4.2`.
|
||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
@@ -24,9 +24,9 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||
[GPL v3](licenses/GPL-3.0.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f/COPYRIGHT).
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f).
|
||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
|
||||
|
||||
## Go runtime and standard library
|
||||
|
||||
@@ -142,7 +142,7 @@ License and notices: [Apache 2.0](licenses/Apache-2.0.txt).
|
||||
|
||||
## PromptKit
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.11.2`.
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.12.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
@@ -150,37 +150,37 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||
[GPL v3](licenses/GPL-3.0.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2/COPYRIGHT).
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3/COPYRIGHT).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2).
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3).
|
||||
|
||||
## Promptkit OpenRouter Catalog
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2` at `v2.20260926.0`.
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3` at `v3.20260930.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/LICENSE).
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3).
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
|
||||
|
||||
## Promptkit Rakestrawhome Catalog
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2` at `v2.20260926.0`.
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3` at `v3.20260930.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/LICENSE).
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596).
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
|
||||
|
||||
## yaml.v3
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# Release Notes
|
||||
|
||||
- [Audita v3.0.0](v3.0.0.md)
|
||||
- [Audita v2.0.1](v2.0.1.md)
|
||||
- [Audita v2.0.0](v2.0.0.md)
|
||||
- [Audita v1.1.0](v1.1.0.md)
|
||||
|
||||
59
docs/releases/v3.0.0.md
Normal file
59
docs/releases/v3.0.0.md
Normal file
@@ -0,0 +1,59 @@
|
||||
# Audita v3.0.0
|
||||
|
||||
Audita v3 adopts native configuration inheritance and layered PromptKit
|
||||
definitions, and allows transcript and report destinations to be selected
|
||||
independently. This is an intentional breaking release: existing PromptKit
|
||||
profile files require migration from the v2 release format.
|
||||
|
||||
## Changes
|
||||
|
||||
- ConfigKit is updated from v0.4.2 to v0.5.0. Application configuration supports
|
||||
native sibling inheritance, and configuration explanations include value-free
|
||||
inheritance provenance.
|
||||
- PromptKit is updated from v0.11.2 to v0.12.0, with both backend catalogs at
|
||||
`v3.20260930.0`. Native keyed definitions support profile inheritance and
|
||||
operator prompt overrides. The selected `profiles_dir` can contain profiles,
|
||||
backends, and prompts; application backend endpoints, capacity, and credentials
|
||||
remain authoritative.
|
||||
- Effective definitions and referenced prompt message contents are captured at
|
||||
runtime construction. Consumed host configuration and message files are
|
||||
protected from output collisions. Audita retains its per-call schemas, domain
|
||||
validators, retry and repair budgets, and resilient processing behavior.
|
||||
- An explicit `--report-dir` now selects reports for either file or stdout
|
||||
transcript output. File output without that flag still places reports beside
|
||||
the transcript. This supports Narratio's independent output destinations;
|
||||
debug capture remains independently selectable.
|
||||
|
||||
## Upgrading from v2
|
||||
|
||||
1. Convert each old PromptKit profile to a keyed definition under
|
||||
`profiles: {<profile-id>: ...}`, remove its inner `id`, and replace
|
||||
`base_profile` with `extends`. Profile IDs come from mapping keys rather than
|
||||
filenames. No compatibility loader or automatic conversion is provided.
|
||||
2. Review explicit zero, omitted, and null generation controls. For example,
|
||||
`temperature: 0` now sends an explicit zero, omission leaves the control
|
||||
unspecified, and null suppresses an inherited control. A zero profile timeout
|
||||
disables PromptKit's generation deadline; Audita's operation deadline still
|
||||
applies.
|
||||
3. Select a configuration root whose final definitions are valid. PromptKit now
|
||||
validates unused definitions too, including backend references, and conflicting
|
||||
values within a directory source group are errors. The maintained examples
|
||||
use separate [local](../../examples/profiles/local/) and
|
||||
[service](../../examples/profiles/service/) roots. Runtime source edits take
|
||||
effect on the next invocation.
|
||||
4. Rename any application backend literally named `extends` and update its
|
||||
profile references. ConfigKit now reserves that key for inheritance.
|
||||
5. Validate the migrated installation with
|
||||
`audita config validate --config /path/to/config.yml`, including any selected
|
||||
testing overlay. Application configuration continues to use `version: 2`.
|
||||
|
||||
The [configuration reference](../config.md#promptkit-profiles-and-operator-prompts)
|
||||
contains the migrated format and presence rules. Start from the maintained
|
||||
[local](../../examples/minimal-config.yml),
|
||||
[service](../../examples/production-config.yml), or
|
||||
[application inheritance](../../examples/configkit-inheritance.yml) examples.
|
||||
The [CLI reference](../cli.md) describes report selection, and the
|
||||
[release procedure](../release.md) describes source publication and asynchronous
|
||||
Linux binary delivery.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/docs/oss/attribution.md)
|
||||
@@ -1,171 +1,133 @@
|
||||
# ConfigKit Adoption Roadmap
|
||||
# ConfigKit v0.5.0 Adoption Roadmap
|
||||
|
||||
**Last updated:** 2026-09-24
|
||||
**Last updated:** 2026-10-03
|
||||
|
||||
**Status:** Production implementation complete. The selected configuration and
|
||||
overlay model is implemented; [implementation evidence](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md)
|
||||
records verification. No release has been published.
|
||||
**Status:** Complete. The implementation and tests are committed through
|
||||
`8175ed8`. Final documentation and example changes were included in the full
|
||||
build, test, example-validation, and local-link checks before the documentation
|
||||
completion commit. The [implementation plan](implementation.md) records the
|
||||
ordered work and verification.
|
||||
|
||||
## Purpose and Direction
|
||||
## Purpose and Baseline
|
||||
|
||||
Adopt ConfigKit to harmonize Audita's configuration policy with the other
|
||||
applications in the suite. Consistent configuration conventions are the primary
|
||||
benefit; reducing application-owned configuration infrastructure is a secondary
|
||||
benefit and a useful measure of integration quality.
|
||||
Adopt ConfigKit v0.5.0 as an independently reviewable step before PromptKit
|
||||
v0.12.0. Use the library's configuration and inheritance conventions while
|
||||
preserving Audita's domain ownership and existing operator workflows.
|
||||
|
||||
This is an intentional breaking migration. There is no goal of backward
|
||||
compatibility with existing installations, configuration files, CLI flags,
|
||||
environment variables, or configuration-inspection output. Provide a migration
|
||||
guide instead of legacy loaders, aliases, dual operating modes, or an ongoing
|
||||
compatibility layer. Retain existing syntax only when it independently suits the
|
||||
new model, not because compatibility requires it.
|
||||
At the reviewed Audita revision `6ae1d7cd5f548b26de5787372d3f2249eaa4757d`,
|
||||
[go.mod](../../go.mod) selects ConfigKit v0.4.2, PromptKit v0.11.2, and Go
|
||||
1.25.5. ConfigKit already owns loading, presence, secret resolution, and
|
||||
configuration inspection. Audita's `internal/core/config` supplies typed
|
||||
settings, defaults, derived values, and domain validation; `internal/cli`
|
||||
selects files, overlays, and explicit overrides. The
|
||||
[configuration contract](../config.md) and [CLI contract](../cli.md) own the
|
||||
implemented behavior.
|
||||
|
||||
The initial review and prototype used ConfigKit v0.3.1, then verified v0.4.0
|
||||
selected overlays. Those findings remain historical; the prototype was removed
|
||||
after production integration. The application depends on a versioned module,
|
||||
without a sibling checkout or local module replacement.
|
||||
The reviewed upstream release is
|
||||
[ConfigKit v0.5.0](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.5.0/docs/releases/v0.5.0.md),
|
||||
commit `dc09959cca4e105f49ec316c6d4ba585bd2340bd`. It adds layered underlays,
|
||||
caller-owned filesystem sources, mapping sources, source resource contexts, and
|
||||
built-in sibling inheritance. The Go minimum and YAML dependency do not change.
|
||||
Ordinary `Load` remains available; adopting `LoadLayered` is optional.
|
||||
|
||||
The shared boundary/prototype unit is complete. Its [selected contract, inventory,
|
||||
and evidence](library-boundary-findings.md) define the detailed target alongside
|
||||
this roadmap. Production configuration migration is complete.
|
||||
This roadmap supersedes the completed original adoption plan at this path.
|
||||
The [historical roadmap](https://gitea.maximumdirect.net/eric/audita/src/commit/6ae1d7cd5f548b26de5787372d3f2249eaa4757d/docs/roadmap/configkit.md)
|
||||
records that earlier migration. Its coordinated breaking-refactor instructions
|
||||
and release target do not govern this incremental upgrade.
|
||||
|
||||
## Target End State
|
||||
|
||||
- ConfigKit owns strict YAML loading, input presence, default application, explicit
|
||||
override mechanics, recorded resolution, secret references, and safe inspection.
|
||||
- Audita owns its configuration types, field meanings, defaults, schema-version
|
||||
policy, file selection, CLI parsing, domain validation, and runtime initialization.
|
||||
- Adopt ConfigKit's precedence and validation lifecycle rather than reproduce
|
||||
Audita's current per-layer validation and environment-overlay behavior.
|
||||
- General configuration environment overrides are retired. Credentials use typed
|
||||
environment-name secret references; supplied references must resolve according
|
||||
to ConfigKit's contract. File selection follows the application-owned
|
||||
discovery rules in the selected boundary.
|
||||
- Standardize duration representations, explicit empty/null/zero handling, and
|
||||
inheritance on the shared conventions. Express genuine Audita domain rules
|
||||
through the supported resolution and validation APIs.
|
||||
- Retained domain-setting flags and proposal/validation profile flags become
|
||||
explicit ConfigKit overrides according to the migration inventory. Remove old
|
||||
provider/key/retry/concurrency flags and aliases. Credentials use secret references.
|
||||
- Use ConfigKit's redacted effective output and provide source/explanation access
|
||||
through the selected views below. Legacy JSON inspection is replaced.
|
||||
Diagnostic configuration snapshots use the same new model.
|
||||
- Delete superseded loading and override machinery. Do not hide the old model
|
||||
behind adapters that defeat the adoption's consistency benefit.
|
||||
- Audita uses the released ConfigKit v0.5.0 module, with no local replacement.
|
||||
PromptKit stays at v0.11.2 throughout this adoption unit.
|
||||
- Configuration retains the current file-selection, overlay, explicit-override,
|
||||
resolution, and validation flow, subject to the reserved-key compatibility
|
||||
impact described below. Retain the application schema version, CLI interface,
|
||||
and main-file-relative resource rules defined in the current contracts.
|
||||
- Application configuration can use ConfigKit's native sibling `extends`
|
||||
relationships. Audita does not implement a second inheritance resolver,
|
||||
special backend-only syntax, or an opt-in switch around the library feature.
|
||||
- Inherited values participate in presence-aware defaults, domain validation,
|
||||
secret resolution, and safe inspection just like other effective input.
|
||||
In particular, inherited backend settings must not be overwritten merely
|
||||
because they were not written directly on the child.
|
||||
- Effective settings and explanations remain useful and secret-safe when
|
||||
values are inherited. Preserve existing inspection fields and add compact,
|
||||
value-free inheritance evidence from ConfigKit's metadata rather than
|
||||
reconstructing provenance from YAML.
|
||||
- Canonical documentation and a maintained example explain the adopted
|
||||
inheritance behavior. Attribution identifies the exact new upstream source.
|
||||
|
||||
Keep Audita's domain capabilities, such as ordered and repeated correction modules,
|
||||
expressible in the new model. Their existing configuration spelling is not binding.
|
||||
Transcript/glossary formats, processing algorithms, and output schemas are outside
|
||||
this migration unless later planning identifies a concrete dependency or an
|
||||
explicitly coordinated change.
|
||||
The upstream [configuration contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.5.0/docs/config.md#sibling-inheritance)
|
||||
owns generic inheritance mechanics. The integration must account for three
|
||||
consequences: `extends` is reserved automatically throughout input mappings;
|
||||
relationships resolve after source precedence and before typed decoding; and
|
||||
inherited zero, false, empty, or null values count as explicit input. An inherited
|
||||
value still has to satisfy Audita's field-specific rules. Parent definitions
|
||||
also undergo ordinary application validation; inheritance does not introduce
|
||||
abstract, validation-exempt backend entries.
|
||||
|
||||
## Scope and Adoption Boundary
|
||||
|
||||
Keep the existing `Load` integration unless implementation reveals a concrete
|
||||
need to change it. Source-backed defaults, additional operator underlay tiers,
|
||||
imports, and source-relative reinterpretation of application paths are outside
|
||||
this feature. Their availability upstream does not require expanding Audita's
|
||||
configuration surface. Imports and secret-directory bootstrap retain their
|
||||
current disabled state.
|
||||
|
||||
ConfigKit inheritance applies to Audita configuration in this step. PromptKit
|
||||
profile formats, profile inheritance, engine construction, embedded prompts,
|
||||
and backend catalogs remain governed by the existing PromptKit integration.
|
||||
There is no application-owned translation of profile documents in this work.
|
||||
|
||||
This is an independent ConfigKit adoption ahead of the planned PromptKit
|
||||
migration. A separate public release between the two units is optional.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- The released ConfigKit v0.5.0 dependency and exact source attribution are
|
||||
adopted; PromptKit remains at v0.11.2 with its existing catalogs.
|
||||
- Native inheritance preserves effective endpoints, capacity, explicit zero
|
||||
queue capacity, child overrides, and surviving credential references through
|
||||
Audita's loading, resolution, and validation boundary.
|
||||
- Selected overlays and explicit overrides retain their precedence; omitted
|
||||
validation-role selection still derives from the final proposal selection.
|
||||
- Null can remove a child's credential while surviving references on parents
|
||||
and other unselected backends still follow the existing credential contract.
|
||||
- Selected roles use inherited backend settings and share its configured
|
||||
capacity under the existing PromptKit runtime contract.
|
||||
- Effective settings remain redacted, and both CLI explanations and debug
|
||||
configuration origins expose compact, value-free inheritance evidence while
|
||||
retaining existing fields and application-derived explanations.
|
||||
- A maintained complete example and canonical documentation describe native
|
||||
inheritance, its field-validation implications, and the reserved-key
|
||||
compatibility impact. Existing workflows and applicable offline checks pass.
|
||||
|
||||
## Completion and Release Treatment
|
||||
|
||||
Adoption is complete when the released dependency is pinned, existing workflows
|
||||
pass, representative inheritance works through loading and runtime wiring,
|
||||
inspection includes value-free inheritance evidence and remains secret-safe,
|
||||
and the example, canonical docs, and attribution match the implementation. No
|
||||
PromptKit migration or provider behavior change is required to close this unit.
|
||||
|
||||
Release classification remains provisional. A minor release would be appropriate
|
||||
for a backward-compatible addition, but the newly reserved `extends` key can
|
||||
invalidate currently valid application configuration. For example, Audita
|
||||
currently permits a backend literally named `extends`; ConfigKit v0.5.0 treats
|
||||
that mapping key as an inheritance directive rather than a backend ID. Do not
|
||||
promise blanket compatibility or settle on a minor release until the final
|
||||
compatibility review resolves this impact under the
|
||||
[release policy](../policy/release.md). Describe any required migration in the
|
||||
canonical documentation and release notes when the upgrade ships. Do not select
|
||||
or publish a release as part of roadmap preparation. A later PromptKit
|
||||
migration's profile-format compatibility decision belongs to that separate
|
||||
feature.
|
||||
|
||||
## Selected Overlays for Testing
|
||||
|
||||
Enable ConfigKit's optional selected-overlay feature in the migration. A testing
|
||||
configuration that selects cheaper, faster LLMs is the initial use case. ConfigKit
|
||||
**overlays** modify application settings; PromptKit **execution profiles** still
|
||||
own model and generation settings. Keep `profiles_dir`, the PromptKit `profiles/`
|
||||
directory, and proposal/validation profile names unchanged by this terminology
|
||||
change.
|
||||
|
||||
The planned interface is `--overlay <name>` on `process`, `config validate`, and
|
||||
`config print-effective`. Treat selection as invocation state passed to
|
||||
`Options.Overlay`, not as another persistent setting or environment overlay.
|
||||
Follow ConfigKit v0.4.0's selection contract: omission, an empty name, or `default`
|
||||
selects no overlay; a named selection loads `overlays/<name>.yml` beneath the selected
|
||||
configuration root. A missing selected overlay is an error. Use the library's
|
||||
single-overlay semantics without adding stacking, inheritance, or legacy-directory
|
||||
fallback.
|
||||
|
||||
Precedence is application defaults, main configuration, selected overlay, then
|
||||
explicit setting flags. Perform Audita's role inheritance and semantic validation
|
||||
on the resolved configuration. ConfigKit's base parsing and schema-version checks
|
||||
still precede overlay application; overlays cannot replace the schema version or
|
||||
contain ConfigKit directives. General environment-variable overrides remain retired.
|
||||
Imports and secret-directory bootstrap remain disabled.
|
||||
|
||||
Provide a maintained `overlays/testing.yml` example that selects testing execution
|
||||
profiles through `llm.proposal_profile` and `llm.validation_profile`. Model names and
|
||||
generation controls belong in the corresponding PromptKit profile files. Set both
|
||||
role selections in this example so a base configuration's explicit validation
|
||||
profile cannot accidentally retain the production model. Normal omission-based
|
||||
validation-profile inheritance continues to apply to the final effective settings.
|
||||
The example must make its backend/credential requirements clear and use no real
|
||||
secrets. Model choice is operator-controlled; “testing” does not promise a price,
|
||||
performance level, or equivalent correction quality.
|
||||
|
||||
Overlays may modify other ordinary application settings supported by the schema;
|
||||
they are not restricted to profile selection. The resulting configuration must
|
||||
still satisfy shared backend capacity, credential, and domain rules. Validation,
|
||||
effective-config inspection, selected-profile inspection, and processing must all
|
||||
use the same selected overlay. Preserve overlay provenance in configuration
|
||||
explanations and diagnostics while retaining secret redaction.
|
||||
|
||||
The [recorded overlay results](library-boundary-findings.md#selected-overlay-extension-2026-09-24)
|
||||
prove the representative load, inspection, and HTTP path. Production acceptance
|
||||
requires the same selection to reach both actual LLM roles, with explicit flags
|
||||
winning over overlays and surviving secret references resolved/redacted correctly.
|
||||
The isolated prototype is evidence, not a second supported runtime.
|
||||
|
||||
## Inspection and Invocation Contract
|
||||
|
||||
`process`, `config validate`, and `config print-effective` share file/overlay
|
||||
selection and retained setting overrides. Invocation I/O paths remain process-only.
|
||||
Only explicitly supplied flags override settings. Validation requires an explicit
|
||||
config file; processing and inspection use the selected discovery order. Help is
|
||||
available before configuration loading. All three commands validate selected
|
||||
profiles without requiring a provider call during setup.
|
||||
|
||||
`config print-effective --view settings|profiles|explain` exposes three YAML views:
|
||||
|
||||
- `settings` (default): ConfigKit's redacted effective configuration.
|
||||
- `profiles`: an Audita-owned summary of each selected role's profile, backend,
|
||||
model, temperature, token limit, top-p, timeout, service tier, and reasoning
|
||||
effort where present. Arbitrary extra provider parameters and credentials are
|
||||
omitted; this is a selected-controls view, not every execution input.
|
||||
- `explain`: field paths, sources, and derivation records from ConfigKit, without
|
||||
resolved secret values.
|
||||
|
||||
A ConfigKit snapshot alone does not describe profile-owned controls. Profile
|
||||
inspection must enforce the registered-backend and credential boundary. Diagnostic
|
||||
snapshots use these safe views and preserve overlay/explicit-override provenance.
|
||||
The migration guide explains the intentional replacement of legacy inspection JSON.
|
||||
|
||||
## Integration and Completion Criteria
|
||||
|
||||
ConfigKit and PromptKit form one intentional breaking refactor. Intermediate
|
||||
commits need not build or run. The target contains no temporary old-runtime adapter,
|
||||
compatibility shim, processing gate, dual loader, or automatic legacy conversion.
|
||||
The [combined feature roadmap](library-adoption.md) owns execution/reporting intent;
|
||||
[implementation.md](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) alone owns stage order and implementation tasks.
|
||||
|
||||
The final application loads all retained domain settings, assembles both selected
|
||||
profiles, and processes through PromptKit. Controlled HTTP evidence must connect
|
||||
configuration selection to actual requests, domain outcomes, safe diagnostics,
|
||||
and shared finite backend capacity. Maintained examples and one migration guide
|
||||
cover the complete file/flag/environment transition. Obsolete application-owned
|
||||
loading, transport, rendering, and scheduler machinery is removed.
|
||||
|
||||
The combined target is Audita v2.0.0. Release compliance and publication remain
|
||||
last and separate from implementing the refactor. Retention and transcript timestamp
|
||||
changes are not included; the selected retry/repair contract resolves the retry
|
||||
follow-up in [runtime behavior](runtime-behavior.md).
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- Pin ConfigKit v0.4.0 in production and align Go/toolchain requirements. ConfigKit requires
|
||||
Go 1.25; the combined PromptKit prototype requires Go 1.25.5. Adopting these libraries
|
||||
does not automatically remove Audita's existing YAML dependency used for glossary parsing.
|
||||
- Implement the selected configuration and final PromptKit runtime directly,
|
||||
without temporary runtime adapters, compatibility shims, or process gates.
|
||||
- Enable selected overlays as specified above. Keep imports and secret-directory
|
||||
bootstrap disabled unless a separate concrete requirement justifies them.
|
||||
- Validation and inspection resolve surviving credential references and fail on
|
||||
missing or empty values; no credential-free bypass is planned.
|
||||
- Inspection and diagnostics use the safe selected-profile view and ConfigKit
|
||||
provenance described above. Changed formats are covered by the migration guide.
|
||||
|
||||
The eventual completion test is that Audita follows the suite's ConfigKit
|
||||
conventions while retaining its intended domain capabilities, with a clear
|
||||
migration guide and no ongoing obligation to support the previous configuration
|
||||
model.
|
||||
This heading preserves links from historical adoption documents. The original
|
||||
[overlay plan](https://gitea.maximumdirect.net/eric/audita/src/commit/6ae1d7cd5f548b26de5787372d3f2249eaa4757d/docs/roadmap/configkit.md#selected-overlays-for-testing)
|
||||
is historical; current overlay behavior belongs to [configuration](../config.md).
|
||||
This upgrade preserves that workflow and adds native ConfigKit inheritance
|
||||
without reviving the earlier combined migration plan.
|
||||
|
||||
@@ -1,162 +0,0 @@
|
||||
# Partial Success for Validator Decision Sets
|
||||
|
||||
**Last updated:** 2026-09-25
|
||||
|
||||
**Status:** Implemented. This document owns the target behavior and user intent.
|
||||
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/91d9be68a7bb3a102900253fdfd8438f0a43f666/docs/roadmap/implementation.md) owns concrete decisions, ordered work,
|
||||
and verification.
|
||||
|
||||
## Purpose and User Intent
|
||||
|
||||
Preserve usable validator decisions when other entries in the same model response
|
||||
have missing, duplicate, or unexpected correction indices. One defective entry
|
||||
should not prevent unrelated candidates from completing their required validation.
|
||||
This extends Audita's existing partial-success approach without another model call.
|
||||
|
||||
Implement this simpler improvement before reconsidering guided domain recovery.
|
||||
Missing validation must never authorize applying a correction, but neither should
|
||||
it invalidate a different candidate's unambiguous approval or editorial rejection.
|
||||
The existing [resilience design](resilience.md) remains the foundation.
|
||||
|
||||
## Historical Baseline
|
||||
|
||||
The reviewed application baseline is `d2a6fcf`:
|
||||
|
||||
- PromptKit validates the complete model response against Audita's embedded JSON
|
||||
Schema and performs its configured structural repairs. Only structurally valid,
|
||||
decoded responses reach domain decision mapping.
|
||||
- `mapLLMResponseToDecisions` in
|
||||
[`llm_validators.go`](../../internal/framework/validators/llm_validators.go)
|
||||
requires exactly one decision for every requested correction index. An empty
|
||||
decision list, missing index, duplicate index, or unexpected index returns an
|
||||
error instead of retaining usable decisions.
|
||||
- `LLMBackedValidator.Validate` consequently marks the entire batch unavailable
|
||||
with `validation_decisions_invalid`. Other batches continue, but usable decisions
|
||||
within that batch are lost.
|
||||
- The runner's cardinality checks enforce an internal contract: each candidate
|
||||
receives one outcome from each validator it reaches. These checks apply to
|
||||
Audita-owned results, not directly to raw model entries.
|
||||
- Deterministic validators already filter individual proposals. Unpreviewable
|
||||
candidates and oversized validation inputs are isolated; recoverable provider
|
||||
failures remain batch-local; application skips individual conflicts.
|
||||
- Ordinary reports distinguish failed operations from successful operations with
|
||||
recoveries. Several failed operations in one stage/chunk share one work-unit
|
||||
error record. The [output contract](../integrations/output-files.md) owns the
|
||||
current counts and serialized representation.
|
||||
|
||||
The defect addressed here is the all-or-nothing domain mapping boundary, not
|
||||
validation itself or the runner's internal consistency checks.
|
||||
|
||||
## Target Decision Semantics
|
||||
|
||||
Reconcile each structurally valid response against the indices actually requested
|
||||
in that particular validator batch. Correction indices, rather than segment IDs
|
||||
or positions in the response array, identify candidates. Several corrections may
|
||||
legitimately target the same segment.
|
||||
|
||||
| Response condition | Target behavior |
|
||||
| --- | --- |
|
||||
| Exactly one valid decision for a requested index | Preserve its approval or rejection and associated decision evidence. |
|
||||
| Requested index omitted | Produce an explicit validation-unavailable outcome for that candidate. |
|
||||
| Requested index appears more than once | Treat that candidate's validation as unavailable; preserve unrelated decisions. |
|
||||
| Index was not requested in this batch | Ignore the extraneous entry and record the handled defect. |
|
||||
| Empty decision list for a nonempty request | Mark all requested candidates unavailable; do not fail the run. |
|
||||
| Decisions appear in a different order | Match by identity and retain deterministic internal results. |
|
||||
|
||||
Duplicate handling does not choose the first or last answer, nor infer a verdict
|
||||
by voting. Even identical duplicates follow the agreed duplicate-index handling;
|
||||
accepting them specially is not needed for this feature. Duplicate extraneous
|
||||
indices remain extraneous entries, not reasons to invalidate requested candidates.
|
||||
|
||||
An unavailable candidate remains unapplied and does not proceed as approved to
|
||||
later validators. It is distinct from a valid `approved: false` decision, which
|
||||
is ordinary editorial processing. Valid approvals continue through the remaining
|
||||
required validators and apply-time checks; this feature grants no bypass.
|
||||
|
||||
The mapper returns one internal outcome for every requested candidate and no
|
||||
outcome for an extraneous index. Preserve strict runner cardinality checks so
|
||||
application bugs are still detected. Do not convert genuine internal invariant
|
||||
failures, cancellation, or fatal setup failures into tolerated model defects.
|
||||
|
||||
## Outcomes and Reporting
|
||||
|
||||
Keep existing work-unit aggregation and operation accounting:
|
||||
|
||||
- If any requested candidate lacks an unambiguous usable decision, record a
|
||||
validation-operation error identifying the affected requested indices. Other
|
||||
candidates' usable decisions from that operation remain effective. An operation
|
||||
error does not mean every candidate in its batch failed.
|
||||
- If all requested candidates have usable decisions and extra entries were
|
||||
ignored, record a successful operation with a recovery for that handled defect.
|
||||
This remains a recovery even if all legitimate decisions are editorial rejections.
|
||||
- If missing/duplicate requested decisions and ignored extras coexist, retain the
|
||||
operation error. Follow the existing rule that failed operations do not also
|
||||
produce recovery records; retain extra-entry details in debug evidence.
|
||||
- A valid complete response with no handled defect adds neither error nor recovery.
|
||||
Valid editorial rejection alone never adds either category.
|
||||
- Preserve actual structural repair and service-retry counts. On successful
|
||||
operations, consolidate applicable recovery reasons in one recovery record.
|
||||
Do not count reconciliation as another LLM operation or structural repair.
|
||||
|
||||
Error attribution must distinguish affected candidates from the rest of the batch.
|
||||
Debug evidence should preserve supplied decisions, ignored entries, and the reason
|
||||
an expected candidate became unavailable. Do not fabricate candidate ledger entries
|
||||
for unknown indices or mislabel missing decisions as deliberate editorial rejection.
|
||||
Ordinary reports must retain safe metadata without raw transcript or response text.
|
||||
|
||||
Use a decision-specific recovery reason rather than treating ignored validator
|
||||
entries as defective correction proposals. Concrete reason names and minimal data
|
||||
model changes belong in the implementation plan. Keep YAML, Markdown, and CLI
|
||||
counts consistent, with the existing reporting-disable and independent debug options.
|
||||
|
||||
## Ownership and Target Architecture
|
||||
|
||||
Audita's validator layer reconciles structurally valid model responses into
|
||||
complete internal candidate outcomes. The runner preserves strict internal
|
||||
cardinality and advances only approved candidates. Reporting consumes operation
|
||||
facts without treating partial validation failure as loss of every decision.
|
||||
Debug evidence distinguishes unavailable validation from editorial rejection.
|
||||
|
||||
The schema/transport boundary remains with PromptKit; reconciliation needs no
|
||||
additional model call. Concrete helper structures, reason names, mapping changes,
|
||||
and verification steps belong in the [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/91d9be68a7bb3a102900253fdfd8438f0a43f666/docs/roadmap/implementation.md).
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
PromptKit's whole-response structural validation and repair remain unchanged.
|
||||
A malformed entry, including an extraneous entry that violates the schema, may
|
||||
still invalidate the response and exhaust structural repairs before reconciliation
|
||||
can run. Ignoring schema-valid extras does not promise salvage from malformed JSON,
|
||||
missing required fields, wrong types, or out-of-range values.
|
||||
|
||||
General per-entry structural salvage is a separate design question, not part of
|
||||
this fix. Likewise, guided regeneration remains deferred for reconsideration after
|
||||
this change; no new retries, feedback prompts, budgets, upstream dependency changes,
|
||||
or operator controls are required here.
|
||||
|
||||
Preserve module scope, literal replacement behavior, fixed chunks, stage barriers,
|
||||
span-deletion safeguards, backend capacity limits, and fatal cancellation behavior.
|
||||
No additional restrictions on valid corrections or guarantees about model judgment
|
||||
are introduced. This work does not itself authorize tagging or publishing v2.0.0.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A schema-valid response can retain usable decisions despite missing, duplicate,
|
||||
or unexpected indices elsewhere in the same batch.
|
||||
- Only the affected requested candidates become unavailable; extras never create
|
||||
candidates or attach decisions to a different batch's candidates.
|
||||
- Valid negative decisions remain editorial outcomes, and every applied correction
|
||||
completes all required validation.
|
||||
- Internal validator results satisfy existing strict cardinality checks.
|
||||
- Missing/duplicate requested decisions yield accurately attributed errors without
|
||||
losing valid work. Extra-only defects yield a consolidated recovery; mixed
|
||||
defects follow existing failed-operation reporting semantics.
|
||||
- Final transcript, debug evidence, YAML reports, Markdown counts, and CLI summary
|
||||
agree about applied work and failures/recoveries.
|
||||
- Reconciliation makes no additional provider calls. Existing structural failures,
|
||||
provider failures, and cancellation retain their documented handling.
|
||||
|
||||
## Open Questions
|
||||
|
||||
No unresolved product-policy questions block implementation. Guided recovery and
|
||||
structural salvage remain separate future decisions.
|
||||
@@ -1,15 +1,15 @@
|
||||
# Deferred Feature Work
|
||||
|
||||
**Last updated:** 2026-09-25
|
||||
**Last updated:** 2026-10-03
|
||||
|
||||
**Status:** Deferred or exploratory work; not implemented or scheduled. These
|
||||
entries preserve scope outside the [resilience roadmap](resilience.md) and
|
||||
[span-deletion consistency roadmap](span-deletion.md). Each entry states its
|
||||
intent; inclusion does not commit us to implementing every capability.
|
||||
entries preserve work beyond the completed resilience and span-deletion
|
||||
features. Each entry states its intent; inclusion does not commit us to
|
||||
implementing every capability.
|
||||
|
||||
## Domain-Aware Response Recovery
|
||||
|
||||
The [partial-success decision reconciliation](decision-set-reconciliation.md)
|
||||
The implemented [partial-success decision reconciliation](../internal/validators.md#llm-validator-batching)
|
||||
preserves usable decisions without another model call. Guided recovery remains
|
||||
deferred pending evidence from failures that remain. General per-entry
|
||||
salvage from structurally invalid responses is also a separate design question;
|
||||
@@ -59,7 +59,7 @@ No new scheduler, global limiter, or execution mode is selected by this entry.
|
||||
|
||||
Potential future work, subject first to evaluating whether it is worth implementing
|
||||
at all. Empty replacement text and deletion of a span within a surviving segment
|
||||
are covered by the accepted [span-deletion consistency roadmap](span-deletion.md).
|
||||
are covered by the implemented [span-deletion behavior](../internal/modules.md#proposal-generation-ownership).
|
||||
That feature does not remove segment records or permit blank resulting segments.
|
||||
|
||||
Before designing segment removal, review representative production examples to
|
||||
|
||||
@@ -1,313 +1,523 @@
|
||||
# Implementation Plan: Standard Policy Alignment
|
||||
# Implementation Plan: PromptKit v0.12.0 Adoption
|
||||
|
||||
**Last updated:** 2026-09-28
|
||||
|
||||
**Status:** Complete; Stages 1–4 implemented.
|
||||
**Audience:** gpt-5.6-terra coding agent.
|
||||
**Audita planning baseline:** `91d9be68a7bb3a102900253fdfd8438f0a43f666`.
|
||||
**Selected upstream revision:** `d84a38ebab26f048cafbe59a2246700c35d29843`.
|
||||
**Last updated:** 2026-10-04
|
||||
**Status:** Complete; Stages 1–9 completed.
|
||||
**Audience:** Luna coding agent.
|
||||
|
||||
## Scope and Execution
|
||||
|
||||
Implement the accepted [Standard Policy Alignment roadmap](policy.md). It owns
|
||||
the intended end state and reasons for local variations; this plan owns ordered
|
||||
implementation and verification. Complete Stage 1 through Stage 4 in order, each
|
||||
as one focused agent task. Record completion evidence under each stage and commit
|
||||
completed units with short, plain English messages under the agent policy.
|
||||
Implement the accepted target in the [PromptKit roadmap](promptkit.md). That
|
||||
document owns feature intent and policy choices; this plan owns the ordered
|
||||
implementation work. The planning baseline is Audita commit
|
||||
`b31955fc97adec707b4e5f9a1b0c43c4cbc66750`: ConfigKit v0.5.0 is adopted,
|
||||
PromptKit is v0.11.2, and the finalized roadmap includes the selected-profile,
|
||||
operator-override, example-layout, and major-release clarifications.
|
||||
|
||||
Begin with [development guidance](../development.md) and the
|
||||
[agent policy](../policy/agents.md). Read the policies governing each stage;
|
||||
reuse unchanged policy context. For policy edits, read that policy and affected
|
||||
cross-references. Follow the adopted selective reading guide once installed.
|
||||
Use graph-first discovery for application code; document and shell/configuration
|
||||
inspection does not need a graph lookup.
|
||||
This replaces the completed ConfigKit plan at this path. Its
|
||||
[completed implementation record](https://gitea.maximumdirect.net/eric/audita/src/commit/b31955fc97adec707b4e5f9a1b0c43c4cbc66750/docs/roadmap/implementation.md)
|
||||
remains available for readers following the ConfigKit completion roadmap.
|
||||
|
||||
Use the public upstream repository at
|
||||
`https://gitea.maximumdirect.net/eric/policy` and check out the exact revision above
|
||||
in a temporary directory outside Audita. Verify its full commit before copying.
|
||||
Read its `docs/adoption.md`, relevant `docs/policy/` files, `COPYRIGHT`, and
|
||||
referenced assets. Do not replace the selected revision with current main or
|
||||
reconstruct prior adoption baselines. The public repository and its attribution
|
||||
are now explicitly approved.
|
||||
Execute Stage 1 through Stage 9 in order. Each is sized as one bounded Luna
|
||||
implementation prompt. Commit completed units with short, plain English messages
|
||||
under the [agent policy](../policy/agents.md); no approval round is needed between
|
||||
stages. Record actual checks and any remaining expected migration failures.
|
||||
Stages 1 and 2 change formats before Stage 3 changes their consumer. Intermediate
|
||||
commits need not be runnable or release-ready. Do not add temporary readers,
|
||||
compatibility shims, fallback engines for execution, or scaffolding to make them
|
||||
so. Later stages complete the same migration rather than separate product modes.
|
||||
|
||||
This plan replaces the completed decision-set plan, preserved at
|
||||
[the planning baseline](https://gitea.maximumdirect.net/eric/audita/src/commit/91d9be68a7bb3a102900253fdfd8438f0a43f666/docs/roadmap/implementation.md).
|
||||
Repair historical roadmap references in Stage 4; do not reopen those features.
|
||||
Start with [development guidance](../development.md) and the applicable agent,
|
||||
[architecture](../policy/architecture.md), [testing](../policy/testing.md),
|
||||
[documentation](../policy/documentation.md), and [OSS](../policy/oss.md)
|
||||
policies. Use graph-first discovery. Read the current
|
||||
[configuration](../config.md), [LLM runtime](../internal/llm-runtime.md),
|
||||
[prompt registry](../internal/prompts.md), and
|
||||
[LLM integration](../integrations/openai-compatible-llm.md) contracts for the
|
||||
established behavior this migration preserves.
|
||||
|
||||
Implement directly toward the final state. Intermediate commits need no temporary
|
||||
policy files, compatibility machinery, or complete dependency inventory. Keep
|
||||
partial work truthful and finish cross-document consistency by Stage 4. Do not
|
||||
release, tag, push, rewrite published material, change dependency versions, or
|
||||
refactor application code as part of this work. No extra approval round is needed
|
||||
between stages. If factual verification uncovers a concrete unresolved rights or
|
||||
publication issue, report it rather than inventing an obligation or claiming it
|
||||
resolved.
|
||||
Target the released PromptKit v0.12.0 at commit
|
||||
`4b8169da286917fde7875425bb32b799b12c6af3`, ConfigKit v0.5.0, and both
|
||||
maintained `/v3` catalogs at `v3.20260930.0`. Read the pinned
|
||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
|
||||
and [migration guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api).
|
||||
The sibling checkout is reference material, not a dependency replacement.
|
||||
|
||||
## Common Adoption Decisions
|
||||
### Selected Implementation Decisions
|
||||
|
||||
- Start each policy from the selected upstream file. Preserve shared substantive
|
||||
wording verbatim. Remove its `Base Policy Template` section after fulfilling
|
||||
the instructions; include only the project customizations selected below.
|
||||
- Retain the upstream base version and set `Last Updated` to the actual adoption
|
||||
date. Add `**Upstream Source:**` with a Markdown link to that same file at the
|
||||
full selected commit, using the URL prefix
|
||||
`https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/`.
|
||||
Do not claim this was the source of old local documents.
|
||||
- Keep discovery URLs in `meta.md` pointed to the public repository and
|
||||
`/src/branch/main/docs/adoption.md`. Pin example links in the adopted OSS policy
|
||||
to this selected commit, after checking their paths. Preserve examples upstream.
|
||||
- Explain substantive local variations briefly where they apply. No separate
|
||||
exception register, synchronization system, policy snapshot tests, or legal
|
||||
approval workflow is needed.
|
||||
- Keep policy credit in the single canonical attribution entry. Describe only
|
||||
material adopted so far, expanding it as later stages copy more material.
|
||||
Metadata and template removal mean an adopted file can truthfully be described
|
||||
as adapted even when its substantive rules are verbatim.
|
||||
- Ordinary link/text review is sufficient for documentation changes. Do not run
|
||||
application tests simply because a policy document changed. Each stage ends
|
||||
with `git diff --check` and review of its staged diff before committing.
|
||||
- Keep Go 1.25.5, application `version: 2`, current flags, role selection,
|
||||
application overlays, and main-file-relative path rules. Retain the
|
||||
`profiles_dir` name and default; broaden its accepted documents as agreed.
|
||||
- Use PromptKit's catalog and owned defaults, then caller underlays for embedded
|
||||
Audita prompts and operator directory inputs, then `Sources.Main` for
|
||||
application backend settings. Directory inputs compose strictly within their
|
||||
group; discovery order supplies no field precedence. Keep imports disabled.
|
||||
- Use one ConfigKit mapping for the final application source. For each
|
||||
application backend, supply its endpoint, `concurrency_limit`,
|
||||
`queue_capacity`, and explicit `api_key_env: null`. The null suppresses
|
||||
lower-tier credential lookup. Leave other supported provider fields available
|
||||
to native layering. Do not put resolved credential values in this mapping.
|
||||
- Keep selected-role registration, endpoint-match, competing-credential, and
|
||||
required-key checks. Add explicit nonblank effective model and endpoint checks
|
||||
for both selected roles. Valid unselected partial profiles remain permitted.
|
||||
- Preserve the adapter's complete per-call schemas, configured repairs/retries,
|
||||
operation context, shared variables, observation, and prepare/check/run flow.
|
||||
Operator messages can omit shared-variable references; do not append the
|
||||
shared text unconditionally or require particular message counts or roles
|
||||
beyond PromptKit's own format rules.
|
||||
- Use captured source inspection for consumed host configuration and message
|
||||
paths. Keep source-relative resource resolution with PromptKit and ConfigKit;
|
||||
Audita projects paths for its existing output collision checks.
|
||||
- Replace the old profile-reload test expectation with snapshot behavior.
|
||||
Successful construction uses only final effective definitions. A failed
|
||||
construction may make an embedded-only diagnostic construction to distinguish
|
||||
asset defects from operator errors; it never executes or substitutes for the
|
||||
requested configuration.
|
||||
- Maintained example roots are `examples/profiles/local/` and
|
||||
`examples/profiles/service/`. Service testing profiles stay in the service
|
||||
root. Keep the literal `temperature: 0` in examples as an explicit outbound
|
||||
setting, and use native presence semantics without a compatibility translator.
|
||||
|
||||
## Stage 1 — Establish Provenance and Shared Agent/Testing Policies
|
||||
This unit is an intentional major-release migration. Exact version selection,
|
||||
release notes, tag/push, and publication timing remain release preparation under
|
||||
the [release policy](../policy/release.md). Guided recovery, new report formats,
|
||||
hot reload, extra configuration commands, and scheduling changes remain outside
|
||||
this plan. Tests should protect changed integration behavior and established
|
||||
contracts, not reproduce the libraries' entire validation matrices.
|
||||
|
||||
**Completed:** 2026-09-28. Adopted meta 1.0, agents 1.3, and testing 1.3 from
|
||||
the verified upstream commit, preserving Audita's graph-first discovery rule.
|
||||
The initial attribution and scoped copyright notice identify only these three
|
||||
adapted policies; the CC-BY-SA text matches upstream byte-for-byte. Substantive
|
||||
policy text, pinned source paths, local links, and `git diff --check` were checked.
|
||||
## Stage 1 — Convert Embedded Prompt Definitions
|
||||
|
||||
**Files:** `docs/policy/meta.md`, `docs/policy/agents.md`,
|
||||
`docs/policy/testing.md`, `docs/oss/attribution.md`,
|
||||
`docs/oss/licenses/CC-BY-SA-4.0.txt`, and `COPYRIGHT`.
|
||||
**Primary files:** The seven YAML files under
|
||||
`internal/prompts/assets/prompts/`.
|
||||
|
||||
1. Adopt meta 1.0, agents 1.3, and testing 1.3 with the common metadata and
|
||||
template-removal rules. Preserve the existing Codebase Discovery section as
|
||||
the one local addition to agents: Audita deliberately uses graph-first
|
||||
discovery. Adopt the remaining agent rules and the testing rules verbatim.
|
||||
2. Create attribution with `# Open Source Attribution Notices`, followed directly
|
||||
by the canonical entry from `examples/licensing/policy-attribution.md`.
|
||||
Substitute the selected commit in its COPYRIGHT, LICENSE, and Source URLs.
|
||||
Initially identify the three newly adapted policy files in its `Material:`
|
||||
sentence. Do not claim the remaining files have already been updated.
|
||||
3. Copy the selected revision's `docs/oss/licenses/CC-BY-SA-4.0.txt` byte-for-byte.
|
||||
Keep the supplied upstream copyright and links to its grant/disclaimer.
|
||||
4. Add the adoption guide's scoped CC-BY-SA notice to Audita's `COPYRIGHT`, with
|
||||
affected material identified through attribution. Preserve Audita's name,
|
||||
copyright, GPL-3.0-or-later application grant, warranty statement, and third-party
|
||||
statement. Leave the root GPLv3 `LICENSE` unchanged.
|
||||
5. Keep metadata local and truthful. `meta.md` points to the local attribution
|
||||
and developer guide even though selective reading guidance is added next.
|
||||
1. Wrap each existing definition beneath `prompts`, its exact current ID, and
|
||||
the string version `v1`. Remove the old inner `id` and `version` fields.
|
||||
Keep filenames and Audita registry identifiers unchanged; filenames are
|
||||
asset locations rather than PromptKit identities.
|
||||
2. Preserve messages, block-scalar text, variable references, task examples,
|
||||
output settings, and schema paths. Preserve the applicable references to
|
||||
`HardeningPolicy`, `ProposalContract`, and `ValidationContract`. Do not edit
|
||||
shared fragments, response schemas, or domain policies to accommodate the
|
||||
new container format.
|
||||
3. Review each before/after definition structurally, including YAML indentation
|
||||
inside block scalars. Do not introduce default profile selections or new
|
||||
repair counts. The adapter continues to own the runtime output contract.
|
||||
|
||||
**Validation and completion:** Compare the substantive agent/testing text directly
|
||||
with the selected upstream documents, allowing only template removal, metadata,
|
||||
and the identified discovery addition. Verify each new source link's exact file
|
||||
and revision, local attribution links, copied license bytes, and narrow grant
|
||||
scope. No application test or publication is required.
|
||||
**Verification and completion:** Check all seven migrated identities and
|
||||
version keys against `RegisteredMetadata`, and run `git diff --check`. Registry
|
||||
preparation tests become runnable with the new constructor in Stage 3. Commit
|
||||
the asset conversion; do not introduce an old-format reader to test it here.
|
||||
|
||||
## Stage 2 — Customize Remaining Policies and Install Selective Orientation
|
||||
**Completion marker:** Stage 1 complete. All seven registered prompt assets
|
||||
use keyed prompt/version mappings; parsed definitions were structurally checked
|
||||
against their prior content, and `git diff --check` passed.
|
||||
|
||||
**Completed:** 2026-09-28. Adopted architecture 1.3, documentation 1.11,
|
||||
release 1.4, and OSS 1.8 with the selected Audita variations. Installed the
|
||||
upstream root orientation and verbatim Policy Reading Guide, expanded the
|
||||
scoped attribution, and repaired the release documentation heading link.
|
||||
Checked versions, provenance and example destinations, local links, policy
|
||||
differences, and `git diff --check`.
|
||||
## Stage 2 — Migrate Operator Profiles, Example Roots, and YAML Fixtures
|
||||
|
||||
**Files:** `docs/policy/architecture.md`, `docs/policy/documentation.md`,
|
||||
`docs/policy/release.md`, `docs/policy/oss.md`, `AGENTS.md`,
|
||||
`docs/development.md`, `docs/oss/attribution.md`, and `COPYRIGHT` as needed for
|
||||
accurate copied-guidance scope.
|
||||
**Primary files:** `examples/profiles/`, the three maintained configuration
|
||||
examples, and profile-producing test fixtures in
|
||||
`internal/framework/llm/{runtime_inspection,adapter,adapter_capacity}_test.go`,
|
||||
`internal/framework/proposal_generation/adapter_integration_test.go`,
|
||||
`internal/framework/validators/llm_integration_test.go`,
|
||||
`internal/framework/runner/cancellation_test.go`, and
|
||||
`cmd/audita/main_integration_test.go`.
|
||||
|
||||
1. Adopt architecture 1.3. Retain the upstream general Purpose And Ownership,
|
||||
Architectural Approach, and Adopted Constraints And Implemented Structure
|
||||
guidance. Replace Application-Specific Topics with Audita's established
|
||||
purpose, ownership/dependency boundaries, correction/concurrency rules,
|
||||
recovery, state/compatibility, secret handling, and selected library ownership.
|
||||
Reuse existing local decisions; avoid duplicating general upstream prose.
|
||||
Replace example threat discussion with the existing secret/input boundaries
|
||||
and canonical links where useful. Do not infer new threat exclusions,
|
||||
filesystem guarantees, interfaces, or package restructuring from the template.
|
||||
2. Adopt documentation 1.11. Keep applicable upstream ownership rows verbatim;
|
||||
omit public HTTP and public Go API rows. Add the existing concrete ownership
|
||||
of `docs/integrations/output-files.md` and contributor change recipes, and
|
||||
retain the deliberate retirement of standalone troubleshooting documentation.
|
||||
Account for the retained agent discovery/commit responsibilities in its owner
|
||||
row. Keep reading triggers in development guidance, not a second policy table.
|
||||
3. Adopt release 1.4. Retain upstream completion, validation, guard, immutability,
|
||||
recovery, and preparation language. Preserve the existing Supported Platforms
|
||||
table and unsupported-platform statement as Audita's support commitment.
|
||||
Replace the upstream discretionary human-gate paragraph with Audita's selected
|
||||
rule against manual approvals, sign-offs, and routine confirmations as release
|
||||
gates. Remove the obsolete list describing now-shared upstream rules as local
|
||||
deviations. Keep concrete atomic-push mechanics in the runbook.
|
||||
4. Adopt OSS 1.8. Add a concise Audita context paragraph identifying the existing
|
||||
GPL-3.0-or-later grant and linking to COPYRIGHT and the release procedure for
|
||||
Gitea distribution. Use the upstream House Format and interpretation language
|
||||
verbatim. Pin application/library COPYRIGHT and attribution examples to the
|
||||
selected upstream commit; remove template-only setup text. No license change
|
||||
to application code or blanket dependency compatibility claim is authorized.
|
||||
5. Replace root orientation with upstream `agents/AGENTS.md` wording while
|
||||
retaining any existing applicable repository instructions. Keep detailed
|
||||
discovery rules in their current agent-policy owner rather than duplicating
|
||||
them in a newly created block.
|
||||
6. Insert the adoption guide's Policy Reading Guide and accompanying reading rules
|
||||
into `docs/development.md` verbatim. Remove the blanket architecture-reading
|
||||
instruction; preserve contributor commands and component-specific task rows.
|
||||
Adjust cross-links that would otherwise contradict selective reading.
|
||||
7. Expand the single policy attribution entry to identify all `docs/policy/`
|
||||
documents, adapted root orientation, and the copied Policy Reading Guide in
|
||||
`docs/development.md`. Scope the COPYRIGHT CC-BY-SA notice to those materials,
|
||||
not unrelated contributor prose or application documentation. Keep the material
|
||||
sentence factual and concise; retain the pinned upstream credit links.
|
||||
8. Find and repair local links to policy headings renamed by adoption, including
|
||||
the release correction heading. Do not edit published historical release
|
||||
notes to retrofit new headings or licensing conventions.
|
||||
1. Move local proposal/validation examples beneath `profiles/local/`; move the
|
||||
four service proposal/validation and testing files beneath
|
||||
`profiles/service/`. Convert each to `profiles: {<ID>: <fields>}` in readable
|
||||
YAML. Keep IDs, models, positive timeouts, and controls, including explicit
|
||||
zero temperature. No example needs a real credential or live model.
|
||||
2. Set `profiles_dir: profiles/local` in `examples/minimal-config.yml` and
|
||||
`examples/configkit-inheritance.yml`; set `profiles_dir: profiles/service`
|
||||
in `examples/production-config.yml`. Preserve the testing overlay's role IDs
|
||||
and the existing ConfigKit inheritance example. Select the leaf root, since
|
||||
recursive discovery at their common parent would still load both groups.
|
||||
3. Convert profile YAML emitted by the listed fixtures to keyed definitions.
|
||||
Search for remaining old profile construction strings and removed
|
||||
`base_profile` syntax, and migrate any genuine profile fixture found.
|
||||
Do not rewrite transcript/proposal `id` fields or domain schema fixtures.
|
||||
Keep test source names independent of profile IDs where already exercised.
|
||||
4. Keep negative tests negative for their selected behavioral reason. Give
|
||||
cases clean temporary layouts or replace their contribution completely
|
||||
rather than leaving an unrelated invalid definition behind for later cases.
|
||||
Eager validation now reads unselected final definitions too. Do not remove
|
||||
such definitions merely to simulate the old lazy loader where the test is
|
||||
intended to exercise eager validation.
|
||||
|
||||
**Validation and completion:** Compare all seven policies to the pinned source;
|
||||
review every substantive difference against the roadmap's justified variations.
|
||||
Check versions (1.3, 1.3, 1.11, 1.3, 1.4, 1.0, 1.8 for agents, architecture,
|
||||
documentation, testing, release, meta, OSS respectively), file-specific source
|
||||
links, removed template sections, example destinations, ownership consistency,
|
||||
and selective reading. Temporary pending release/attribution implementation is
|
||||
completed by the following stages; do not create scaffolding for it.
|
||||
**Verification and completion:** Review the six profiles, explicit root
|
||||
selections, and changed fixture text; run `git diff --check` and format touched
|
||||
Go test files. These formats become consumable in Stage 3. Commit this unit.
|
||||
Current documentation links and recipes are updated together in Stage 8.
|
||||
|
||||
## Stage 3 — Complete Component Attribution and README Navigation
|
||||
**Completion marker:** Stage 2 complete. Six maintained profile documents are
|
||||
keyed and split into separately selected local/service roots; the listed Go
|
||||
fixtures use keyed definitions. Touched Go files were formatted and
|
||||
`git diff --check` passed. Runtime tests are deferred until Stage 3 replaces
|
||||
the current consumer, as the plan allows intermediate commits not to run.
|
||||
|
||||
**Completed:** 2026-09-28. Both CGO-disabled Linux target dependency lists
|
||||
selected the same eight production modules. Inspected their selected source
|
||||
notices and copied assets, excluded test-only dependencies, and resolved the
|
||||
Go 1.25.5 source tag and its four incorporated vendored modules. Added 14
|
||||
House Format entries including policy provenance, complete applicable license
|
||||
texts, and the README footer. Checked 33 pinned source and notice destinations,
|
||||
license bytes, local links, entry shape, and `git diff --check`.
|
||||
## Stage 3 — Adopt the Released Source API and Selected-Role Checks
|
||||
|
||||
**Files:** `docs/oss/attribution.md`, applicable texts under
|
||||
`docs/oss/licenses/`, and `README.md`. Add factual exception treatment to the OSS
|
||||
policy only if the actual inventory uncovers a concrete need.
|
||||
**Primary files:** `go.mod`, `go.sum`,
|
||||
`internal/framework/llm/runtime.go`, and `internal/prompts/registry_test.go`.
|
||||
|
||||
1. Inventory actual incorporated material using the selected modules, production
|
||||
package dependencies for both Linux targets, source/assets, and shipped Go
|
||||
runtime. Use `go list -deps` for `./cmd/audita` with `CGO_ENABLED=0` and each
|
||||
target architecture, plus source inspection for copied material. Inspect test
|
||||
dependencies where relevant to included material; a go.mod entry or tool use
|
||||
alone is not sufficient reason for attribution. Do not change go.mod/go.sum.
|
||||
2. At the planning baseline, production dependencies include ConfigKit v0.4.1,
|
||||
PromptKit v0.11.1, both catalogs v2.20260926.0, jsonschema/v6 v6.0.2,
|
||||
yaml/v4 v4.0.0-rc.6, x/text v0.14.0, and yaml.v3 v3.0.1. Start with the four
|
||||
verified source commits recorded in the feature roadmap; resolve remaining
|
||||
module versions to exact source commits and inspect their COPYRIGHT, LICENSE,
|
||||
NOTICE, PATENTS, and relevant bundled material as applicable. Account for
|
||||
additional material discovered in that inspection rather than treating this
|
||||
starting list as an exhaustive guarantee.
|
||||
3. For the Go runtime, use the release build's configured Go 1.25.5 source tag
|
||||
and resolve it to its full source commit. Do not silently substitute the
|
||||
inspecting agent's installed toolchain (Go 1.26.8 during planning). Preserve
|
||||
actual runtime/standard-library notices, including relevant bundled third-party
|
||||
material. Document maintenance of this entry when the distributed toolchain
|
||||
changes; do not pin new toolchain behavior or reject other supported local Go
|
||||
versions merely to simplify attribution.
|
||||
4. Add entries following OSS 1.8 exactly, with selected module/version or material
|
||||
scope where useful. Preserve supplied copyrights and accurate license
|
||||
expressions; use the prescribed link labels and `[Source](COMMIT_URL).`
|
||||
spelling. Keep policy attribution first. Order other entries consistently by
|
||||
project name; ordering is presentation, not a new public contract.
|
||||
5. Copy applicable complete standard texts from verified sources, using the
|
||||
pinned reference collection where applicable. Share identical standard texts
|
||||
and include required companions (LGPLv3 and GPLv3 for the two libraries).
|
||||
Include upstream commit links when a file contains project-specific material
|
||||
in addition to standard wording. Preserve existing incorporated notices.
|
||||
6. Apply the adopted house interpretation for project-specific notice links.
|
||||
Put concrete exception reasoning in OSS policy or an appropriate decision
|
||||
record, never in attribution. Do not add missing-copyright filler, research
|
||||
narratives, inferred licensing choices, or legal conclusions to entries.
|
||||
7. End README with the two OSS statements, identifying Audita as GPL v3.0 or later
|
||||
and linking to `COPYRIGHT` and `docs/oss/attribution.md`. The linked grant owns
|
||||
the separately licensed policy scope; no duplicated notice list is needed.
|
||||
1. Upgrade with `go get gitea.maximumdirect.net/eric/promptkit@v0.12.0` and
|
||||
reconcile the module files. Verify ConfigKit stays v0.5.0, the catalog
|
||||
dependencies become `/v3` at `v3.20260930.0`, and obsolete `/v2` requirements
|
||||
are removed when unused. Retain the Go minimum and unrelated selections;
|
||||
introduce no local replacements.
|
||||
2. Resolve `profiles_dir` exactly as the current runtime does. Obtain embedded
|
||||
inputs with `pk.FilesystemInputs` and operator inputs with `pk.DirectoryInputs`.
|
||||
An unavailable selected root is an operator error; an empty directory can
|
||||
contribute no definitions. Do not require one profile per file, IDs matching
|
||||
filenames, or profile-only top-level content.
|
||||
3. Construct the agreed `pk.ConfigurationSources`: two ordered caller underlays
|
||||
for embedded and operator groups, and one highest-priority application
|
||||
`ck.Input` whose source is the mapping specified above. Retain
|
||||
`pk.WithSchemaFS`, the supplied HTTP client, and configured transient retries.
|
||||
Remove old prompt/profile/backend registration options.
|
||||
4. Inspect registered prompt ID/version selections, then both selected roles.
|
||||
Reject a selected role whose effective model or endpoint is empty or
|
||||
whitespace-only. Retain backend registration, normalized endpoint equality,
|
||||
nonblank effective `APIKeyEnv` rejection, and required-credential checks.
|
||||
Preserve literal IDs; do not trim/re-key identities or require all profiles
|
||||
to be complete. Populate the same role inspection map for both clients.
|
||||
5. Migrate the registry test engine to `FilesystemInputs` and a programmatic
|
||||
`ProfileSource` fixture in its own tier, retaining its schema filesystem.
|
||||
A complete endpoint/model fixture without an application backend is valid
|
||||
for this direct library test; Audita's selected-role guard belongs in its
|
||||
runtime tests. Keep the embedded prompt preparation assertions scoped to
|
||||
embedded prompts.
|
||||
6. Remove the old `profileSourceFS` wrapper once unused. Source accounting is
|
||||
completed in Stage 5. Preserve the adapter boundary; change other production
|
||||
callers only to resolve actual API incompatibilities. Search for remaining
|
||||
uses of removed source options and old `pk.Profile`/`pk.Backend` values.
|
||||
Run `go mod tidy` after migrating construction callers and review its diff.
|
||||
|
||||
**Validation and completion:** Inspect entries against selected source evidence,
|
||||
resolve every source/notice commit and path, compare standard license bytes, and
|
||||
check local links. Confirm that actual distributed material is accounted for
|
||||
without blindly including dependency graph or development-tool entries. Record
|
||||
concise inventory/check evidence under this stage; no new SBOM, scanner, durable
|
||||
inventory generator, or test suite is required.
|
||||
**Verification and completion:** Run `go test ./internal/prompts` and the
|
||||
existing offline profile assembly, unsafe-selection, and absolute-directory
|
||||
tests in `internal/framework/llm`. Run `git diff --check` and review the module
|
||||
diff. The obsolete file-reload assertion is intentionally replaced in Stage 6;
|
||||
do not restore reload behavior to make that assertion pass. Commit the runtime
|
||||
API migration.
|
||||
|
||||
## Stage 4 — Integrate Release Notes and Finish Cross-Document Validation
|
||||
**Completion marker:** Stage 3 complete. Runtime construction now uses the
|
||||
released PromptKit source API with the selected application mapping; module
|
||||
selections match PromptKit v0.12.0 and both `/v3` catalogs at
|
||||
`v3.20260930.0`. `go mod tidy`, prompt registry tests, offline profile
|
||||
assembly, unsafe-selection, absolute-directory tests, and `git diff --check`
|
||||
passed. Captured source accounting remains in Stage 5.
|
||||
|
||||
**Completed:** 2026-09-28. Wired the checked-in release note into the hosted
|
||||
release plugin, documented immutable-tag notice links and release preparation,
|
||||
repaired the specified historical roadmap references, and marked the feature
|
||||
roadmap implemented. Parsed the YAML, substituted `v2.0.0` to a tracked note,
|
||||
checked non-overwrite settings and candidate footer paths, reviewed all seven
|
||||
policies against the pinned source, and checked repository-local Markdown links
|
||||
and `git diff --check`. Hosted execution awaits the next real release.
|
||||
## Stage 4 — Preserve Startup Failure Classification and Validate Assembly
|
||||
|
||||
**Files:** `.woodpecker/release.yml`, `docs/release.md`, affected documentation
|
||||
links, `docs/roadmap/policy.md`, and this plan. Repair the historical references
|
||||
specified below; do not retire unrelated roadmaps in this sprint.
|
||||
**Primary files:** `internal/framework/llm/runtime.go`,
|
||||
`internal/framework/llm/runtime_inspection_test.go`, and configuration-command
|
||||
tests in `internal/cli`. Update `internal/cli/run.go` only where the changed
|
||||
startup path needs existing exit behavior preserved.
|
||||
|
||||
1. Add this setting to the existing `publish-release` step:
|
||||
1. Treat embedded discovery failures as `AssetError`; treat operator-directory
|
||||
discovery failures as configuration input errors. Preserve wrapped causes
|
||||
for `errors.Is`/`errors.As` and retain configured-secret redaction at CLI
|
||||
boundaries. Caller cancellation remains a runtime exit, not an operator
|
||||
mistake, and should not trigger further diagnostic loading.
|
||||
2. PromptKit wraps both definition-loading and template/content failures in
|
||||
`ErrInvalidConfig`; some materialization errors have no public source/path
|
||||
type. Do not classify by parsing error strings or by that sentinel alone.
|
||||
On final engine-construction failure, perform an embedded-only diagnostic
|
||||
construction with the same embedded prompt/schema sources and library
|
||||
defaults, excluding operator inputs and application backend mappings. If
|
||||
that also fails, surface an `AssetError` with the asset cause; otherwise
|
||||
return the original final-construction error as an operator/configuration
|
||||
error. The diagnostic engine does no provider work and is never retained
|
||||
for execution. Keep a small unexported assembly seam accepting an embedded
|
||||
filesystem so tests can exercise asset failures without modifying shipped
|
||||
files or creating a global test hook.
|
||||
3. Do not run this diagnostic construction as a successful-path preflight.
|
||||
A valid final snapshot remains accepted even if an operator contribution
|
||||
repairs an otherwise invalid lower-tier prompt. Verify that property with
|
||||
the same small assembly test seam. This avoids strengthening upstream
|
||||
final-value validation into mandatory validation of displaced definitions.
|
||||
4. Cover incomplete selected profiles using omitted, empty, and null model
|
||||
contributions and a cleared selected endpoint. Exercise proposal and
|
||||
validation selection, including omission-derived validation selection, with
|
||||
a rejecting HTTP transport. Confirm valid unselected partial profiles are
|
||||
accepted; malformed unselected definitions, undefined backend references,
|
||||
and conflicting directory contributions still fail under native rules.
|
||||
5. Add focused boundary coverage that invalid operator YAML/template/resource
|
||||
input produces configuration exit `2`, and test the asset classification
|
||||
directly at the runtime seam. Reuse existing CLI handling of `AssetError`
|
||||
for exit `1`; no damaged-binary subprocess fixture is needed. Cover the
|
||||
shared `NewRuntime` validation route and preserve cancellation exit `1` in
|
||||
configuration commands if the new error path would otherwise classify it
|
||||
as `2`. Avoid exact complete diagnostic wording assertions.
|
||||
|
||||
```yaml
|
||||
note: docs/releases/${CI_COMMIT_TAG}.md
|
||||
**Verification and completion:** Run targeted startup/inspection tests and
|
||||
`go test ./internal/cli`; check whitespace and commit. Preserve the original
|
||||
failure cause where available rather than flattening it to a generic error.
|
||||
|
||||
**Completion marker:** Stage 4 complete. Embedded discovery and construction
|
||||
failures are distinguished from operator inputs through an embedded-only
|
||||
diagnostic construction; cancellation skips that diagnostic. Selected-profile
|
||||
completeness and native directory validation are covered, and both config
|
||||
commands preserve exit 1 for cancellation/assets and exit 2 for invalid
|
||||
operator YAML, templates, and resources. Targeted runtime tests,
|
||||
`go test ./internal/cli`, and `git diff --check` passed.
|
||||
|
||||
## Stage 5 — Protect Consumed Configuration and Message Files
|
||||
|
||||
**Primary files:** `internal/framework/llm/runtime.go` or a focused source-path
|
||||
helper in that package, `internal/cli/process.go`, runtime source-accounting
|
||||
tests, and `cmd/audita/main_integration_test.go`.
|
||||
|
||||
1. Rename the internal `Runtime.ProfilePaths` field to `SourcePaths` and update
|
||||
its process caller. It now covers consumed host YAML and message resources,
|
||||
rather than profiles alone. Populate it from the successfully captured final
|
||||
engine; do not expose source metadata as a new public output.
|
||||
2. Call `InspectConfiguration`. Index `Sources()` by source ID, and collect
|
||||
paths with `Kind == ck.SourceKindFile`; embedded filesystem and synthetic
|
||||
sources are not host paths. All loaded host YAML counts, including unused
|
||||
definitions and files whose contributions were later overlaid. Deduplicate
|
||||
paths and sort for a stable internal projection.
|
||||
3. For message resources, enumerate `Metadata().HistoricalPaths()` and retain
|
||||
effective, nonnull paths shaped as
|
||||
`/prompts/<id>/<version>/messages/<index>/content_file`. Use `Lookup` and the
|
||||
effective field's origin, not an inherited destination's guessed source.
|
||||
PromptKit successfully captures these fields only with a unique usable
|
||||
source context. Obtain the effective scalar from `Explain(path).Effective`
|
||||
and decode that scalar YAML into a string with the existing YAML dependency.
|
||||
This decodes captured inspection data; it is not a second configuration
|
||||
loader. When the contributing source is a host file, join the untrimmed
|
||||
resource name against that source file's directory and add it to the ledger.
|
||||
Ignore displaced references and non-host resources. Do not reread operator
|
||||
YAML, infer IDs from filenames, or reimplement resource-root validation.
|
||||
4. Pass `SourcePaths` through existing `diagnostics.PrepareOutput` protection.
|
||||
Leave canonicalization, symlink/hardlink checks, permitted in-place transcript
|
||||
handling, and output initialization with their current owner. Do not protect
|
||||
every unrelated file in the selected directory or add filesystem mutation
|
||||
guarantees beyond the existing collision contract.
|
||||
5. Extend `TestSubprocessProfileSources` and a focused runtime case to cover an
|
||||
arbitrarily named YAML contribution, a consumed source-relative message
|
||||
file, and conflicting transcript/report destinations. Use a non-YAML message
|
||||
filename, such as `instruction.md`, so directory discovery does not mistake
|
||||
plain prompt text for a configuration document. Include an inherited message
|
||||
reference from a different directory and a prompt ID/version with a JSON
|
||||
Pointer escape character so source provenance and literal keys determine
|
||||
the path. Check rejection before provider dispatch and unchanged input bytes.
|
||||
Preserve acceptance of an unused alternate output filename; do not turn the
|
||||
whole directory into a forbidden output location.
|
||||
|
||||
**Verification and completion:** Run focused source-ledger tests and subprocess
|
||||
collision cases, plus `git diff --check`, then commit. Check that resource
|
||||
accounting uses the frozen snapshot and never changes the engine's source or
|
||||
resource-resolution behavior to facilitate tracking.
|
||||
|
||||
**Completion marker:** Stage 5 complete. `Runtime.SourcePaths` now projects
|
||||
host YAML and effective source-relative message files from PromptKit's captured
|
||||
configuration inspection, using field provenance for inherited references.
|
||||
Process output protection consumes that ledger. Focused source-accounting and
|
||||
subprocess collision tests passed, including unchanged input bytes, no provider
|
||||
dispatch, JSON Pointer-escaped keys, and acceptance of an unused alternate
|
||||
output; `git diff --check` passed.
|
||||
|
||||
## Stage 6 — Verify Operator Prompt Overrides and Captured Execution
|
||||
|
||||
**Primary files:** `internal/framework/llm/adapter_test.go` and focused runtime
|
||||
tests. Change `adapter.go` only for an actual integration defect.
|
||||
|
||||
1. Add a native keyed operator contribution overriding messages for an active
|
||||
prompt/version. Use the real runtime and controlled HTTP responses to prove
|
||||
the operator text reaches both the prepared snapshot and dispatched request.
|
||||
One case deliberately omits shared-variable references and still succeeds;
|
||||
do not require Audita to inject their text. A reference-bearing case renders
|
||||
the adapter-supplied value and retains existing protection against caller
|
||||
variables replacing the reserved shared variables.
|
||||
2. Override the prompt's definition-level output contract with another valid
|
||||
upstream contract. Prove that Audita's per-call JSON schema and configured
|
||||
repair budget still govern execution: a structurally invalid response is
|
||||
repaired or exhausted as the controlled budget dictates. Domain validators
|
||||
remain downstream. Reuse existing validator/proposal integration coverage
|
||||
for their rules rather than weakening them to fit an operator prompt.
|
||||
3. Replace the reload portion of
|
||||
`TestPreparedAdapterRejectsTargetDriftAndPreparationErrors`. After successful
|
||||
construction, change a profile or message file and verify the existing
|
||||
runtime still uses its captured settings/text. Construct a new runtime to
|
||||
observe a valid edit or reject a newly mismatched endpoint. Keep the missing
|
||||
prompt preparation failure case.
|
||||
4. Retain a focused test of the prepared-target check independently of reload:
|
||||
deliberately change the test runtime's application backend endpoint after
|
||||
construction and verify the inconsistency is blocked before dispatch. This
|
||||
tests the existing guard, not a supported live mutation/reload feature.
|
||||
Continue executing the same prepared handle whose details were inspected;
|
||||
do not add another preparation or an application-side retry loop.
|
||||
|
||||
**Verification and completion:** Run the prompt override, snapshot, and
|
||||
prepared-adapter tests. Compare semantic messages and selected targets, not old
|
||||
hash values or exact YAML formatting. Check whitespace and commit.
|
||||
|
||||
**Completion marker:** Stage 6 complete. Native operator messages reach both
|
||||
the prepared snapshot and provider request, with and without shared-variable
|
||||
references. The adapter's per-call schema and repair budget remain effective
|
||||
under a valid operator output-contract override. Existing runtimes retain
|
||||
captured prompt/profile values after source edits, and a new runtime observes
|
||||
valid edits; the target guard and preparation failure remain covered. The
|
||||
`internal/framework/llm` tests and `git diff --check` passed.
|
||||
|
||||
## Stage 7 — Verify Native Profiles, Backend Authority, and Resilience
|
||||
|
||||
**Primary files:** Runtime inspection and adapter tests,
|
||||
`internal/framework/llm/adapter_capacity_test.go`, and existing framework/CLI
|
||||
integration tests where their established behavior is affected.
|
||||
|
||||
1. Add a compact native profile inheritance fixture with literal keyed IDs
|
||||
unrelated to filenames, multiple definitions in one file, a partial parent,
|
||||
and a complete selected child. Verify inherited controls survive a child
|
||||
model change and child overrides win. Reuse the metadata-facing literal-key
|
||||
coverage from Stage 5 rather than repeating it in profile tests.
|
||||
2. Use controlled HTTP request bodies to distinguish explicit zero temperature
|
||||
from omission and inherited-control suppression by null. Check a
|
||||
representative recursively overlaid provider parameter and null clearing;
|
||||
do not promise that all providers accept every valid configured zero.
|
||||
Reuse existing deadline/cancellation coverage for unchanged outer deadlines;
|
||||
explicit zero profile timeout must not disable the operation context.
|
||||
3. Verify an operator backend contribution can add supported provider fields,
|
||||
while a conflicting endpoint/concurrency/queue/environment contribution
|
||||
cannot displace the final application fields. Preserve an explicit profile
|
||||
environment lookup as a selected-role error rather than hiding it with an
|
||||
adapter override. Include an application backend using a catalog ID to show
|
||||
it is allowed and its catalog credential lookup is suppressed. Use synthetic
|
||||
environment values and local HTTP fixtures, not provider catalog endpoints.
|
||||
4. Reuse `TestRolesShareBackendCapacityAndReleaseAfterCancellation` and
|
||||
`TestDistinctBackendsHaveIndependentCapacity`; verify the final mapped
|
||||
limits, including zero queue capacity, still coordinate both roles and
|
||||
remain backend-local. Strengthen a fixture with conflicting lower-tier
|
||||
capacity only where needed to prove the new mapping wins. Keep existing
|
||||
synchronization; do not infer admission with sleeps or add another limiter.
|
||||
5. Preserve offline selected-profile inspection and value-free application
|
||||
explanations. Check configured secrets, endpoint strings, arbitrary provider
|
||||
fields, and prompt text remain outside the current safe profile summary.
|
||||
Do not replace it with a full upstream dump. Its numeric projection is not
|
||||
proof that an omitted provider setting is sent; wire presence belongs to
|
||||
the HTTP coverage above.
|
||||
6. Run the existing transient retry, structural repair, attempt-count,
|
||||
observation, partial-validation, and all-model-failure subprocess cases
|
||||
after fixture migration. Preserve their error/recovery/debug classifications,
|
||||
cancellation, and continued output. Fix proven integration regressions;
|
||||
do not rewrite domain outcomes or reduce budgets to obtain a passing suite.
|
||||
|
||||
**Verification and completion:** Run
|
||||
`go test ./internal/framework/llm ./internal/framework/proposal_generation ./internal/framework/validators ./internal/framework/runner ./internal/cli ./cmd/audita`
|
||||
and `git diff --check`. Review whether each new test addresses a distinct changed
|
||||
boundary. Commit the native-profile and runtime verification unit.
|
||||
|
||||
**Completion marker:** Stage 7 complete. Native profile inheritance, control
|
||||
presence, provider-map overlays, application backend authority, selected
|
||||
credential rejection, catalog-ID handling, shared backend capacity, and safe
|
||||
inspection behavior are covered. The six targeted packages passed, along with
|
||||
focused wire, timeout, capacity, credential, and safe-view tests; `git diff
|
||||
--check` passed.
|
||||
|
||||
## Stage 8 — Document the Migration and Update Attribution
|
||||
|
||||
**Primary files:** `docs/config.md`, `docs/internal/llm-runtime.md`,
|
||||
`docs/internal/prompts.md`, `docs/integrations/openai-compatible-llm.md`,
|
||||
`docs/operations.md`, `docs/internal/cli-config.md`, `README.md`, and
|
||||
`docs/oss/attribution.md`.
|
||||
|
||||
1. Update the configuration contract for unified `profiles_dir` content, exact
|
||||
keyed identities, directory composition, agreed tier precedence, profile
|
||||
inheritance/presence, selected-role completeness, eager validation of all
|
||||
final definitions, selected-root failures, and construction-time capture.
|
||||
Replace the obsolete statement that PromptKit profiles do not gain ConfigKit
|
||||
inheritance. Retain application overlays, defaults, and credential ownership.
|
||||
2. Include concise migration instructions in the profile section: wrap old
|
||||
profile fields beneath `profiles/<id>`, remove inner `id`, change
|
||||
`base_profile` to `extends`, review explicit values/nulls, and select roots
|
||||
whose final definitions have valid backend references. Show a small keyed
|
||||
inheritance/override snippet and link complete maintained examples. Explain
|
||||
explicit zero temperature versus omitted or null controls and link upstream
|
||||
for the full field table. State that this ships as an Audita major migration;
|
||||
do not invent an exact release version or automatic file conversion.
|
||||
3. Document intentional prompt overrides and optional shared-variable references.
|
||||
Supplying shared variables does not inject omitted text, while per-call
|
||||
schemas and domain validators remain enforced. Internal prompt documentation
|
||||
distinguishes shipped asset instructions from operator-effective messages;
|
||||
no prompt-quality rewrite or new global prompt-policy requirement is added.
|
||||
4. Describe layered runtime assembly, source accounting, snapshot use, and
|
||||
startup classification in their internal owner. Update the pinned PromptKit
|
||||
consumer link in the LLM integration contract. Expand operations' loaded
|
||||
source description to include consumed operator message files; keep physical
|
||||
output/collision rules with operations and diagnostics.
|
||||
5. Repair example navigation for separate local/service roots, especially the
|
||||
README's local-profile link. Update validation recipes to use the maintained
|
||||
config files' explicit roots and keep the service testing overlay workflow.
|
||||
Review other affected links without rewriting historical release notes.
|
||||
6. Update the PromptKit and both catalog attribution entries to selected versions
|
||||
and exact commit-pinned source/notice URLs. Resolve catalog tag commits from
|
||||
the selected module origins or authoritative upstream refs, not annotated
|
||||
tag object IDs. Check the releases' COPYRIGHT/LICENSE files and retain the
|
||||
applicable unchanged local LGPL/GPL and MIT texts. Keep the existing House
|
||||
Format; no new license selection, policy rewrite, or attribution inventory
|
||||
is needed. Review `go.mod`/`go.sum` against those entries.
|
||||
|
||||
**Verification and completion:** Check affected local links/anchors, format
|
||||
examples against the implemented loader, verify source revisions/notices, and
|
||||
run `git diff --check`. Commit the documentation and attribution unit.
|
||||
|
||||
**Completion marker:** Stage 8 complete. Configuration, runtime, prompt,
|
||||
integration, operations, contributor, and README guidance now reflects native
|
||||
PromptKit layering, migration, presence, snapshot, and separate example-root
|
||||
behavior. Attribution matches PromptKit v0.12.0 and both v3 catalog module
|
||||
origins and notices. Local links/anchors and `git diff --check` passed.
|
||||
|
||||
## Stage 9 — Validate the Integrated End State and Close the Plan
|
||||
|
||||
**Primary files:** This plan and the feature roadmap; correct an implementation
|
||||
file only if the final checks reveal an actual gap.
|
||||
|
||||
1. Verify the full roadmap acceptance criteria against the landed code and the
|
||||
distinct tests from these stages. Review for unintended requirements:
|
||||
profile-only filtering, mandatory shared text in operator messages,
|
||||
filename-derived IDs, complete unselected profiles, catalog ID bans,
|
||||
successful-path asset preflights, compatibility readers, and new output
|
||||
guarantees are not part of the accepted target.
|
||||
2. Run `go build -o ./bin/audita ./cmd/audita` and `go test ./...`. Keep the
|
||||
generated binary out of commits. Reuse successful checks on unchanged
|
||||
content; rerun checks affected by any corrective code changes.
|
||||
3. Validate the maintained examples offline using the built binary:
|
||||
|
||||
```sh
|
||||
./bin/audita config validate --config examples/minimal-config.yml
|
||||
./bin/audita config validate --config examples/configkit-inheritance.yml
|
||||
AUDITA_SERVICE_KEY=synthetic-example ./bin/audita config validate --config examples/production-config.yml
|
||||
AUDITA_SERVICE_KEY=synthetic-example ./bin/audita config validate --config examples/production-config.yml --overlay testing
|
||||
./bin/audita config print-effective --config examples/minimal-config.yml --view profiles
|
||||
./bin/audita config print-effective --config examples/configkit-inheritance.yml --view explain
|
||||
```
|
||||
|
||||
The [official release-plugin documentation](https://woodpecker-ci.org/plugins/release)
|
||||
supports a file path in `note`; the
|
||||
[environment reference](https://woodpecker-ci.org/docs/usage/environment)
|
||||
makes `CI_COMMIT_TAG` available for configuration substitution. These interfaces
|
||||
were checked during planning. Use the checked-out tagged note directly; no
|
||||
generator, custom publisher, or additional dependency is needed. Preserve
|
||||
`overwrite: false`, `file-exists: skip`, existing assets/checksum settings,
|
||||
secret reference, image, and step dependency.
|
||||
2. In `docs/release.md`, add the footer preparation instruction with this pattern
|
||||
(substitute the selected release version for `<version>`):
|
||||
Confirm local/service roots and testing selections work without provider
|
||||
dispatch. Check safe views exclude resolved secrets and preserve the
|
||||
established explanation fields. Do not call a model to validate examples.
|
||||
4. Check the selected dependency graph, attribution, affected links, and final
|
||||
diff. Run `git diff --check`. Confirm only the released modules are used and
|
||||
no obsolete `/v2` catalog or removed construction API remains in active code.
|
||||
Historical release documents and historical roadmap links may retain old
|
||||
versions as accurate records.
|
||||
5. Mark this plan and the feature roadmap complete only after the full target
|
||||
passes. Record the tested implementation revision and checks accurately;
|
||||
distinguish a subsequent documentation-only completion commit. Commit the
|
||||
completion documentation and report remaining work, if any. Do not select
|
||||
a release number, publish, tag, push, poll CI, or add permanent release gates.
|
||||
|
||||
```markdown
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/docs/oss/attribution.md)
|
||||
```
|
||||
**Completion marker:** Stage 9 complete. The tested implementation revision is
|
||||
`5525fe8`; the preceding documentation migration commit is `9bcf183`, and this
|
||||
roadmap closure is documentation-only. At `9bcf183`, the application built,
|
||||
`go test ./...` passed, all six maintained example commands passed offline,
|
||||
`go mod verify` passed, the selected module graph contained PromptKit v0.12.0
|
||||
and both `/v3` catalogs at `v3.20260930.0` with no `/v2` catalogs, and
|
||||
`git diff --check` passed. The active Go/module scan found no removed PromptKit
|
||||
construction options.
|
||||
|
||||
Explain that the plugin uses the checked-in note for new hosted releases and
|
||||
preserves existing release information on retries. Keep old notes and pages
|
||||
intact. Validate intended paths locally during preparation; do not add remote
|
||||
footer checks, a footer parser, or a new release-script gate.
|
||||
3. Align the runbook with the adopted policy: retain `origin`, `main`, atomic
|
||||
source/tag publication, one offline validation pass, required Linux builds,
|
||||
optional check mode, and asynchronous immutable delivery retries. Reference
|
||||
the policy for requirements rather than introducing additional ones. Add
|
||||
proportionate completed-roadmap retirement guidance for future release
|
||||
preparation without making it a script-enforced condition. No release script
|
||||
change is expected from this adoption.
|
||||
4. Repair both current-plan links in `decision-set-reconciliation.md` to the
|
||||
historical plan at the full Audita planning baseline listed above. Remove
|
||||
the stale sentence in `library-adoption.md` saying the current implementation
|
||||
plan covers span deletion. Preserve historical records and deferred work.
|
||||
5. Review the final adoption diff against upstream and the feature roadmap.
|
||||
Check all policy source metadata, template removal, local exception reasons,
|
||||
copied-material scope, license text fidelity, local links/anchors, and pinned
|
||||
upstream destinations. Ensure the feature roadmap records implemented status
|
||||
only after its accepted outcomes are achieved.
|
||||
6. Parse/review the affected YAML using an available YAML parser or Woodpecker
|
||||
linter. Substitute a representative existing release tag locally and confirm
|
||||
the note path resolves to the tracked note; old notes need not have the new
|
||||
footer. Check a temporary example of the new footer against candidate paths.
|
||||
Verify non-overwriting publication settings remain unchanged. Do not call the
|
||||
forge to create a test release, run a plugin with production credentials, or
|
||||
wait for CI. Static configuration and documented plugin-interface validation
|
||||
are sufficient for this wiring change; report that hosted execution awaits
|
||||
the next real release rather than claiming it was exercised.
|
||||
|
||||
**Validation and completion:** Run `git diff --check`, review staged content and
|
||||
all affected links, and record the final verification evidence. No full Go suite
|
||||
or build matrix is required when only documents and the note setting changed.
|
||||
If an actual release-script behavior change becomes necessary, justify its
|
||||
scope and run focused `go test ./scripts` coverage for that behavior rather than
|
||||
inventing additional release gates. Commit the completed work; leave a clean
|
||||
workspace without publishing.
|
||||
|
||||
## Open Questions
|
||||
|
||||
None requiring a policy or design decision before implementation. The upstream
|
||||
revision, local exceptions, attribution arrangement, target documents, and
|
||||
release-body mechanism are selected. Exact component notice details and source
|
||||
revisions are factual verification work in Stage 3, not permission gates. If
|
||||
that inspection reveals an uncovered obligation, follow the adopted exception
|
||||
procedure and identify the concrete decision instead of silently expanding scope.
|
||||
**End state:** Native PromptKit configuration and catalogs are adopted,
|
||||
operator overrides and captured resources work through the real runtime,
|
||||
application backend/domain ownership and resilience remain effective, and
|
||||
maintained examples and migration documentation match the implementation.
|
||||
No implementation-blocking decisions remain; exact major-release numbering
|
||||
and publication timing remain outside this implementation unit.
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
# Coordinated ConfigKit and PromptKit Adoption
|
||||
|
||||
**Last updated:** 2026-09-28
|
||||
|
||||
**Status:** Production implementation complete. ConfigKit v0.4.2 and PromptKit
|
||||
v0.11.2 are adopted; [historical implementation evidence](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) records the
|
||||
completed checks. Release compliance and publication remain separate work.
|
||||
|
||||
## Goal and Ownership
|
||||
|
||||
Adopt ConfigKit and PromptKit to align Audita with the application suite's shared
|
||||
configuration and LLM execution conventions. Consistency is the primary benefit;
|
||||
removing overlapping application infrastructure is a secondary benefit. Preserve
|
||||
Audita's transcript-correction capabilities and domain protections.
|
||||
|
||||
| Owner | Responsibilities |
|
||||
| --- | --- |
|
||||
| ConfigKit | Strict application configuration loading, presence/default semantics, explicit overrides, recorded resolution, secret references, and safe inspection |
|
||||
| PromptKit | Prompt/profile formats and rendering, provider calls, structural output validation and repair, transient retries, engine-local backend admission, and execution observations |
|
||||
| Audita | Setting meanings, file/CLI selection, role/profile association, normalization/chunking, ordered modules, domain validation, correction application, reports, diagnostic redaction, and artifact lifecycle |
|
||||
|
||||
The [configuration roadmap](configkit.md) owns configuration and inspection policy.
|
||||
The [boundary findings](library-boundary-findings.md) own the selected defaults,
|
||||
full migration inventory, failure mapping, and prototype evidence. The selected
|
||||
production versions are ConfigKit v0.4.2 and PromptKit v0.11.2, with Go 1.25.5 as
|
||||
the minimum. Use published modules, never sibling checkouts or local replacements.
|
||||
The existing YAML dependency remains while glossary parsing needs it.
|
||||
|
||||
## Migration Posture
|
||||
|
||||
This is an intentional breaking transition targeting Audita v2.0.0. No backward
|
||||
compatibility is required for existing installations, config files, CLI flags,
|
||||
environment overlays, or inspection output. Provide one migration guide, without
|
||||
legacy loaders, aliases, dual modes, or automatic conversion tooling.
|
||||
|
||||
Both libraries belong to one refactor. Intermediate commits need not build or run;
|
||||
there is no supported intermediate product state. No temporary adapters, compatibility
|
||||
shims, or processing gates are needed. The complete application must work and pass
|
||||
its checks before the refactor is finished. Release compliance and publication
|
||||
come last, with no intermediate public release.
|
||||
|
||||
## Configuration and Execution Target
|
||||
|
||||
ConfigKit selected overlays enable a testing configuration that chooses cheaper or
|
||||
faster models. The example selects both proposal and validation profiles. ConfigKit
|
||||
application overlays and PromptKit execution profiles remain separate concepts.
|
||||
Complete operator-maintained profiles own model/generation controls; embedded
|
||||
application prompts and schemas are not operator-selected files. No catalog wiring,
|
||||
profile-inheritance feature, ConfigKit imports, or secret-directory bootstrap is
|
||||
part of this migration.
|
||||
|
||||
Capacity controls serve two needs: respecting a local service's hard simultaneous-
|
||||
query limit and bounding concurrent paid API activity. One finite limit per backend
|
||||
is shared across both roles and every initial, retry, and repair attempt. Limits
|
||||
are engine-local, not coordinated across processes. No additional global or
|
||||
per-role limiter or fine-grained scheduling policy is required. Profiles must use
|
||||
registered backends and ConfigKit-resolved credentials; endpoint-only bypass and
|
||||
competing profile credential lookup are excluded.
|
||||
|
||||
Structural repair is enabled by default for both roles because additional attempts
|
||||
to obtain valid output are desired. Transient retries and structural repairs have
|
||||
separate budgets; explicit zero disables the respective behavior. Each generation,
|
||||
including a repair, receives its own transient-retry budget. The Audita logical-call
|
||||
deadline covers preparation, waiting, backoff, all generations, and repairs; profile
|
||||
HTTP timeouts apply per attempt. Exact defaults and bounds are in the
|
||||
[selected contract](library-boundary-findings.md#configuration-and-profile-contract).
|
||||
No Audita outer retry loop restores the old broader retry policy.
|
||||
|
||||
Audita retains all four correction modules, five LLM-validator prompts,
|
||||
deterministic validators, confidence/applicability checks, glossary protection,
|
||||
validation batching/cardinality, deterministic proposal and ledger ordering,
|
||||
transcript encoders, and artifact retention. PromptKit-valid JSON still passes
|
||||
Audita's domain checks. Repair exhaustion, provider failure, and domain rejection
|
||||
follow the [selected failure table](library-boundary-findings.md#execution-and-failure-mapping).
|
||||
Retention and transcript timestamp changes remain separate work; this migration
|
||||
resolves the [retry-count follow-up](runtime-behavior.md).
|
||||
|
||||
## Reporting and Diagnostic Target
|
||||
|
||||
Keep stable Audita-owned serialized records; do not expose raw library structs.
|
||||
Use process report schema `v2` with the existing domain outcomes and a new
|
||||
`llm_summary` separating logical calls, dispatched provider attempts, transient
|
||||
retries, structural repairs, and usage. Transcript schema versions remain unchanged.
|
||||
The implementation plan specifies the concrete new fields and artifact wiring.
|
||||
|
||||
Usage distinguishes known zero, unknown, and incomplete totals. Preserve observations
|
||||
from failed calls even when no final result exists. Count terminal totals once;
|
||||
attempt observations provide diagnosis without being added to those totals again.
|
||||
This is operational evidence, not a complete billing ledger.
|
||||
|
||||
Replace global/role scheduler utilization with per-backend configured capacity and
|
||||
observed queue/generation/retry/logical-call timing. Do not fabricate measurements
|
||||
that the event API does not provide. Keep domain timing and deterministic correction
|
||||
ledger ordering. Record prompt definition/rendered hashes and selected profile/model
|
||||
without promising the old hash values. Callbacks are short, concurrency-safe, and
|
||||
free of blocking I/O or waits on their own execution.
|
||||
|
||||
Effective configuration and selected-profile snapshots remain distinct safe views.
|
||||
Secret values must not escape through inspection, logs, reports, runtime errors,
|
||||
or retained diagnostic files. Observation records exclude payloads, URLs, raw
|
||||
provider messages, prompts, and credentials. Existing artifact lifecycle and
|
||||
partial-failure handling remain Audita responsibilities.
|
||||
|
||||
## Completion Criteria and Release Boundary
|
||||
|
||||
- Both libraries own their selected responsibilities in the production application.
|
||||
Superseded loaders, transport, renderers, schedulers, aliases, and the isolated
|
||||
prototype are gone once its useful evidence is represented in production tests.
|
||||
- Offline tests prove configuration-to-execution routing for both roles, selected
|
||||
overlays and explicit overrides, shared backend capacity, retry/repair outcomes,
|
||||
cancellation/deadlines, domain decisions, redaction, and truthful partial reports.
|
||||
Tests exercise real internal collaborators and controlled HTTP boundaries.
|
||||
- Maintained examples and canonical docs describe the implemented final model.
|
||||
One migration guide covers files, flags, environment inputs, profiles, credentials,
|
||||
overlays, inspection, and changed runtime/report/diagnostic semantics.
|
||||
- The assembled application passes its required checks; Linux amd64 and arm64 build.
|
||||
macOS arm64 remains best effort. No live-provider or CI-success gate is added.
|
||||
|
||||
PromptKit v0.11.0's reviewed retry, observation, and capacity APIs remove the earlier
|
||||
upstream feature prerequisites; no upstream edit is required for adoption. The
|
||||
prototype proved representative behavior, not release compliance or model quality.
|
||||
|
||||
The selected PromptKit release uses LGPL-3.0-or-later. A later release unit must
|
||||
review the actual production dependency graph and distribution requirements,
|
||||
including notices and source/rebuild materials, before binary publication. This
|
||||
implementation plan does not claim that review is complete. Release publication
|
||||
uses the existing [procedure](../release.md) and completes on successful source/tag
|
||||
push, without waiting for CI or artifact delivery.
|
||||
@@ -1,392 +0,0 @@
|
||||
# Shared Library Boundary: Decisions and Evidence
|
||||
|
||||
**Status:** Boundary and offline prototype completed, 2026-09-23. Selected prototype contract and proposed
|
||||
production migration target; not the current Audita configuration contract.
|
||||
The [bounded roadmap](library-boundary.md) owns scope and completion criteria.
|
||||
|
||||
**Subsequent scope decision:** The [ConfigKit overlay plan](configkit.md#selected-overlays-for-testing)
|
||||
adds optional selected overlays, with v0.4.0 as the migration baseline. It supersedes
|
||||
the original no-overlay decision and adds an input layer between the main file and
|
||||
explicit setting overrides. The v0.3.1 prototype results below remain historical
|
||||
evidence; they do not establish overlay integration.
|
||||
|
||||
## Selected Boundary
|
||||
|
||||
The former prototype is retained in Git history at `590604b`; it was a separate
|
||||
Go module using published ConfigKit v0.4.0, PromptKit v0.11.0, and
|
||||
Audita v1.1.0. The Audita dependency lets this consumer reuse actual internal domain
|
||||
helpers through its Audita module namespace without a local replacement. It does
|
||||
not make those internal packages a new public API. Remove this dependency along
|
||||
with the prototype after equivalent production integration tests landed.
|
||||
|
||||
The initial prototype results below were recorded with ConfigKit v0.3.1. On
|
||||
2026-09-24, upgrading only its ConfigKit pin and sums to v0.4.0 required no source
|
||||
changes. With Go 1.26.8 on linux/amd64, the nested offline `go test ./...` passed
|
||||
and `go run .` still reported one accepted correction, four attempts, zero
|
||||
transient retries, and two structural repairs. This verifies the dependency
|
||||
upgrade, not selected-overlay behavior.
|
||||
|
||||
ConfigKit owns loading, presence, explicit overrides, secret resolution, and
|
||||
redacted effective settings. PromptKit owns profile parsing, rendering, provider
|
||||
execution, transient retries, structural repair, and backend admission. Audita
|
||||
owns file selection, setting meanings, role selection, domain acceptance, and
|
||||
mapping execution facts into reports. Production CLI and runtime remain unchanged.
|
||||
|
||||
## Configuration and Profile Contract
|
||||
|
||||
Selected for the prototype and recommended for production:
|
||||
|
||||
- Main YAML requires integer `version: 2`; absent, null, unknown fields, and older
|
||||
versions fail. No legacy parser, aliases, environment overlays, or automatic
|
||||
conversion. Defaults precede the selected file; explicitly supplied CLI settings
|
||||
become ConfigKit overrides. Validation runs on the resolved result, so an override
|
||||
may repair an invalid lower-precedence setting when decoding succeeds.
|
||||
- Production file selection: explicit `--config`, then
|
||||
`/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`. Select one file;
|
||||
do not merge them. Missing system files are skipped; explicit missing or empty
|
||||
paths and inaccessible selected files fail. No selected file fails with a request
|
||||
to configure backend/profile selection: no hidden provider or model default.
|
||||
Remove `AUDITA_CONFIG`. The prototype loads its explicit fixture only; discovery
|
||||
and command parsing belong to the ConfigKit implementation stage.
|
||||
- `profiles_dir` defaults to `profiles`, resolved relative to the selected main
|
||||
configuration file, with absolute paths accepted. PromptKit reads that directory.
|
||||
Prompts and schemas are application-embedded, not operator-selected prompt files.
|
||||
The prototype uses complete application-owned profile files. Catalog profiles and
|
||||
profile inheritance are not needed for this migration; do not add catalog wiring.
|
||||
- `backends.<id>.endpoint` is required. Use one application backend identity per
|
||||
service capacity pool. Do not split proposal and validation into different backend
|
||||
IDs when they use the same service. Limits are engine-local, not shared across
|
||||
multiple Audita processes or other applications.
|
||||
- `backends.<id>.concurrency` defaults to **1**, must be positive, and is shared by
|
||||
both roles and all initial/retry/repair attempts. Operators can select **4** for
|
||||
the described local backend or a suitable finite paid API limit. There is no
|
||||
additional global or per-role limiter. `queue_capacity` defaults to **1024**,
|
||||
following PromptKit; non-negative, with zero meaning no waiting admission.
|
||||
This is an operational escape hatch rather than a second scheduling hierarchy.
|
||||
- `backends.<id>.credential` is an optional ConfigKit secret reference: a scalar
|
||||
environment variable name. Omission or null means no credential. A supplied
|
||||
reference must resolve to a nonempty value, including during validation and
|
||||
inspection. There is no direct key field or CLI key flag. ConfigKit resolves it
|
||||
once and Audita supplies `RunRequest.APIKey`. Profiles must not introduce another
|
||||
`api_key_env` lookup. Backend credentials also support unauthenticated local use.
|
||||
- Profiles own model, temperature, output token limit, top-p, per-generation timeout,
|
||||
service tier, reasoning effort, and extra provider parameters using PromptKit's
|
||||
format. They select the application `backend`; their effective endpoint must
|
||||
equal that backend's endpoint. Endpoint-only execution is rejected because it
|
||||
bypasses the requested finite capacity boundary. A named registered backend is
|
||||
mandatory even if both roles use the same profile.
|
||||
- `llm.proposal_profile` is required. Omitted `llm.validation_profile` inherits the
|
||||
final proposal profile as a whole, with ConfigKit recording the derivation.
|
||||
Explicit empty or null values fail; no individual model/endpoint/key inheritance.
|
||||
- `llm.transient_retries` defaults to **2** additional attempts per generation;
|
||||
explicit zero disables retries; negative is invalid. PromptKit owns which failures
|
||||
are eligible. Do not add an Audita retry loop.
|
||||
- `llm.repair_attempts` defaults to **2** additional generations for both roles;
|
||||
explicit zero disables repair when deliberately requested. PromptKit supports
|
||||
0–3. Positive defaults are exercised in the prototype. Audita owns this common
|
||||
setting and supplies a complete request output contract with the role's embedded
|
||||
schema, avoiding separate user-facing repair defaults in each prompt.
|
||||
- `llm.operation_timeout` defaults to **15m**, must be positive, and uses a Go
|
||||
duration string. It starts before each logical proposal/assessment call and covers
|
||||
preparation, capacity waiting, transient backoff, generation, and repairs. It is
|
||||
separate from whole-process cancellation. Profile `timeout_seconds` defaults to
|
||||
PromptKit's **600** seconds and is applied afresh to each built-in HTTP attempt.
|
||||
It does not bound the combined retry/repair sequence. A stricter caller/transport timeout can
|
||||
also end execution. The two defaults are selected starting points, not throughput
|
||||
measurements. With the default T=2 and R=2, each logical call dispatches at most
|
||||
**9** attempts before deadline/cancellation; proposal plus assessment may use 18.
|
||||
- Omission retains static defaults or invokes the recorded role/backend derivations.
|
||||
Null is not a general synonym for omission: nonoptional scalars reject it. Optional
|
||||
credentials and the future optional section target accept null. Explicit empty
|
||||
module lists will mean an empty correction pipeline, not “restore defaults.”
|
||||
Repeated modules retain their existing ordered meaning.
|
||||
- The original prototype disabled ConfigKit imports, selected configuration layers
|
||||
(then called configuration profiles), and secret-directory bootstrap. Selected
|
||||
overlays are now verified under the linked overlay decision; the other two
|
||||
features remain disabled. PromptKit execution profiles retain their meaning.
|
||||
|
||||
Configuration validation/inspection must assemble and inspect both selected profiles
|
||||
without provider calls. ConfigKit's effective YAML describes only its loaded state;
|
||||
a separate selected-profile view shows effective model controls/backend selection.
|
||||
Do not imply either snapshot alone represents the full execution configuration.
|
||||
The prototype exercises the two APIs but does not implement production subcommands.
|
||||
Do not serialize raw profile/library objects as a permanent Audita report schema.
|
||||
|
||||
## Migration Inventory
|
||||
|
||||
This inventory covers the current configuration and process flags documented in
|
||||
[configuration](../config.md) and [CLI](../cli.md), including environment-only fields.
|
||||
“Retain” below means retain domain meaning in the new schema, not accept old files.
|
||||
All legacy environment overlays listed afterward are removed.
|
||||
|
||||
| Current file field / setting | Current CLI flag(s) | Target and rationale |
|
||||
| --- | --- | --- |
|
||||
| File selection | `--config` | Retain explicit invocation flag; discovery described above. |
|
||||
| Transcript input | positional path | Invocation only; retain one transcript path. |
|
||||
| Glossary input | `--glossary` | Invocation only; retain required glossary path. |
|
||||
| Corrected output | `--output` | Invocation only; retain stdout fallback. |
|
||||
| Report output | `--report-json` | Invocation only; retain separate report file. |
|
||||
| `pipeline.modules` | `--modules` | Retain sequence/default `[glossary, homophones, glossary, spoken_word, grammar]`; explicit empty clears rather than inherits. |
|
||||
| `output.schema` | `--output-schema` | Retain encoder selection/default `bare-segments`. |
|
||||
| `llm.proposal.base_url`, `llm.validation.base_url` | `--base-url`, `--validation-base-url` | Replace with backend endpoint; remove per-role endpoint flags. |
|
||||
| Both `llm.*.model` | `--model`, `--validation-model` | Replace with profile model; new `--proposal-profile` / `--validation-profile` select whole profiles. |
|
||||
| Both `llm.*.api_key_env`, direct runtime keys | `--llm-api-key`, `--validation-llm-api-key` | Replace with backend ConfigKit secret reference; remove direct key flags. |
|
||||
| Both `llm.*.timeout` | `--llm-timeout-seconds`, `--validation-llm-timeout-seconds` | Replace with profile `timeout_seconds`, plus `llm.operation_timeout`; no duplicate timeout flags. |
|
||||
| Both `llm.*.max_retries` | `--max-retries`, `--validation-max-retries` | Replace with common `llm.transient_retries` and distinct `llm.repair_attempts`; remove misleading old flags. |
|
||||
| `concurrency.total_llm` | `--total-llm-concurrency`, `--llm-concurrency` | Replace with per-backend concurrency; remove global/alias flags. |
|
||||
| `concurrency.proposal_llm`, `concurrency.validation_llm` | `--proposal-llm-concurrency`, `--validation-llm-concurrency` | Remove role budgets; shared backend handles service capacity. |
|
||||
| Validation prompt budget (no old YAML field) | `--validation-max-prompt-tokens` | Add `validation.max_prompt_tokens`, default 2048, positive; retain explicit flag override. Audita owns batching. |
|
||||
| `chunking.target_sections` | `--target-sections` | Retain optional positive target; omission/null means automatic, zero invalid. |
|
||||
| `chunking.max_section_tokens` | `--max-section-tokens` | Retain 8192, positive. |
|
||||
| `chunking.min_section_tokens` | `--min-section-tokens` | Retain 2048, positive, no greater than maximum. |
|
||||
| `normalization.max_segment_gap` | `--normalize-max-segment-gap` | Retain 4.0 numeric seconds, non-negative; drop alternate YAML duration syntax for these domain numeric fields. |
|
||||
| `normalization.ellipsis_gap` | `--normalize-ellipsis-gap` | Retain 3.5 numeric seconds, non-negative. |
|
||||
| `normalization.max_segment_duration` | `--normalize-max-segment-duration` | Retain 60.0 numeric seconds, positive. |
|
||||
| `normalization.max_segment_tokens` | `--normalize-max-segment-tokens` | Retain 2048, positive. |
|
||||
| `thresholds.glossary` | `--glossary-confidence-threshold` | Retain 0.8 in [0,1]. |
|
||||
| `thresholds.homophones` | `--homophones-confidence-threshold` | Retain 0.8 in [0,1]. |
|
||||
| `thresholds.spoken_word` | `--spoken-word-confidence-threshold` | Retain 0.8 in [0,1]. |
|
||||
| `thresholds.grammar` | `--grammar-confidence-threshold` | Retain 0.8 in [0,1]. |
|
||||
| `context.description` | `--transcript-description` | Retain trimming and 500-byte limit; empty clears context. |
|
||||
| `diagnostics.work_dir` | `--work-dir` | Retain `/tmp/audita`, nonempty; artifact lifecycle remains Audita-owned. |
|
||||
| `diagnostics.retention` | `--work-dir-retention` | Retain `auto` default and selection; existing `never` follow-up is separate. |
|
||||
| Help/version/config commands | `--help`, `-h` | Retain command purpose; help should parse before loading configuration. Effective-config output intentionally becomes ConfigKit YAML, not legacy JSON. |
|
||||
|
||||
Retained setting flags become explicit ConfigKit overrides only when supplied;
|
||||
CLI parser defaults must not overwrite file values. Normalization flag values stay
|
||||
numeric seconds. Domain values convert into existing domain structs after validation;
|
||||
do not spread ConfigKit or PromptKit types into correction algorithms.
|
||||
|
||||
Complete environment removal map (names remain permissible if explicitly named as
|
||||
secret references; none have implicit configuration meaning):
|
||||
|
||||
| Old environment names | Replacement |
|
||||
| --- | --- |
|
||||
| `AUDITA_CONFIG` | File selection above. |
|
||||
| `AUDITA_MODULES` | `pipeline.modules` / explicit flag. |
|
||||
| `AUDITA_LLM_API_KEY`, `OPENROUTER_API_KEY`, `AUDITA_VALIDATION_LLM_API_KEY` | Explicit backend credential references; no implicit fallback. |
|
||||
| `AUDITA_MODEL`, `AUDITA_VALIDATION_MODEL` | Profile files and role selection. |
|
||||
| `AUDITA_BASE_URL`, `AUDITA_VALIDATION_BASE_URL` | Backend endpoints. |
|
||||
| `AUDITA_LLM_TIMEOUT_SECONDS`, `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | Profile timeout and operation deadline. |
|
||||
| `AUDITA_MAX_RETRIES`, `AUDITA_VALIDATION_MAX_RETRIES` | Common transient/repair settings. |
|
||||
| `AUDITA_TOTAL_LLM_CONCURRENCY`, `AUDITA_LLM_CONCURRENCY`, `AUDITA_PROPOSAL_LLM_CONCURRENCY`, `AUDITA_VALIDATION_LLM_CONCURRENCY` | Backend concurrency; remove alias/role budgets. |
|
||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `validation.max_prompt_tokens` / explicit flag. |
|
||||
| `AUDITA_TARGET_SECTIONS`, `AUDITA_MAX_SECTION_TOKENS`, `AUDITA_MIN_SECTION_TOKENS` | Corresponding chunking fields / flags. |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`, `AUDITA_NORMALIZE_ELLIPSIS_GAP`, `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`, `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | Corresponding normalization fields / flags. |
|
||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`, `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`, `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`, `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | Corresponding threshold fields / flags. |
|
||||
| `AUDITA_WORK_DIR`, `AUDITA_WORK_DIR_RETENTION` | Diagnostics fields / flags. |
|
||||
|
||||
## Execution and Failure Mapping
|
||||
|
||||
The executable uses grammar proposal plus grammar review, current response schemas,
|
||||
current hardening text, transcript-section/description helpers, proposal preview, and
|
||||
actual decision-cardinality enforcement from Audita v1.1.0. PromptKit renders the
|
||||
adapted templates; the custom HTTP transport supplies OpenAI-compatible responses.
|
||||
The prototype pre-expands Audita's `hardening` template function into the embedded
|
||||
PromptKit definitions; it passes existing named string variables unchanged. No
|
||||
provider, paid call, or injected high-level LLM client participates.
|
||||
|
||||
Generated JSON is structurally validated/repaired by PromptKit before Audita decodes
|
||||
it. Audita then checks proposal applicability, confidence, and decision identity and
|
||||
cardinality. The demonstration previews corrections without mutation. Production
|
||||
retains every module's full validator chain, glossary protection, batching, and
|
||||
apply-time checks; the representative prototype does not claim to replace them.
|
||||
|
||||
| Outcome | Section/batch effect | Overall run |
|
||||
| --- | --- | --- |
|
||||
| Structural repair succeeds | Continue domain validation. | Normal execution. |
|
||||
| Proposal repair exhausted | No proposals from this section; malformed-output warning. | Continue other work; successful completion may include warnings. |
|
||||
| Assessment repair exhausted | Reject this batch; malformed-output warning. | Continue other work. |
|
||||
| Schema-valid invalid/missing/duplicate decision indices | Reject batch and warn; no semantic retry/repair. | Continue other work. |
|
||||
| Inapplicable/low-confidence proposal or negative assessment | Reject affected proposal; preserve domain reasons. | Continue other work. |
|
||||
| Provider failure, retry exhaustion, transport or malformed HTTP response | Stop affected operation; retain available observations. | Runtime failure; cancel remaining work, retain existing Audita diagnostic handling. |
|
||||
| Deadline, caller cancellation, capacity admission failure | Stop affected operation; no outer retry or fabricated output. | Runtime failure with distinct classified cause. |
|
||||
| Invalid settings/profile/schema setup | Fail before processing where detectable. | Configuration error for user input, runtime error for broken embedded assets. |
|
||||
|
||||
Zero transient retries produces one ordinary provider failure, not
|
||||
`ErrRetriesExhausted`; the latter signals an exhausted **positive** budget.
|
||||
Provider failures retain a typed cause. Do not expose raw provider messages through
|
||||
user diagnostics: continue Audita's separate secret-redaction boundary. A failed
|
||||
logical call may return no result while its observations still record completed
|
||||
attempts or an earlier successful generation. Already completed domain results can
|
||||
remain in diagnostics; failure must not be presented as a fully corrected output.
|
||||
|
||||
## Observation and Report Handoff
|
||||
|
||||
The prototype maps library events into Audita-owned records under a short mutex.
|
||||
Records contain logical run/role/backend/profile identity, generation and attempt
|
||||
indices, dispatch state, queue/generation/retry/wall timing, classified outcomes,
|
||||
HTTP status, exhaustion, repair/retry counts, and each usage field's value,
|
||||
knownness, and completeness. It omits URLs, credentials, payloads, raw provider
|
||||
errors, and prompt text. Identifiers themselves are metadata, not a secret channel.
|
||||
|
||||
Use terminal records once for logical totals; retain attempt records for diagnosis
|
||||
without adding them to terminal totals. A known zero is distinct from unknown;
|
||||
partial totals remain incomplete after unmetered failed attempts. These observations
|
||||
are not a billing ledger. Callback code must not wait on its own run's progress.
|
||||
|
||||
Production report changes must distinguish logical calls from dispatched provider
|
||||
attempts. Replace scheduler utilization fields based on global/proposal/validation
|
||||
pools with per-backend admission/active timing. Preserve deterministic ledger and
|
||||
validator ordering. Map prompt definition/rendered hashes and selected profile/model
|
||||
into request metadata, but do not promise equality with legacy prompt hashes or
|
||||
serialize PromptKit's raw event struct as the stable report contract. Update the
|
||||
output integration reference, report mapper tests, and downstream migration guide
|
||||
when the production representation is implemented.
|
||||
|
||||
## Verification and Remaining Handoff
|
||||
|
||||
The focused matrix is implemented in the nested module:
|
||||
|
||||
| Roadmap scenario | Evidence |
|
||||
| --- | --- |
|
||||
| Successful assembled path | `go run .`; `TestWorkflowBoundaries/repair_both_roles`; actual Audita domain helpers and both selected profiles. |
|
||||
| Presence, overrides, inheritance | `TestConfigurationContract`: omitted role/backend values, explicit zero, null/empty, version, unknown keys, and explicit overrides. |
|
||||
| Credentials | Missing/unset and empty references fail; optional null accepted; effective/profile inspection redacted; HTTP authorization verified; no secret/prompt text in observations. `TestProfilesRespectSharedBackendBoundary` rejects competing lookup and capacity bypass. |
|
||||
| Both roles repaired | `repair_both_roles`: four attempts, two repairs, one accepted correction. |
|
||||
| Repair exhausted | Proposal and assessment exhaustion subtests: bounded counts and selected warning/rejection outcomes. |
|
||||
| Semantic boundary | Missing span, missing/duplicate decision indices, negative assessment; no semantic repair loop. |
|
||||
| Retry/zero/exhaustion | 429 recovery, zero-budget 503, positive-budget exhaustion after completed proposal; typed error assertions. |
|
||||
| Combined budgets | 503 → invalid JSON → repair 503 → valid proposal → assessment: five attempts, two transient retries, one repair. |
|
||||
| Shared capacity | `TestSharedCapacityAndRecovery`: caps four and two, both roles, a retry and repair, queue rejection, measured peak equal to cap, later successful call. |
|
||||
| Waiting/cancellation/deadline | `TestQueuedCancellationAndDeadline`; `TestDeadlineStopsRetryAfter`: no queued dispatch after cancellation/deadline and a 60-second provider retry delay interrupted by the operation deadline. |
|
||||
| Failed-run observations | `TestPartialUsageOnFailedRun`: valid HTTP output with known-zero usage followed by repair failure; nil result, retained events, incomplete known aggregate, correct counts. |
|
||||
| Representative assets | `TestRealPromptRenderingParity` compares both rendered message pairs to Audita v1.1.0; `TestSchemaFixturesMatchAudita` compares schema bytes. |
|
||||
|
||||
Commands executed from `examples/library-adoption` on Linux amd64 with Go 1.26.8:
|
||||
|
||||
```sh
|
||||
GOWORK=off go mod download all
|
||||
GOWORK=off GOPROXY=off GOSUMDB=off go mod verify
|
||||
GOWORK=off GOPROXY=off GOSUMDB=off go run .
|
||||
GOWORK=off GOPROXY=off GOSUMDB=off go test -race ./...
|
||||
GOWORK=off GOPROXY=off GOSUMDB=off go vet ./...
|
||||
```
|
||||
|
||||
All passed. The demonstration reports accepted=1, rejected=0, attempts=4,
|
||||
transient_retries=0, structural_repairs=2. Module verification reports all modules
|
||||
verified. Published tag availability was checked with `git ls-remote --tags` for
|
||||
ConfigKit v0.3.1 and PromptKit v0.11.0; successful module acquisition verifies this
|
||||
is not merely a local-checkout experiment. The module has no replacements.
|
||||
|
||||
The minimum Go 1.25.5 toolchain was also exercised with offline tests and the same
|
||||
successful demonstration. The exact installed-toolchain invocation was:
|
||||
|
||||
```sh
|
||||
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off /home/eric/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.5.linux-amd64/bin/go test ./...
|
||||
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off /home/eric/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.5.linux-amd64/bin/go run .
|
||||
```
|
||||
|
||||
An initial attempt to select that cached toolchain through `GOTOOLCHAIN=go1.25.5`
|
||||
with checksum verification disabled was rejected by Go's toolchain verifier. Using
|
||||
the installed binary with `GOTOOLCHAIN=local` proved offline execution without
|
||||
weakening dependency checks. The portable README commands assume an installed
|
||||
sufficiently new Go executable on PATH and disable toolchain switching.
|
||||
|
||||
From the Audita root, `go test ./...` and
|
||||
`go build -o ./bin/audita ./cmd/audita` passed. These check existing production code;
|
||||
they do not cover the nested module. Affected relative documentation links,
|
||||
inventory coverage, and `git diff --check` were checked separately.
|
||||
|
||||
### Findings and Limits
|
||||
|
||||
No adoption-blocking upstream API gap was found. Two integration details deserve
|
||||
explicit attention: PromptKit's profile GoDoc calls `timeout_seconds` a generation
|
||||
deadline, but its built-in HTTP path applies it separately to each attempt; the
|
||||
consumer therefore needs the selected total-operation deadline. Also,
|
||||
`APIKeyRequired` is an in-memory profile facility, not a profile-YAML field. The
|
||||
file-profile design uses the presence of ConfigKit's optional credential reference
|
||||
to require local secret resolution; it does not invent a new PromptKit YAML key.
|
||||
|
||||
The chosen defaults are proposed operational policy, not measured model-quality or
|
||||
cost recommendations. Tests use controlled transport responses and synthetic data;
|
||||
they do not establish real provider compatibility, model quality, network behavior,
|
||||
throughput, cross-process coordination, complete billing, or release compliance.
|
||||
Only the representative LLM-facing subset is implemented in the prototype; the
|
||||
inventory specifies the remaining domain fields for the ConfigKit migration.
|
||||
Production file discovery, CLI parsing, full module chains, all nine prompts,
|
||||
filesystem diagnostics/redaction, and serialized report migration remain their
|
||||
assigned production tasks. Those are deliberate scope boundaries, not missing
|
||||
claims of completion for the prototype.
|
||||
|
||||
The new profile/credential/capacity and deadline choices are explicit above; no
|
||||
unresolved upstream requirement prevents starting the ConfigKit implementation
|
||||
plan. Production report field names and final runtime wiring are selected in the
|
||||
[implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md), following the outcome and observation
|
||||
contract here. The coordinated refactor targets both libraries directly; intermediate commits do not require a working legacy runtime.
|
||||
|
||||
### Production Implementation Reference
|
||||
|
||||
The [implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) owns ordered production work,
|
||||
component changes, tests, and acceptance criteria. The [combined roadmap](library-adoption.md)
|
||||
and [configuration roadmap](configkit.md) own the final target. This evidence does
|
||||
not require an interim runtime bridge or separately runnable configuration migration.
|
||||
|
||||
Retained domain scope includes transcript/glossary schemas, normalization/chunking,
|
||||
output encoding, proposals and apply-time checks, deterministic validators, ordered
|
||||
modules, LLM-validator batching/cardinality, ledger ordering, and artifact lifecycle.
|
||||
The production integration replaces loading, provider execution, prompt rendering,
|
||||
and scheduler composition while preserving those responsibilities. Retention and
|
||||
timestamp changes remain separate work.
|
||||
|
||||
### Distribution Follow-Up
|
||||
|
||||
The pinned graph includes ConfigKit, PromptKit, both PromptKit backend catalogs,
|
||||
jsonschema/v6, yaml.v3, yaml.v4, x/text, and kr/text (including module/test dependency
|
||||
edges). Before distribution, review the actual production dependency graph and
|
||||
its license files, including PromptKit/catalog LGPL-3.0-or-later terms. Decide and
|
||||
verify the notices, license copies, corresponding library source/modifications,
|
||||
and rebuild/relink materials needed for Audita's Go binary/container distribution.
|
||||
This unit does not publish binaries or claim that release compliance is complete.
|
||||
No upstream relicensing request or library implementation change is required for
|
||||
this prototype.
|
||||
|
||||
## Selected-Overlay Extension: 2026-09-24
|
||||
|
||||
The isolated prototype now pins published ConfigKit v0.4.0. ConfigKit loads
|
||||
`overlays/testing.yml` relative to the selected main file; the overlay selects
|
||||
complete PromptKit execution profiles from the separate `profiles/` directory.
|
||||
The original v0.3.1 observations above are historical and do not prove this
|
||||
extension. No production Audita behavior changed.
|
||||
|
||||
With Go 1.26.8 on linux/amd64, from `examples/library-adoption/`, these offline
|
||||
checks passed with `GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off`:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go run .
|
||||
go run . --overlay testing
|
||||
go run . --overlay testing --proposal-profile proposal
|
||||
```
|
||||
|
||||
Each demonstration reported one accepted correction, four attempts, zero
|
||||
transient retries, and two structural repairs. The base run selected
|
||||
`synthetic-grammar`/`synthetic-review`; testing selected
|
||||
`synthetic-grammar-testing`/`synthetic-review-testing`; the explicit proposal
|
||||
override restored only `synthetic-grammar`. These are synthetic routing IDs,
|
||||
not price, latency, quality, or provider-compatibility measurements.
|
||||
|
||||
| Requirement | Consumer evidence |
|
||||
| --- | --- |
|
||||
| Base, empty/default, testing selection, role overrides, final inheritance, and relative profile resolution | `TestSelectedOverlayConfiguration` loads temporary on-disk layouts and assembles both roles. |
|
||||
| Actual base/testing/override HTTP models and repairs | `TestDemonstrationSelectsOutboundModels` captures every request body and matches model IDs to the reported selections and domain outcome. |
|
||||
| Missing/invalid selection, version/directive rejection, and ordinary backend-setting validation | `TestOverlayFailuresAndApplicationValidation` checks ConfigKit diagnostic codes and effective backend limits. |
|
||||
| Surviving credential resolution, absent/empty failures, null unauthenticated use, and profile-only switching | `TestOverlayCredentialResolutionAndInspection` controls synthetic environment values and observes HTTP Authorization. |
|
||||
| Redacted effective output, explanation and metadata origin, profile inspection, and observations | `TestOverlayCredentialResolutionAndInspection` and `TestOverlaySelectionProvenance` inspect safe output and overlay/CLI origins. |
|
||||
| Existing retry, repair, domain, capacity, and concurrent-observation behavior | `TestWorkflowBoundaries`, `TestSharedCapacityAndRecovery`, `TestQueuedCancellationAndDeadline`, `TestDeadlineStopsRetryAfter`, and `TestPartialUsageOnFailedRun` pass, including under `-race`. |
|
||||
|
||||
Changing a profile does not remove a backend credential reference from the
|
||||
effective configuration. ConfigKit validates a surviving reference when only
|
||||
profile selection changes. Replacing the reference in an
|
||||
overlay resolves only the replacement; explicit null allows unauthenticated use.
|
||||
This distinction is part of the selected configuration boundary, not an
|
||||
application fallback. Imports and secret-directory bootstrap remain disabled.
|
||||
|
||||
The tests use a controlled built-in HTTP transport and synthetic credentials;
|
||||
there were no live provider calls. The prototype still covers one representative
|
||||
proposal/assessment pair, not all production modules, prompts, CLI paths, or
|
||||
diagnostic artifacts. Root `go test ./...` does not run this nested module. No
|
||||
upstream API gap appeared in this extension. The [production implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md)
|
||||
keeps implementation and release pending.
|
||||
@@ -1,269 +0,0 @@
|
||||
# Library Adoption: Shared Boundary and Prototype
|
||||
|
||||
**Last updated:** 2026-09-24
|
||||
|
||||
**Status:** Original boundary-design and offline-prototype unit completed;
|
||||
overlay extension and production-planning handoff completed, 2026-09-24.
|
||||
The work packages below retain the original scope;
|
||||
[decisions, evidence, limitations, and handoff](library-boundary-findings.md)
|
||||
record the outcome. The executable prototype was removed after production
|
||||
integration; Git history at `590604b` retains it. No publication or upstream
|
||||
changes were made in that historical unit.
|
||||
|
||||
## Next Unit: Testing Overlays and Production Migration Plan
|
||||
|
||||
**Status:** Complete. The [production implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md)
|
||||
owns subsequent work; the [overlay findings](library-boundary-findings.md#selected-overlay-extension-2026-09-24)
|
||||
record this unit's checks. The original work packages below remain
|
||||
historical scope for the completed first unit.
|
||||
|
||||
Extend the existing prototype to prove the [selected-overlay contract](configkit.md#selected-overlays-for-testing)
|
||||
with ConfigKit v0.4.0, then turn the verified findings into a detailed production
|
||||
ConfigKit migration plan. Do not repeat the original inventory or redesign the
|
||||
established library boundary without new evidence.
|
||||
|
||||
Required work:
|
||||
|
||||
- Upgrade only the nested prototype module from ConfigKit v0.3.1 to v0.4.0.
|
||||
- Pass an explicit overlay selection through ConfigKit and add `overlays/testing.yml`
|
||||
plus two PromptKit testing profiles. Select both roles and retain one shared
|
||||
finite backend. Synthetic model names demonstrate routing; they do not benchmark
|
||||
real model cost, speed, or quality.
|
||||
- Make the offline executable accept overlay and explicit role-profile selections,
|
||||
and show the selected models alongside domain outcomes and retry/repair counts.
|
||||
- Verify base/overlay/CLI precedence, role inheritance, invalid selections,
|
||||
surviving credential references, safe inspection, and overlay provenance.
|
||||
Prove the actual HTTP request models for both roles; loading changed strings
|
||||
alone is insufficient evidence.
|
||||
- Record new results separately from the historical v0.3.1 findings. Expand the
|
||||
existing ConfigKit roadmap into a concrete production implementation handoff,
|
||||
including final runtime integration and removal of legacy configuration.
|
||||
|
||||
Target end state: reviewers can run the prototype with and without the testing
|
||||
selection and see both LLM roles switch models. The offline test suite establishes
|
||||
that selection, inspection, and execution agree, and the production ConfigKit work
|
||||
has bounded tasks with explicit acceptance criteria.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- The published v0.4.0 dependency is pinned without local replacements; the original
|
||||
no-overlay path and existing integration tests continue to pass.
|
||||
- `testing` switches both roles, explicit profile flags override its selections,
|
||||
and omission/empty/`default` retains base selection according to ConfigKit.
|
||||
- Invalid overlays fail clearly; inheritance and secret resolution use surviving
|
||||
effective settings. Inspection records overlay origin without disclosing keys.
|
||||
- Both selected testing models reach the controlled HTTP transport; structural
|
||||
repair, shared capacity, observations, and Audita domain checks remain active.
|
||||
- Updated examples, executable instructions, findings, and a detailed production
|
||||
migration plan are reviewable. Exact verification commands and limitations are
|
||||
recorded, including the fact that root tests do not cover the nested module.
|
||||
- Production code, root dependencies, and upstream libraries remain unchanged.
|
||||
No release or live provider call is part of this unit.
|
||||
|
||||
This unit ends with a production migration **plan**, not production implementation.
|
||||
The ConfigKit → PromptKit → combined release sequence remains unchanged. Imports
|
||||
and secret-directory bootstrap remain disabled; PromptKit execution profiles keep
|
||||
their existing names and meaning.
|
||||
|
||||
## Objective
|
||||
|
||||
Define and prove the boundary between Audita, ConfigKit, and PromptKit before
|
||||
replacing Audita's production configuration or LLM runtime. This unit covers the
|
||||
shared design and representative prototype supporting the
|
||||
[coordinated adoption roadmap](library-adoption.md). The
|
||||
[ConfigKit roadmap](configkit.md) continues to own the broader configuration scope.
|
||||
|
||||
The deliverable is a concrete target contract supported by an offline executable
|
||||
prototype, sufficient to implement the ConfigKit migration without subsequently
|
||||
redesigning its LLM-facing settings. A successful prototype is not completion of
|
||||
either production migration.
|
||||
|
||||
## Established Direction
|
||||
|
||||
- This is an intentional breaking migration with a migration guide, not legacy
|
||||
configuration support, aliases, or a permanent compatibility mode.
|
||||
- ConfigKit owns shared configuration mechanics; PromptKit owns shared prompt and
|
||||
generation mechanics; Audita retains domain meaning and transcript correction.
|
||||
- Structural repair is enabled from the outset for proposal and LLM-validator calls.
|
||||
Positive default repair budgets must be selected, not silently left at zero.
|
||||
- Transient retries and structural repair are separate. Explicit zero disables
|
||||
transient retries; Audita must not add a second automatic retry loop.
|
||||
- One finite concurrency limit per backend protects local capacity and bounds paid
|
||||
API activity. Calls from both roles share that limit. Fine-grained role budgets
|
||||
and an additional global concurrency control are unnecessary.
|
||||
- Keep the sequence: boundary/prototype, ConfigKit migration, PromptKit migration,
|
||||
then combined major release. No intermediate public release is planned.
|
||||
|
||||
## Scope and Deliverables
|
||||
|
||||
| Deliverable | Required result |
|
||||
| --- | --- |
|
||||
| Configuration inventory and migration map | Every existing setting and invocation option has a proposed destination, replacement, or removal, with rationale. |
|
||||
| Boundary specification | One authoritative source for each setting, explicit precedence/defaults, credential flow, runtime conversion, and failure handling. |
|
||||
| Representative executable prototype | New-style configuration drives one proposal and one assessment through released libraries and a controlled provider. |
|
||||
| Focused behavioral evidence | Retry, repair, capacity, credential, observation, and domain-boundary scenarios below are demonstrated. |
|
||||
| Decision and integration handoff | Concrete choices, measured limitations, affected components, removal plan, and next implementation tasks are recorded. |
|
||||
|
||||
Do not switch the production CLI to the prototype, migrate every module/prompt,
|
||||
redesign transcript/glossary formats, change correction algorithms, modify retention
|
||||
or timestamp behavior, or publish a release in this unit. Do not reproduce the
|
||||
libraries' full test suites. Upstream changes require a separately scoped task if
|
||||
an actual gap is found.
|
||||
|
||||
## Work Package 1: Inventory and Decide Ownership
|
||||
|
||||
Read the current [configuration](../config.md), [CLI](../cli.md),
|
||||
[LLM runtime](../internal/llm-runtime.md), [prompts](../internal/prompts.md),
|
||||
[validators](../internal/validators.md), and [output contracts](../integrations/output-files.md).
|
||||
Use graph-first discovery to verify the relevant code and tests, especially
|
||||
configuration application, prompt construction, structured responses, scheduling,
|
||||
and report assembly.
|
||||
|
||||
Build a migration table grouped by application/domain settings, invocation state,
|
||||
backend settings, execution profiles, output contracts, and inspection. Include
|
||||
current environment settings and aliases without treating their existence as a
|
||||
requirement to preserve them. Identify report fields whose meanings change when
|
||||
one logical generation contains multiple provider attempts.
|
||||
|
||||
Resolve these design decisions explicitly:
|
||||
|
||||
| Decision | Proposed direction to prove |
|
||||
| --- | --- |
|
||||
| File selection and version | Audita selects the main configuration and validates a new schema version; specify discovery and the treatment of old versions. |
|
||||
| Application versus invocation state | ConfigKit loads persistent settings; Audita parses invocation-only input/output paths and translates explicitly supplied setting flags into supported overrides. |
|
||||
| Backend identity | One registration represents the service capacity shared by proposal and validation calls. Avoid separate identities for the same service merely because roles differ. |
|
||||
| Endpoint and capacity ownership | Define them once per backend, with finite limits and an explicit policy for waiting capacity/exhaustion. Do not expose endpoint-only unlimited execution accidentally. |
|
||||
| Execution profiles | Prefer complete application-owned profiles using PromptKit conventions. Decide their source and selection without duplicating model/settings fields in application configuration. Catalog selection remains a deliberate option. |
|
||||
| Role selection and inheritance | Specify how proposal and validation choose profiles, including omission and any inheritance. Do not inherit old behavior solely for compatibility. |
|
||||
| Credentials | Prefer one ConfigKit-owned secret-resolution path passed through PromptKit's supported request credential field. Define backend/profile credential association and avoid competing environment lookups. Validate this against profile requirements. |
|
||||
| Retry, repair, and deadlines | Assign one owner and explicit defaults for each control. Use positive structural-repair defaults and document combined limits and total-operation scope. |
|
||||
| Optional library features | The original prototype omitted imports, selected configuration layers, and secret-directory bootstrap. Selected overlays are now planned for testing; see the [overlay plan](configkit.md#selected-overlays-for-testing). Imports and secret-directory bootstrap remain out. |
|
||||
| Inspection | Define effective configuration and selected-profile inspection without implying that one library's snapshot contains the other's complete state. |
|
||||
|
||||
Choose concrete field names, example values, defaults, units, and omission/null/zero
|
||||
semantics for the prototype. In particular, decide what validation and inspection
|
||||
do when configured credentials are unavailable. Classify choices as proposed,
|
||||
selected, or unresolved; do not present candidates as implemented contracts.
|
||||
|
||||
## Work Package 2: Specify Execution and Reporting Semantics
|
||||
|
||||
Document the flow from selected configuration through backend/profile assembly,
|
||||
prompt inputs, generation, structural validation/repair, typed decoding, and Audita
|
||||
domain validation. Identify which values cross the boundary and which component
|
||||
owns each conversion. Keep library types out of unrelated domain code where a
|
||||
small existing domain value is sufficient.
|
||||
|
||||
Use the current correction-set and validator-decision schemas as representative
|
||||
inputs. Reuse domain checks where practical. Select one real proposal/validator
|
||||
pair; account for prompt helpers, template variables, shared fragments, and schema
|
||||
references rather than testing only an unrelated hello-world prompt.
|
||||
|
||||
Record the following distinctions:
|
||||
|
||||
- Repair seeks output conforming to its structural contract. Success does not
|
||||
establish semantic correctness or authorize applying a correction.
|
||||
- A schema-valid decision set still requires Audita's cardinality and proposal
|
||||
identity checks. Semantic rejection is not automatically another repair request.
|
||||
- Exhausted structural repair yields failed content validation; provider failures,
|
||||
retry exhaustion, cancellation, and capacity rejection have separate outcomes.
|
||||
- Specify each outcome's intended effect on an Audita section/batch and overall run,
|
||||
including warnings, rejected work, partial domain results, and terminal failures.
|
||||
Preserve domain intent without preserving obsolete transport behavior by accident.
|
||||
- For transient retry budget T and structural repair budget R, one logical call can
|
||||
dispatch up to `(T + 1) * (R + 1)` provider attempts. For example, T=2 and R=2
|
||||
permits nine attempts; these illustrative values do not select production defaults.
|
||||
- Choose the total-operation deadline scope for a logical proposal or assessment;
|
||||
distinguish it from per-attempt timeouts and whole-process cancellation. Provider
|
||||
retry timing may exceed local backoff, so deadline behavior must be explicit.
|
||||
|
||||
Design the Audita observation record and aggregation rules. Include logical-call
|
||||
and attempt identity, role/backend correlation, queue/generation/retry timing,
|
||||
repair and retry counts, outcomes, and available usage. Preserve unknown/incomplete
|
||||
usage rather than inventing zeros or complete costs. Do not add terminal totals to
|
||||
attempt totals. Collect events safely across concurrent calls, without blocking
|
||||
callbacks on progress by the same run or persisting raw library events as a stable
|
||||
Audita schema. Transcript-bearing artifacts remain separately governed by Audita.
|
||||
|
||||
## Work Package 3: Build the Bounded Offline Prototype
|
||||
|
||||
Use ConfigKit v0.3.1 and PromptKit v0.11.0 as reviewed starting points; verify their
|
||||
published availability and APIs before pinning the prototype. Recheck the combined
|
||||
Go minimum (the reviewed PromptKit release requires Go 1.25.5). Do not depend on
|
||||
sibling checkouts, an active Go workspace, or committed local module replacements.
|
||||
|
||||
A proposed home is `examples/library-adoption/` as an explicitly experimental,
|
||||
standalone Go module, isolating prototype dependencies from the production binary.
|
||||
Confirm this placement during execution. Its README must give exact offline run
|
||||
and test commands after dependency acquisition; root `go test ./...` must not be
|
||||
claimed to cover a nested module. Keep complete proposed configuration and prompt
|
||||
fixtures with it and clearly label them as prototype-only.
|
||||
|
||||
The prototype should:
|
||||
|
||||
1. Load representative application settings with ConfigKit, using synthetic secret
|
||||
values supplied by the test environment and an application-owned version rule.
|
||||
2. Assemble shared backend registrations and proposal/validation profile selections.
|
||||
3. Supply representative transcript/proposal inputs to PromptKit definitions and
|
||||
actual response schemas using supported sources.
|
||||
4. Use the built-in HTTP generation path with a controlled local endpoint or
|
||||
transport. An injected LLM client alone cannot prove built-in HTTP retry behavior.
|
||||
5. Run proposal generation, decode and check candidates, run assessment, and perform
|
||||
the selected Audita decision checks without modifying real transcript files.
|
||||
6. Collect selected safe observations and show a compact result understandable to a
|
||||
reviewer, including repaired output and final domain acceptance/rejection.
|
||||
|
||||
No paid calls, real credentials, host environment assumptions, persistent caches,
|
||||
or general-purpose test framework are needed. Make concurrency and cancellation
|
||||
checks deterministic through controlled synchronization. Keep delays bounded and
|
||||
use library-supported controls; do not patch library internals to make tests pass.
|
||||
|
||||
## Focused Verification Matrix
|
||||
|
||||
| Scenario | Evidence required |
|
||||
| --- | --- |
|
||||
| Successful assembled path | ConfigKit-loaded selections drive both real prompt roles through PromptKit and Audita checks. |
|
||||
| Presence, overrides, and inheritance | Representative omission, explicit values, and role selection follow the proposed contract; unknown fields and unsupported versions fail clearly. |
|
||||
| Credentials | Required missing/empty references fail as intended; controlled calls receive the chosen synthetic credential; inspection and observations do not expose it. |
|
||||
| Structural repair succeeds | Invalid proposal output and invalid assessment output are repaired in their respective workflows; positive budgets are actually exercised. |
|
||||
| Repair exhausted | Repeated invalid output produces the selected Audita outcome and accurate accounting, without an unbounded outer retry. |
|
||||
| Domain rejection | Structurally valid but unacceptable correction or decision is rejected by Audita, demonstrating the semantic boundary. |
|
||||
| Transient retry and explicit zero | An eligible temporary status recovers; zero makes one attempt. Exhaustion retains a useful classified cause. |
|
||||
| Combined repair and retry | A repaired generation encountering a transient failure respects both budgets and reports distinct counts. |
|
||||
| Shared capacity | Concurrent calls from both roles, including retry/repair activity, stay within one selected backend cap; test a representative local cap of four and a finite API cap. |
|
||||
| Waiting, cancellation, and deadline | Cancellation/deadline stops pending work and retry waits; capacity exhaustion has the intended application outcome and later calls can still proceed. |
|
||||
| Failed-run observations | Completed work remains observable after later failure; unknown usage, known zero, and aggregate counts are mapped correctly. |
|
||||
|
||||
Tests should protect Audita's integration decisions and important assembled behavior.
|
||||
Rely on PromptKit's tests for exhaustive HTTP-status, backoff, and scheduler details;
|
||||
rely on ConfigKit's tests for its parser and presence machinery. Use race testing for
|
||||
new concurrent observation collection. Keep consumer checks offline and credential-free.
|
||||
|
||||
## Completion Criteria and Handoff
|
||||
|
||||
This unit is complete only when:
|
||||
|
||||
- The settings/ownership table and migration inventory are concrete enough to start
|
||||
the ConfigKit migration, including file/profile sources, CLI scope, credentials,
|
||||
defaults, finite backend limits, positive repair budgets, and deadline policy.
|
||||
- The offline prototype and focused matrix demonstrate the selected boundary with
|
||||
both libraries together, including both LLM roles and Audita domain checks.
|
||||
- Exact verification commands, results, limitations, and unresolved upstream gaps
|
||||
are recorded. Missing evidence is not reported as successful integration.
|
||||
- No adoption-critical decision remains hidden in prototype code. If a gap prevents
|
||||
settling the boundary, record it and revise the plan before declaring readiness.
|
||||
- The existing Audita production behavior remains intact in this unit. Temporary
|
||||
code is clearly isolated and has a removal/integration destination during the
|
||||
production migrations; no second supported runtime is introduced.
|
||||
- A component-level follow-up identifies code to replace, domain code to retain,
|
||||
tests to update, documentation/migration-guide work, and direct integration of
|
||||
the final configuration and PromptKit runtime.
|
||||
- Licensing/distribution follow-up for the selected dependency graph is identified
|
||||
for the later release stage; this prototype does not claim release compliance.
|
||||
- Roadmap links and examples resolve, appropriate checks pass, and commits contain
|
||||
only this unit's work. Significant settled architectural choices may be recorded
|
||||
in an ADR under the [documentation policy](../policy/documentation.md), while
|
||||
implementation status stays in the roadmaps.
|
||||
|
||||
Retain the overall adoption order unless evidence or the other planned large changes
|
||||
justifies revising it. Completing this unit enables detailed production migration;
|
||||
it does not authorize an intermediate release or require unrelated runtime fixes.
|
||||
@@ -1,252 +0,0 @@
|
||||
# Standard Policy Alignment
|
||||
|
||||
**Last updated:** 2026-09-28
|
||||
|
||||
**Status:** Implemented. This roadmap replaces its earlier licensing-only scope
|
||||
and records the accepted end state. [implementation.md](implementation.md)
|
||||
records the ordered work and verification; the adopted policies are in
|
||||
`docs/policy/`.
|
||||
|
||||
## Purpose and Review Revision
|
||||
|
||||
Align Audita with [Standard Repository Policies](https://gitea.maximumdirect.net/eric/policy),
|
||||
using identical policy language except where a substantive, locally significant
|
||||
requirement or decision justifies a variation. Adopt upstream wording verbatim
|
||||
for non-substantive differences. Perform the required template customizations
|
||||
without inventing application contracts, restrictions, or guarantees.
|
||||
|
||||
The accepted review compared Audita's current documents directly against upstream
|
||||
commit **`d84a38ebab26f048cafbe59a2246700c35d29843`**, beginning with the
|
||||
[adoption and update guide at that revision](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/adoption.md).
|
||||
Use that revision for this adoption. Historical adoption baselines need not be
|
||||
reconstructed or assessed. A later upstream revision requires deliberate review,
|
||||
not an automatic substitution during implementation.
|
||||
|
||||
The public upstream repository, discovery links, and copied-policy attribution
|
||||
are part of the accepted direction. This supersedes the earlier instruction to
|
||||
omit references to the policy repository.
|
||||
|
||||
## Current Baseline
|
||||
|
||||
- Audita has five adopted policies: agents 1.0, architecture 1.0, documentation
|
||||
1.1, testing 1.0, and release 1.0. None has per-document `Upstream Source`
|
||||
metadata. `meta.md` and `oss.md` are absent.
|
||||
- [AGENTS.md](../../AGENTS.md) routes readers to the contributor guide and agent
|
||||
policy. [docs/development.md](../development.md) has component-specific reading
|
||||
guidance, but lacks the upstream selective Policy Reading Guide and currently
|
||||
directs all tasks to the architecture policy.
|
||||
- [COPYRIGHT](../../COPYRIGHT) and [LICENSE](../../LICENSE) already establish
|
||||
Audita's GPL-3.0-or-later application grant and full GPLv3 text. The grant does
|
||||
not yet identify the scope of separately licensed copied policy documents.
|
||||
- There is no canonical `docs/oss/attribution.md` or local dependency license-text
|
||||
collection. Relevant component notices still require an evidence-based inventory.
|
||||
- The [README](../../README.md) lacks the prescribed final licensing statements.
|
||||
The [release procedure](../release.md) lacks licensing-footer instructions, and
|
||||
the [release workflow](../../.woodpecker/release.yml) does not explicitly reuse
|
||||
checked-in release notes as hosted release bodies.
|
||||
- Audita already releases through one publication command with an atomic source
|
||||
and tag push. Linux amd64 and arm64 are required; macOS arm64 is best-effort
|
||||
development support. Successful publication completes the release without
|
||||
waiting for CI or artifact delivery.
|
||||
|
||||
## Targeted Policy Set and Justified Customizations
|
||||
|
||||
All seven policy files belong under `docs/policy/`. The versions below are those
|
||||
at the selected upstream revision. No existing policy is wholly unchanged;
|
||||
existing architectural decisions and platform commitments need no substantive
|
||||
revision merely to align the text.
|
||||
|
||||
| Policy | Target version | Required alignment and justified local treatment |
|
||||
| --- | --- | --- |
|
||||
| `agents.md` | 1.3 | Adopt upstream substantive language verbatim. Retain Audita's graph-first Codebase Discovery section because it is an explicitly selected repository workflow. |
|
||||
| `architecture.md` | 1.3 | Adopt general ownership, boundary, and constraint guidance, then perform the template's required application-specific customization. Preserve Audita's ordered pipeline, ConfigKit/PromptKit responsibilities, deterministic processing, recovery, concurrency, and secret-handling decisions. These describe real project requirements rather than stylistic preferences. |
|
||||
| `documentation.md` | 1.11 | Adopt upstream prose and applicable ownership rows. Retain concrete output-contract ownership, contributor change recipes, and the decision to place operating/recovery guidance in operations instead of restoring troubleshooting documentation. Omit HTTP API and public Go API ownership rows because Audita has neither responsibility. |
|
||||
| `testing.md` | 1.3 | Adopt substantive text verbatim. Its distinction between established contracts and invented test requirements needs no local exception. |
|
||||
| `release.md` | 1.4 | Adopt upstream substantive language. Preserve Linux amd64/arm64 requirements, best-effort macOS arm64, unsupported Windows/macOS amd64, and Audita's explicit decision against human release gates. These are selected support and workflow commitments. |
|
||||
| `meta.md` | 1.0 | Adopt provenance, local authority, and public discovery guidance with accurate metadata and local attribution navigation. No additional substantive exception is needed. |
|
||||
| `oss.md` | 1.8 | Adopt the House Format and maintenance rules, with required project-specific facts and reference links. Preserve Audita's GPL-3.0-or-later application grant, Gitea host, and existing distribution channels. |
|
||||
|
||||
Remove each `Base Policy Template` section after carrying out its instructions.
|
||||
Keep local customizations identifiable and explain their project-specific reason
|
||||
in the appropriate policy; do not maintain alternate paraphrases of shared rules.
|
||||
Architecture customization does not authorize a package redesign or additional
|
||||
threat-model guarantees. Supporting documents are warranted by existing
|
||||
responsibilities and substantive content, not by unused template examples.
|
||||
|
||||
Successful-tag-push completion, asynchronous artifacts, minimal automated gates,
|
||||
and proportionate validation are now shared upstream rules. Replace Audita's
|
||||
parallel explanations with the upstream language rather than describing these
|
||||
as continuing exceptions. Keep concrete commands, remotes, build targets, and
|
||||
publication mechanics in the release procedure with appropriate policy links.
|
||||
|
||||
## Provenance and Selective Reading
|
||||
|
||||
For every adopted policy, retain the selected upstream `Base Policy Version`,
|
||||
set `Last Updated` to its actual local adoption date, and introduce `Upstream
|
||||
Source` linking to that exact file at the selected full commit. The URL pattern is:
|
||||
|
||||
```text
|
||||
https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/<filename>.md
|
||||
```
|
||||
|
||||
Substitute each of `agents`, `architecture`, `documentation`, `testing`, `release`,
|
||||
`meta`, and `oss`; a repository-root or shared generic source link is insufficient.
|
||||
These links describe the source actually incorporated, not a reconstructed claim
|
||||
about older local documents. Git and per-document metadata are sufficient; no
|
||||
separate synchronization registry or recurring check is required.
|
||||
|
||||
In `meta.md`, retain these public discovery destinations:
|
||||
|
||||
- [Standard Repository Policies](https://gitea.maximumdirect.net/eric/policy).
|
||||
- [Current adoption and update guide](https://gitea.maximumdirect.net/eric/policy/src/branch/main/docs/adoption.md).
|
||||
|
||||
Discovery links follow the current upstream project and guide. Adopted-source,
|
||||
notice, and example links remain commit-pinned. Preserve valid pinned example
|
||||
references supplied by the selected policy, or pin them to this review revision
|
||||
after checking the referenced assets; do not turn them into moving-branch links.
|
||||
|
||||
Adapt upstream `agents/AGENTS.md` into Audita's root orientation, retaining its
|
||||
existing applicable instructions. Add the adoption guide's Policy Reading Guide
|
||||
to `docs/development.md` and retain the component-specific task guide. Route
|
||||
policy adoption and updates to `meta.md`, repository recommendations and changes
|
||||
to agents, application-code or structure work to architecture, validation choices
|
||||
to testing, documentation work to documentation, dependency/licensing work to
|
||||
OSS, and release/versioning work to release policy. Read applicable policies
|
||||
before their decisions, reuse unchanged context, and expand reading with scope.
|
||||
Do not require reading the entire suite for every task.
|
||||
|
||||
## Copied Policy Licensing and Local Material
|
||||
|
||||
Follow the selected revision's
|
||||
[canonical policy attribution entry](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/policy-attribution.md)
|
||||
and [project grant](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT).
|
||||
Insert the entry beneath the title in `docs/oss/attribution.md`, substitute the
|
||||
full review commit in all three URLs, and accurately identify copied/adapted
|
||||
material, including orientation or reading-guide text where copied. Preserve
|
||||
supplied creator, copyright, license, disclaimer notices, and modification
|
||||
indications. Keep the material statement concise.
|
||||
|
||||
Copy `CC-BY-SA-4.0.txt` into `docs/oss/licenses/`. Add the adoption guide's scoped
|
||||
notice to `COPYRIGHT`, identifying the adapted policies as CC-BY-SA-4.0 and
|
||||
routing readers to attribution for the affected material. Account accurately for
|
||||
other copied guidance in that scope. Audita's application code remains
|
||||
GPL-3.0-or-later; unrelated documentation is not automatically relicensed.
|
||||
The root GPLv3 `LICENSE` needs no change.
|
||||
|
||||
| Material | Destination or treatment |
|
||||
| --- | --- |
|
||||
| Applicable policies, including `meta.md` and `oss.md` | Copy and customize under `docs/policy/`, retaining provenance. |
|
||||
| Upstream `agents/AGENTS.md` | Adapt into root `AGENTS.md`. |
|
||||
| Selective reading guidance | Integrate into Audita's own `docs/development.md`. |
|
||||
| Upstream README, root AGENTS.md, developer guide, and adoption guide | Keep upstream and consult through links; do not copy as local guides. |
|
||||
| Licensing example collection | Keep upstream with commit-pinned references; use it to create actual notices. Original example formatting is CC0 and alone needs no policy attribution entry. |
|
||||
| Applicable standard and companion license texts | Copy complete, unmodified texts locally; no blanket collection. |
|
||||
|
||||
## Component Attribution
|
||||
|
||||
Complete the previously agreed inventory of relevant incorporated or distributed
|
||||
components: direct and transitive dependencies, copied code/documentation/assets,
|
||||
and applicable Go runtime notices. Determine inclusion from actual material and
|
||||
use, not simply presence in `go.mod`; tooling or consultation alone does not
|
||||
require an entry. Inspect selected sources directly rather than copying a
|
||||
dependency's attribution document wholesale.
|
||||
|
||||
Use the exact House Format in OSS policy 1.8: the prescribed title followed by
|
||||
component entries, factual identification, applicable supplied copyrights and
|
||||
SPDX expressions, standard license/notice labels, and `[Source](COMMIT_URL).`
|
||||
links. Omit unavailable optional fields instead of adding explanatory filler.
|
||||
Preserve required upstream material, license alternatives, and exceptions.
|
||||
Keep research narratives, compliance conclusions, and exception reasoning out
|
||||
of attribution; place reasoning in the OSS policy or an appropriate decision
|
||||
record. This replaces the older roadmap's permission to put exception reasoning
|
||||
in a component entry.
|
||||
|
||||
Upstream component and notice links identify exact imported commits. Local
|
||||
standard texts include directly reachable companions, such as GPLv3 for LGPLv3.
|
||||
Apply the adopted house interpretation of notice links without blanket extra
|
||||
local-copy requirements or repeated approval. Concrete obligations beyond notice
|
||||
reproduction receive their own applicable treatment; the format alone does not
|
||||
certify compatibility or resolve every distribution obligation.
|
||||
|
||||
The reviewed dependency revisions remain useful starting evidence, subject to
|
||||
checking the dependencies selected when implementation occurs:
|
||||
|
||||
| Component | Reviewed version | Source commit |
|
||||
| --- | --- | --- |
|
||||
| ConfigKit | v0.4.1 | `0adb94f59e44827908eff401d4381a398fe17107` |
|
||||
| PromptKit | v0.11.1 | `15e40679546f2e21b6f51cc84b6f0a087a8486db` |
|
||||
| OpenRouter catalog | v2.20260926.0 | `4f731d217e0bc39151c010f50fb5912706e7b6b3` |
|
||||
| Rakestrawhome catalog | v2.20260926.0 | `48918a3ad39fd99843cda94a86133b1b3881d596` |
|
||||
|
||||
These library patches changed licensing/documentation rather than library code
|
||||
or catalog profiles. ConfigKit and PromptKit identify LGPL-3.0-or-later; the
|
||||
selected catalogs identify MIT. Other relevant dependencies still need inventory.
|
||||
|
||||
## Discoverability and Release Integration
|
||||
|
||||
End README with the project-license and attribution statements prescribed by the
|
||||
OSS policy, using Audita's actual grant. `COPYRIGHT` owns the separately licensed
|
||||
policy scope; README links to that grant and the attribution document.
|
||||
|
||||
Update `docs/release.md` to specify the final `COPYRIGHT | LICENSE | ATTRIBUTION`
|
||||
links in every new release note and hosted download page. Use absolute Gitea URLs
|
||||
at the release's immutable tag or source commit. Reuse the checked-in note as the
|
||||
hosted body where supported; verify the publication plugin's supported mechanism
|
||||
and make the necessary focused `.woodpecker/release.yml` change. Avoid separately
|
||||
maintained release prose or notice lists.
|
||||
|
||||
Preserve the current atomic source/tag publication approach, supported targets,
|
||||
and completion point. Align procedure wording with upstream's preparation,
|
||||
ambiguous-push recovery, and asynchronous retry rules. Handle completed-roadmap
|
||||
retirement during preparation, preserving planned/deferred work and repairing
|
||||
links; this roadmap update does not authorize deleting other roadmaps now.
|
||||
|
||||
Check intended future-tag paths against the candidate source without requiring
|
||||
the tag to resolve before publication. Apply the formatting convention
|
||||
prospectively; leave published tags, notes, artifacts, and checksums intact.
|
||||
Concrete historical omissions, if discovered, receive deliberate treatment under
|
||||
their applicable terms and release correction rules.
|
||||
|
||||
## Work Required and Acceptance Criteria
|
||||
|
||||
Implementation covers the seven policy documents, `AGENTS.md`,
|
||||
`docs/development.md`, `COPYRIGHT`, `docs/oss/attribution.md`, applicable files in
|
||||
`docs/oss/licenses/`, README, the release procedure, and focused hosted-body
|
||||
publication wiring. Update this roadmap's implementation status when appropriate.
|
||||
The [implementation plan](implementation.md) owns the editing sequence and
|
||||
verification steps.
|
||||
|
||||
The work is complete when:
|
||||
|
||||
- Shared policy language matches the selected upstream revision after required
|
||||
template removal and customization; retained variations have substantive
|
||||
Audita-specific reasons as identified above.
|
||||
- All seven policies have accurate versions, adoption dates, and file-specific
|
||||
commit-pinned source links. Public discovery links and selective orientation
|
||||
are in place, with no leftover template-only instructions or broken links.
|
||||
- Copied policy/guidance scope, modification status, credit, CC-BY-SA text, and
|
||||
root grant notice follow the documented upstream arrangement while preserving
|
||||
Audita's application license.
|
||||
- Relevant components have verified House Format entries, exact source/notice
|
||||
commits, and complete applicable license texts, without editorial commentary.
|
||||
- README and future release documentation provide the required navigation, and
|
||||
hosted release content comes from the checked-in note through a verified
|
||||
publication mechanism.
|
||||
- Audita's application contracts, architectural decisions, platform commitments,
|
||||
release completion semantics, and published history are preserved.
|
||||
- Affected text, links, license copies, metadata, and publication configuration
|
||||
pass proportionate review and validation. No release is needed solely to test
|
||||
adoption; behavioral workflow changes receive focused validation if required.
|
||||
|
||||
## Scope Boundaries and Open Questions
|
||||
|
||||
No policy decisions remain open for the agreed scope. Component-specific notices
|
||||
and the publication plugin's supported release-body mechanism require factual
|
||||
verification during implementation. Raise a decision only for a concrete
|
||||
unresolved obligation or material tradeoff uncovered by that work.
|
||||
|
||||
This feature does not require application refactoring, dependency upgrades,
|
||||
mandatory scanners, a full SBOM, recurring legal approvals, synchronization
|
||||
checks, additional human release gates, publication-time network checks, or CI
|
||||
waiting. The documentation convention alone does not require downloadable
|
||||
license assets, source bundles, or a different binary package format.
|
||||
276
docs/roadmap/promptkit.md
Normal file
276
docs/roadmap/promptkit.md
Normal file
@@ -0,0 +1,276 @@
|
||||
# PromptKit v0.12.0 Adoption Roadmap
|
||||
|
||||
**Last updated:** 2026-10-04
|
||||
|
||||
**Status:** Complete. The implementation plan records the ordered work and
|
||||
verification details.
|
||||
|
||||
## Purpose and Baseline
|
||||
|
||||
Adopt PromptKit v0.12.0 now that the independent
|
||||
[ConfigKit v0.5.0 upgrade](configkit.md) is complete. Align Audita with the
|
||||
libraries' native configuration model while retaining Audita's domain rules,
|
||||
application-owned backend settings, and resilient processing workflow.
|
||||
|
||||
At reviewed Audita revision `8cba751835ba9faad340e933a3cfbb59c03924fd`,
|
||||
[go.mod](../../go.mod) selects ConfigKit v0.5.0 and PromptKit v0.11.2, with
|
||||
OpenRouter and Rakestrawhome `/v2` catalogs at `v2.20260926.0`. Go 1.25.5
|
||||
remains the application minimum. The implemented contracts belong to
|
||||
[configuration](../config.md), the
|
||||
[LLM integration](../integrations/openai-compatible-llm.md), and the
|
||||
[LLM runtime](../internal/llm-runtime.md).
|
||||
|
||||
`internal/framework/llm.NewRuntime` currently assembles one engine using the
|
||||
old prompt/profile filesystem options and programmatic backend registration.
|
||||
Audita embeds seven prompt definitions and separate response schemas. Operators
|
||||
select proposal and validation profiles from `profiles_dir`; the effective
|
||||
backend must be registered in application configuration, its endpoint must
|
||||
match, and its credential comes from Audita's ConfigKit-resolved settings.
|
||||
Successful profile reads are recorded to protect those sources from output
|
||||
collisions. Both roles share the same engine's backend admission controls.
|
||||
|
||||
The reviewed upstream release is
|
||||
[PromptKit v0.12.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/releases/v0.12.0.md),
|
||||
commit `4b8169da286917fde7875425bb32b799b12c6af3`. It depends on ConfigKit
|
||||
v0.5.0 and both maintained `/v3` catalog modules at `v3.20260930.0`. This is an
|
||||
intentional incompatible pre-1.0 minor release: it replaces construction and
|
||||
configuration APIs, rather than simply updating provider data. PromptKit's
|
||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
|
||||
and [consumer migration guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
|
||||
own the upstream details.
|
||||
|
||||
## Target End State
|
||||
|
||||
- Audita imports released PromptKit v0.12.0 and its `/v3` catalogs, retaining
|
||||
ConfigKit v0.5.0 and using no local dependency replacement.
|
||||
- The runtime uses `Config.Sources` and native ConfigKit source groups.
|
||||
PromptKit owns composition, inheritance, presence, final-definition
|
||||
validation, prompt capture, and execution. Audita does not translate old
|
||||
profiles or implement a second YAML loader or inheritance resolver.
|
||||
- Embedded prompts and maintained operator profiles use keyed `prompts` and
|
||||
`profiles` mappings. Existing prompt IDs, versions, role selections,
|
||||
instructions, and response-schema associations remain the adoption baseline.
|
||||
The application configuration schema, overlay selection, CLI interface, and
|
||||
main-file-relative `profiles_dir` resolution remain in place.
|
||||
- Native profile `extends`, field-presence semantics, and source-relative
|
||||
resource rules apply. Documents can contain multiple definitions, and IDs
|
||||
come from mapping keys rather than filenames. The existing `profiles_dir`
|
||||
accepts native unified PromptKit documents containing profiles, backends,
|
||||
and prompts, including intentional operator prompt overrides.
|
||||
- Application backend endpoints, configured concurrency and queue capacity,
|
||||
and credential ownership remain authoritative for selected roles. Preserve
|
||||
the existing profile-selection and prepared-target checks; do not introduce
|
||||
a second admission controller or implicit provider credential lookup.
|
||||
- A runtime uses one captured configuration snapshot for both roles. Each CLI
|
||||
invocation constructs a new engine; hot reload is not part of this feature.
|
||||
Existing inspection remains offline and safe, and consumed host sources
|
||||
remain protected from output collisions.
|
||||
- PromptKit retains responsibility for transport retries, schema repair,
|
||||
observation, and prepared execution. Preserve Audita's configured retry and
|
||||
repair budgets, operation deadlines, cancellation, partial-success behavior,
|
||||
error/recovery reporting, and debug selection.
|
||||
|
||||
## Configuration Assembly and Ownership
|
||||
|
||||
Replace removed `WithPromptFS`, `WithProfileFS`, and `WithBackend` usage with
|
||||
the released source API. Discover embedded YAML through `FilesystemInputs` and
|
||||
operator YAML through `DirectoryInputs`, with explicit caller tiers. Keep the
|
||||
retained schema filesystem integration and HTTP client configuration.
|
||||
Use the helpers' default-disabled imports; this adoption does not require new
|
||||
application source-selection flags or operator underlay tiers.
|
||||
|
||||
PromptKit supplies catalog definitions first and its own defaults next. Caller
|
||||
tiers then overlay in their supplied order, followed by an optional highest
|
||||
priority main input. Files within one directory group compose strictly;
|
||||
alphabetical filenames do not establish override precedence. Use separate
|
||||
tiers for genuine priority differences rather than manufacturing file-order
|
||||
semantics.
|
||||
|
||||
Represent application backend settings as a final programmatic source, using
|
||||
`BackendSource` or ConfigKit mappings where explicit null is needed. Preserve
|
||||
configured zero queue capacity as an explicit value. Suppress backend-derived
|
||||
credential environment lookup explicitly where catalog layering would otherwise
|
||||
introduce it; omission alone can retain a lower-tier value. Continue detecting
|
||||
competing effective profile credential lookups rather than silently clearing
|
||||
operator profile fields. A missing backend credential remains valid for an
|
||||
unauthenticated local service unless the selected profile requires a key.
|
||||
|
||||
Catalog backend IDs are no longer reserved upstream. Audita should not invent a
|
||||
new reservation rule. Catalog selection alone must not bypass application
|
||||
registration, endpoint, capacity, or credential requirements. Catalog profile
|
||||
contents may evolve; this upgrade does not promise fixed model choices or live
|
||||
provider availability.
|
||||
|
||||
The accepted caller order is embedded Audita prompts, operator definitions,
|
||||
then application backend settings. The existing `profiles_dir` accepts native
|
||||
unified PromptKit documents containing profiles, backends, and prompts. Keep
|
||||
the name and path setting for continuity, and document its broader accepted
|
||||
content. This follows the upstream source API without a parallel parser or
|
||||
top-level filtering. Profiles, provider defaults, and intentional prompt
|
||||
overrides use the same native composition model.
|
||||
|
||||
Audita continues to ship its domain prompts, enforce its selected-backend
|
||||
requirements, and supply its per-call schemas. Operator prompt overrides are
|
||||
an accepted capability of this adoption; documentation, source accounting,
|
||||
and focused behavioral coverage should reflect them.
|
||||
|
||||
## Work Required
|
||||
|
||||
### Runtime Construction and Source Accounting
|
||||
|
||||
Migrate `internal/framework/llm/runtime.go` to layered construction and retain
|
||||
the role-bound adapter boundary. Review startup inspection and failure
|
||||
classification because upstream now validates all final definitions, including
|
||||
unused ones, during engine construction. Distinguish invalid operator inputs
|
||||
from broken embedded assets under the existing
|
||||
[CLI exit contract](../cli.md); do not treat every construction failure as an
|
||||
operator error merely because validation moved earlier.
|
||||
|
||||
Upstream permits partial profiles and defers endpoint/model completeness until
|
||||
request overrides are applied. Audita supplies no model override, so explicitly
|
||||
validate that both selected role profiles have a nonblank model and a complete
|
||||
effective endpoint during offline configuration validation and runtime setup.
|
||||
Retain the application-backend, endpoint-match, and credential checks. Do not
|
||||
rely on engine construction or profile inspection alone to establish selected
|
||||
profile completeness. Unselected partial profiles remain permitted if they
|
||||
satisfy upstream definition validation; they need not be runnable.
|
||||
|
||||
Adapt source accounting to the new discovery/capture boundary. The existing
|
||||
recording filesystem wrapper may be replaceable with discovered inputs or
|
||||
upstream source metadata. Protect the host sources actually consumed by the
|
||||
runtime, including inherited and differently named definitions, without
|
||||
guessing filenames from selected profile IDs. Account for consumed operator
|
||||
message files as well. Keep this proportionate to the existing collision
|
||||
contract; do not add filesystem concurrency or power-loss guarantees.
|
||||
|
||||
Keep the existing safe effective-profile summaries and application explanations.
|
||||
Use upstream metadata where useful, but do not expose the full
|
||||
`InspectConfiguration` snapshot as ordinary CLI or report output: prompt text
|
||||
and arbitrary provider fields are outside those summaries' current scope.
|
||||
|
||||
### Prompt Assets, Profiles, and Presence Semantics
|
||||
|
||||
Move each of the seven files in `internal/prompts/assets/prompts` to the keyed
|
||||
prompt/version format. Keep the shared instruction fragments and the complete
|
||||
per-call output contracts owned by Audita. This migration does not authorize
|
||||
another prompt-quality rewrite or changes to domain validators.
|
||||
|
||||
The adapter supplies `HardeningPolicy`, `ProposalContract`, and
|
||||
`ValidationContract` as template variables. Embedded prompts retain their
|
||||
references to the applicable variables. Operator message overrides control
|
||||
whether those variables are rendered; supplying a variable does not inject its
|
||||
text into a template that omits it. Preserve this native override behavior and
|
||||
document the distinction. Per-call schemas and domain validation remain enforced
|
||||
regardless of those references. Unconditional shared-instruction injection is
|
||||
outside this adoption.
|
||||
|
||||
Convert the six maintained profiles in `examples/profiles` and test-generated
|
||||
profile/prompt fixtures to the new format. Convert any old `base_profile`
|
||||
usage to `extends`, and remove embedded identities. Migrate tests that construct
|
||||
engines using removed options or old `Profile`/`Backend` values.
|
||||
|
||||
Reorganize the maintained profiles into separately selected local and service
|
||||
directory roots. The current shared directory includes profiles referring to
|
||||
backends absent from each example's application configuration; eager validation
|
||||
would reject those unselected definitions after a format-only conversion.
|
||||
Update `profiles_dir` in `examples/minimal-config.yml`,
|
||||
`examples/production-config.yml`, and `examples/configkit-inheritance.yml` to
|
||||
select the appropriate root. Keep the service testing profiles in the service
|
||||
root so the testing overlay remains usable. Discovery is recursive: placing
|
||||
both groups beneath a common selected root does not isolate them. Update example
|
||||
links and validation recipes to match the final layout.
|
||||
|
||||
Review presence deliberately rather than mechanically preserving old serialized
|
||||
values. In particular, explicit zero temperature, top-p, or max-tokens now means
|
||||
an outbound zero; zero timeout disables PromptKit's generation deadline while
|
||||
Audita's outer operation deadline still applies. Omission, blank values, and
|
||||
null have field-specific meanings. Empty provider maps do not erase inherited
|
||||
entries, and changing model/backend does not erase unrelated explicit settings.
|
||||
|
||||
Use the native meanings without a compatibility shim. Retaining the examples'
|
||||
literal `temperature: 0` as an explicit provider setting is recommended; explain
|
||||
that omitting the field requests an unspecified value, and null suppresses an
|
||||
inherited control. Do not automatically turn every existing zero into omission
|
||||
or every omission into a zero. Demonstrate representative profile inheritance
|
||||
without requiring every maintained example to use it.
|
||||
|
||||
### Documentation, Migration, and Attribution
|
||||
|
||||
Update the profile section of `docs/config.md` and maintained examples when the
|
||||
implementation lands. Explain the broader accepted content of `profiles_dir`,
|
||||
operator prompt overrides, their precedence and optional use of shared instruction
|
||||
variables, keyed identities, composition versus precedence, profile inheritance,
|
||||
presence, eager final-definition validation, selected-root errors, and captured
|
||||
configuration. Link to the pinned
|
||||
upstream reference for generic field semantics rather than reproducing its
|
||||
entire contract.
|
||||
|
||||
Include operator migration guidance for old profile documents and behaviorally
|
||||
significant zeros/nulls. This is a deliberate format migration, not an automatic
|
||||
upgrade of existing installation files. Preserve supported application overlays,
|
||||
including the cheaper/faster testing-profile workflow.
|
||||
|
||||
Update the pinned consumer link in `docs/integrations/openai-compatible-llm.md`,
|
||||
construction/source-accounting descriptions in `docs/internal/llm-runtime.md`,
|
||||
and asset-format guidance in `docs/internal/prompts.md`. Update other affected
|
||||
references only where implemented behavior changes.
|
||||
|
||||
Update `go.mod`, `go.sum`, and `docs/oss/attribution.md` for PromptKit and the two
|
||||
catalog modules. Verify exact selected source revisions and upstream notices
|
||||
under the [OSS policy](../policy/oss.md); retain the applicable LGPL/GPL and MIT
|
||||
texts rather than duplicating unchanged license documents. Dependency adoption
|
||||
does not change Audita's application license.
|
||||
|
||||
Plan this adoption for an Audita major release under the
|
||||
[release policy](../policy/release.md#versioning-and-compatibility). The public
|
||||
profile format changes incompatibly and no compatibility loader is planned;
|
||||
PromptKit's own pre-1.0 minor version does not change Audita's classification.
|
||||
The exact version and publication timing remain deferred to release preparation.
|
||||
|
||||
## Verification and Completion
|
||||
|
||||
Use existing offline tests as the starting point. Update fixtures intentionally,
|
||||
and add focused behavioral coverage where the source migration creates a
|
||||
distinct risk. Completion means:
|
||||
|
||||
- The released dependency set builds and the complete Audita test suite passes.
|
||||
- All registered embedded prompts load and render from keyed assets, retaining
|
||||
their applicable shared-instruction references and established per-call schemas.
|
||||
- Offline configuration validation rejects incomplete selected profiles, including
|
||||
a missing or blank model, while accepting valid unselected partial profiles.
|
||||
- Selected roles use final application backend settings and direct resolved
|
||||
credentials; both roles still share configured capacity, including explicit
|
||||
zero queue capacity. Catalog layering does not introduce a competing lookup.
|
||||
- Representative keyed profiles, native inheritance, and explicit presence work
|
||||
through the real runtime. Directory conflicts and invalid final definitions
|
||||
produce useful startup errors without filename-derived identity assumptions.
|
||||
- Operator prompt and backend contributions follow the accepted source order;
|
||||
prompt overrides render, including when their messages omit shared-instruction
|
||||
variables, while Audita retains its per-call schemas, domain validation, and
|
||||
authoritative application backend settings.
|
||||
- Source collision protection, safe inspection, and operator-versus-asset error
|
||||
classification remain effective at the changed construction boundary.
|
||||
- Representative prepared execution, schema repair, transient retry, and
|
||||
partial-success tests retain their existing behavior. Reuse adequate coverage
|
||||
rather than duplicating PromptKit's complete upstream test matrix.
|
||||
- Local, service, testing-overlay, and ConfigKit inheritance examples validate
|
||||
offline using their separately selected profile roots. Canonical documentation
|
||||
describes the migrated format, shared-instruction behavior, and major-release
|
||||
migration; attribution identifies the selected library sources accurately.
|
||||
|
||||
No live provider calls, new release gates, or CI-success dependency are needed
|
||||
to establish adoption. Guided domain recovery and the other work recorded in
|
||||
[future.md](future.md) remain separate features. Do not expand this upgrade into
|
||||
new report formats, pipeline scheduling, or general configuration-inspection
|
||||
commands. The [implementation plan](implementation.md) owns execution steps
|
||||
and their verification.
|
||||
|
||||
## Completion Record
|
||||
|
||||
Implementation code is recorded through revision `5525fe8`; documentation was
|
||||
updated in `9bcf183`, and the final roadmap closure is documentation-only. At
|
||||
`9bcf183`, `go build -o ./bin/audita ./cmd/audita`, `go test ./...`, all six
|
||||
offline maintained-example commands in the implementation plan, `go mod
|
||||
verify`, and `git diff --check` passed. The selected dependency graph and
|
||||
commit-pinned attribution match PromptKit v0.12.0 and both `/v3` catalogs at
|
||||
`v3.20260930.0`.
|
||||
@@ -1,161 +0,0 @@
|
||||
# Maintainability Audit: Findings and Recommended Changes
|
||||
|
||||
**Reviewed:** 2026-09-25, source commit `011b021`.
|
||||
**Scope:** Recent partial-decision reconciliation, validation context, prompt
|
||||
alignment, and request-local reference changes, including related existing code.
|
||||
**Status:** R27–R30 addressed; see the resolution record below. Previous findings and resolution records remain in Git
|
||||
history through `011b021`.
|
||||
|
||||
## Architecture Assessment
|
||||
|
||||
The recent changes fit Audita's existing boundaries. Proposal generation,
|
||||
validation, prompt assets, and outcome reporting already have suitable owners;
|
||||
this audit does not recommend a new common package or a broad refactor.
|
||||
|
||||
The strongest opportunities are shared prompt instructions and request-scoped
|
||||
validation context preparation. Two smaller improvements concern recovery-reason
|
||||
classification and partial-result assembly; both predate the latest commits.
|
||||
These are bounded maintenance recommendations, not new release gates or changes
|
||||
to correction policy, response acceptance, or reporting semantics.
|
||||
|
||||
## R27 — P2: Consolidate Shared Prompt Instructions
|
||||
|
||||
**Category:** Repeated behavioral-contract text; risk of future drift.
|
||||
|
||||
**Evidence:** The four [proposal prompts](../../internal/prompts/assets/prompts/modules.grammar.proposal.yaml)
|
||||
repeat literal source-span matching, request-local identity, replacement-text,
|
||||
source-based proposal, and confidence instructions. The three active
|
||||
[validator prompts](../../internal/prompts/assets/prompts/validators.editorial_review.yaml)
|
||||
repeat candidate-reference, supplemental-context, and verdict-confidence guidance.
|
||||
Recent contract changes required coordinated edits across these assets. The
|
||||
[prompt registry](../../internal/prompts/registry.go) already provides a shared
|
||||
hardening fragment supplied through the role-bound adapter.
|
||||
|
||||
**Recommended change:** Extract genuinely common passages into embedded fragments
|
||||
under `internal/prompts/assets/shared`, following the existing fragment-supply
|
||||
approach. Keep the fragments owned by `internal/prompts` and use the existing
|
||||
PromptKit rendering path. Keep module-specific examples, replacement-policy
|
||||
instructions, protected-name exceptions, and semantic criteria in their individual
|
||||
prompt definitions. Do not merge similar-looking rules with different meanings.
|
||||
|
||||
**Acceptance:** Common instructions have one maintained source and appear in all
|
||||
intended rendered prompts. The existing module and validator distinctions remain
|
||||
intact, including glossary replace-all behavior and the spoken-word protected-name
|
||||
exception. Focused rendering checks cover fragment inclusion and representative
|
||||
module-specific guidance without requiring exact-prose snapshots or introducing
|
||||
a separate templating framework.
|
||||
|
||||
## R28 — P2: Prepare Validation Context Once Per Request
|
||||
|
||||
**Category:** Repeated computation and overlapping segment lookup.
|
||||
|
||||
**Evidence:** [BuildLLMValidationRequest](../../internal/framework/validators/llm_validators.go)
|
||||
already indexes transcript segments, but
|
||||
[validationContext](../../internal/framework/validators/llm_context.go) scans the
|
||||
transcript again for each candidate. It also calls
|
||||
[NewProtectedVocabulary](../../internal/framework/validators/protected_terms.go)
|
||||
for each candidate, rebuilding the same vocabulary and compiling its matching
|
||||
expression repeatedly. This is duplicated preparation rather than copied code.
|
||||
|
||||
**Recommended change:** Prepare a segment-position lookup and protected vocabulary
|
||||
once per validation request, then pass that prepared context into the existing
|
||||
context-selection helper. Keep the implementation within `framework/validators`.
|
||||
Continue selecting glossary evidence for each candidate individually because its
|
||||
proposed replacement can mention different terms. A process-wide cache or new
|
||||
context-management package is unnecessary for this cleanup.
|
||||
|
||||
**Acceptance:** Candidates in one validation request reuse the prepared lookup and
|
||||
vocabulary. Context still consists of the available immediate neighbors within the
|
||||
current chunk and relevant glossary entries in glossary order. Candidate-specific
|
||||
replacement text still participates in glossary selection. Existing optional-context
|
||||
trimming, token estimation, and oversized-input handling retain their behavior.
|
||||
Verify these outcomes with focused behavioral tests rather than assertions about
|
||||
private helper call counts or unmeasured performance guarantees.
|
||||
|
||||
## R29 — P3: Share Execution-Recovery Classification
|
||||
|
||||
**Category:** Duplicate reporting rule; predates the latest commits.
|
||||
|
||||
**Evidence:** [Proposal generation](../../internal/framework/proposal_generation/generate.go)
|
||||
and [LLM validation](../../internal/framework/validators/llm_validators.go)
|
||||
independently translate structural-repair and transient-retry counts into
|
||||
`output_repaired` and `service_recovered` reasons. The conversion is the same in
|
||||
both locations; the surrounding success and domain-outcome decisions differ.
|
||||
|
||||
**Recommended change:** Place a small conversion helper in the existing
|
||||
`framework/outcomes` package and use it from both callers. Keep the decision to
|
||||
record recoveries at the call sites, along with domain-specific reasons such as
|
||||
filtered validator decisions. Do not combine proposal and validation orchestration
|
||||
or move domain-success classification into this helper.
|
||||
|
||||
**Acceptance:** Both callers use the same execution-count conversion. Existing
|
||||
reason ordering, successful-operation reporting, and domain-specific additions
|
||||
remain intact. A partially unavailable validation batch continues to record its
|
||||
failure and execution counts without acquiring recovery records merely because a
|
||||
repair or retry occurred. Existing outcome tests should continue to protect these
|
||||
distinctions.
|
||||
|
||||
## R30 — P3: Centralize Partial Validator Result Assembly
|
||||
|
||||
**Category:** Repeated early-return construction; predates the latest commits.
|
||||
|
||||
**Evidence:** Several early returns in
|
||||
[LLMBackedValidator.Validate](../../internal/framework/validators/llm_validators.go)
|
||||
repeat nested slice appends to combine immediate decisions, oversized-input
|
||||
decisions, and decisions from completed model batches, together with accumulated
|
||||
operation outcomes. This makes it easier for a future failure path to omit facts
|
||||
already collected by the validator.
|
||||
|
||||
**Recommended change:** Use a small local result-building helper to assemble the
|
||||
partial result consistently. Keep error handling, batching, success-path ordering,
|
||||
and cardinality enforcement explicit. A general-purpose result-aggregation package
|
||||
is not needed.
|
||||
|
||||
**Acceptance:** Cancellation and failure paths retain the same previously collected
|
||||
decisions and operations. Result ordering and cardinality behavior remain unchanged;
|
||||
the helper does not enforce complete cardinality on interrupted work. Retain the
|
||||
existing partial-progress and fatal-failure coverage, adding tests only for a
|
||||
material gap exposed by the refactor.
|
||||
|
||||
## Boundaries to Preserve
|
||||
|
||||
- Keep proposal and validator reference mapping separate. Proposals can target the
|
||||
same segment repeatedly; duplicate validator decisions make that candidate
|
||||
unavailable. Similar indexing arithmetic alone does not justify a generic
|
||||
reference-management abstraction.
|
||||
- Keep distinct protected-term policy checks separate. They already share the
|
||||
vocabulary implementation and intentionally apply different policies.
|
||||
- Preserve exact source-span matching, request-local references, batching, and
|
||||
partial-success behavior. Consolidation is not a reason to strengthen contracts
|
||||
or broaden the accepted correction scope.
|
||||
|
||||
## Review Verification and Limits
|
||||
|
||||
- This is a source-level maintainability audit, not a finding of demonstrated
|
||||
production failures or a measured performance assessment.
|
||||
- No application tests were rerun for the original read-only audit; its roadmap
|
||||
update used documentation link and whitespace checks. Implementation validation
|
||||
is recorded below.
|
||||
- Future remediation should run focused behavioral tests and the broader suite
|
||||
appropriate to the actual change, following the testing policy. No live-provider
|
||||
evaluation or new evaluation framework is required by these recommendations.
|
||||
- The findings above describe the audited baseline. Implementation and validation
|
||||
are recorded below; no release publication is included.
|
||||
|
||||
## Resolution Record
|
||||
|
||||
Implemented as focused changes within the existing packages. Proposal and validator
|
||||
reference mapping and distinct protected-term policies remain separate.
|
||||
|
||||
| Finding | Resolution and evidence |
|
||||
| --- | --- |
|
||||
| R27 | Common proposal and validation instructions live in two embedded shared fragments, supplied through the existing role-bound PromptKit adapter. Individual YAML definitions retain policy differences and examples. Rendering tests check fragment inclusion once per applicable prompt and representative module-specific guidance. |
|
||||
| R28 | Validation request construction builds one segment-position lookup and protected vocabulary, reused by context selection for every candidate. A new test verifies that two replacements of the same source span receive their own relevant glossary entries. Existing neighbor, glossary-order, chunk-boundary, and budget tests pass. |
|
||||
| R29 | Both proposal generation and validation use `outcomes.ExecutionRecoveries`. Call sites still control success and domain-specific recovery reasons. Existing tests cover combined recoveries and failed validation with repair counts but no recovery record. |
|
||||
| R30 | A local result builder combines collected decisions and operation outcomes for partial returns and completed validation. Sorting and full cardinality checks remain outside it. Existing fatal-batch, partial-progress, runner, and reporting tests pass. |
|
||||
|
||||
Validation: `go test ./...` passed, including PromptKit preparation, partial-decision
|
||||
reconciliation, subprocess reports, and fatal-progress coverage. Documentation
|
||||
links and whitespace checks passed. No live model evaluation or performance
|
||||
measurement was performed; the changes make no new model-quality or latency
|
||||
promise.
|
||||
@@ -1,290 +0,0 @@
|
||||
# Resilient Processing and Focused Run Reports
|
||||
|
||||
**Last updated:** 2026-09-24
|
||||
|
||||
**Status:** Implemented. This document records the selected behavior and user
|
||||
intent. Concrete design decisions and completion evidence are in
|
||||
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/a83a518/docs/roadmap/implementation.md).
|
||||
|
||||
## Purpose and User Intent
|
||||
|
||||
Audita should reliably produce a useful transcript even when its LLM services
|
||||
cannot improve it. Exhausted retries or unusable model responses must not turn
|
||||
an otherwise processable transcript into a failed production run.
|
||||
|
||||
> Failure of every LLM operation across every stage and chunk still produces a
|
||||
> successful output containing the deterministically normalized transcript, with
|
||||
> no LLM corrections applied. Partial success preserves as many fully validated,
|
||||
> applicable corrections as possible.
|
||||
|
||||
Deterministic normalization remains intentional and may change segment layout,
|
||||
identifiers, and text before LLM work begins. Output uses the selected transcript
|
||||
encoder; byte-for-byte preservation of the original input and mapping corrections
|
||||
back to pre-normalization segmentation are not requirements.
|
||||
|
||||
The overriding goal is to minimize how much useful processing is lost because of
|
||||
one problematic transcript segment. A stage/chunk work unit is an attribution
|
||||
boundary, not an all-or-nothing transaction.
|
||||
|
||||
Existing reports contain useful information, but there are no compatibility
|
||||
obligations for reporting or debug output. Reuse existing machinery only when it
|
||||
makes the implementation simpler and clearer; preservation is not itself a goal.
|
||||
Internal structures and measurements must serve Audita's processing workflow.
|
||||
The reporting package adapts those outcomes to structured and human-facing
|
||||
presentations, rather than reshaping processing around a reporting convenience.
|
||||
|
||||
## Historical Baseline and Implemented Change
|
||||
|
||||
The [pipeline](../internal/pipeline.md), [LLM integration](../integrations/openai-compatible-llm.md),
|
||||
[report assembly](../internal/diagnostics-reporting.md), and
|
||||
[artifact lifecycle](../operations.md) describe current behavior.
|
||||
|
||||
Before this work, exhausted structural repairs could become warnings and skipped
|
||||
proposals or rejected validation batches. Other model runtime failures canceled
|
||||
peer work, stopped subsequent modules, and prevented final transcript output.
|
||||
Successful repairs appeared in execution observations rather than a dedicated
|
||||
recovery classification. Detailed artifacts were written during ordinary runs
|
||||
and could later be removed by retention. The working transcript was re-chunked
|
||||
between modules.
|
||||
|
||||
This feature changed those behaviors. It builds on the completed
|
||||
[library adoption](library-adoption.md) and [review remediation](remediation.md).
|
||||
Fatal cancellation remains necessary; unsuccessful model work must be separated
|
||||
from failures that actually require cancellation of the run.
|
||||
|
||||
## Fixed Chunks and Ordered Stages
|
||||
|
||||
Normalize first, then partition the transcript once. Preserve chunk boundaries,
|
||||
segment membership, segment identity, and ordering throughout LLM processing.
|
||||
Text evolves as accepted corrections are applied. Recombine chunks in original
|
||||
order after the LLM stages finish.
|
||||
|
||||
A stage is a configured module instance. Repeated instances of the same module
|
||||
have distinct identities. Proposal generation and each validator are operations
|
||||
within a stage; validation may use multiple batches within a chunk. A work unit
|
||||
is one stage instance applied to one chunk.
|
||||
|
||||
Preserve the overall stage sequence: finish a stage across its chunks before
|
||||
advancing to the next stage. Concurrent chunk work shares PromptKit's backend
|
||||
capacity. Failure in one unit does not prevent other chunks or later stages
|
||||
from executing.
|
||||
|
||||
Retain operator control over chunk size and the current defaults. Production
|
||||
experience indicates that chunk size affects model latency nonlinearly and that
|
||||
useful settings are well below backend context limits. Tuning is an empirical
|
||||
operator decision. Do not add automatic headroom calculations, cross-profile
|
||||
sizing, adaptive repartitioning, or automatic tuning in this feature. A later
|
||||
operation that cannot process its input follows the same safe error handling;
|
||||
it does not silently alter chunk boundaries. Validation batching within a chunk
|
||||
remains supported.
|
||||
|
||||
Independent whole-pipeline traversal by chunks and domain-aware response repair
|
||||
are explicitly deferred in [future work](future.md). Read-only neighboring
|
||||
context, cross-chunk edits, and transcript-wide restructuring are outside this
|
||||
unit, without becoming permanent prohibitions on future capabilities.
|
||||
|
||||
## Partial Success and Failure Isolation
|
||||
|
||||
- Keep bounded transport retries, structural repairs, operation deadlines, and
|
||||
shared backend capacity. Completion comes from safe fallback, not unlimited
|
||||
attempts or an outer retry loop around PromptKit.
|
||||
- Apply a correction only after it passes every required validator and the
|
||||
application checks. Failed validation must never be treated as approval.
|
||||
- If a validation batch fails, exclude its affected candidates while allowing
|
||||
other candidates to continue through the remaining validators. Preserve fully
|
||||
validated, applicable corrections from successful batches in the same unit.
|
||||
- Track candidates by stable proposal identity. Batch membership can change
|
||||
between validators; equal batch numbers do not establish candidate identity.
|
||||
- A failed proposal-generation operation leaves its chunk unchanged for that
|
||||
stage because there are no reliable proposals to salvage. Failures later in
|
||||
validation need not discard unrelated successful candidates.
|
||||
- Preserve earlier stages' corrections and other chunks' successful work.
|
||||
Record partial success explicitly; a unit can apply corrections and have errors.
|
||||
- Continue despite model-operation failures, including exhausted repairs/retries,
|
||||
unusable model decisions, provider/transport failures, local operation deadlines,
|
||||
and admission failures. An operation need not exhaust its attempt budget to fail.
|
||||
- A valid empty proposal list or valid assessments rejecting all corrections is
|
||||
successful processing. Correction count alone does not determine failure.
|
||||
- Keep safe filtering of defective proposals. Do not retry ordinary editorial
|
||||
rejections. For now, an unusable validator decision set excludes the affected
|
||||
candidates and records an error after existing permitted recovery. New bounded
|
||||
feedback/regeneration for domain-invalid responses belongs to future work.
|
||||
- Retain fatal treatment for invalid input/configuration, internal defects,
|
||||
explicit run cancellation, and inability to produce/write the transcript.
|
||||
Caller cancellation is distinct from a local model-operation timeout.
|
||||
- Completed runs with model errors return success. Required report-write failures
|
||||
are an additional fatal output condition when reporting is enabled, as below.
|
||||
|
||||
PromptKit owns transport, transient retries, structural repair, and admission.
|
||||
Audita owns domain acceptance, partial outcomes, fallback, and classification.
|
||||
ConfigKit owns configuration resolution. Debug and reporting choices must not
|
||||
change correction decisions, retry budgets, or continuation.
|
||||
|
||||
## Errors, Recoveries, and Debug
|
||||
|
||||
Classify by eventual outcome and impact. These are three reporting categories;
|
||||
debug is evidence, not a failure severity. Whether execution continues is a
|
||||
separate property.
|
||||
|
||||
| Category | Meaning | Representative cases |
|
||||
| --- | --- | --- |
|
||||
| Errors | Intended work could not complete reliably, or a fatal run-level operation failed. Affected candidates are excluded; other corrections can succeed. | Exhausted repair; unavailable provider; unusable validation decisions; operation deadline. |
|
||||
| Recoveries | A problem was handled successfully through retries, repair, or safe filtering. No operator intervention is implied. | Invalid output repaired; transient service failure recovered; defective proposals safely discarded. |
|
||||
| Debug | Detailed evidence of processing and ordinary decisions. | Proposals, applied corrections, validator decisions, prompts/responses, individual attempts, execution traces. |
|
||||
|
||||
Normal editorial disagreement belongs in debug. A validator correctly rejecting
|
||||
a meaning-changing edit is expected behavior. A proposal referring to nonexistent
|
||||
original text is defective; safely filtering it is a recovery. An unresolved
|
||||
missing/duplicate decision-index response is an error, not a recovery.
|
||||
|
||||
Record an operation's eventual outcome once. Several failed attempts followed by
|
||||
success produce a consolidated recovery. Attempts ending in failure contribute
|
||||
to error details, not additional recoveries. Earlier operations that recovered
|
||||
remain recorded if a later operation introduces an error in the same unit.
|
||||
|
||||
Separate output-quality recovery, defective-proposal filtering, and service
|
||||
recovery with structured reasons. Network instability must not look like a prompt
|
||||
defect. Do not introduce an arbitrary warning threshold or a fourth category.
|
||||
|
||||
### Counting and Processing Volume
|
||||
|
||||
- Count error-bearing work units once per stage/chunk, consolidating underlying
|
||||
operation failures as details. A partially successful unit still counts once.
|
||||
- Count recoveries once per recovered operation. Aggregate defective-proposal
|
||||
filtering per proposal response, retaining affected-proposal counts as detail.
|
||||
Attempts and individual discarded proposals are not independent incidents.
|
||||
- Keep work-unit coverage separate from operation/recovery counts. Show processing
|
||||
volume with explicitly labeled denominators; do not divide recovered-operation
|
||||
counts by work-unit counts and call that a recovery rate.
|
||||
- Report fatal run-level errors separately from the stage/chunk grid when they
|
||||
cannot be attributed to a work unit. Do not invent a chunk to accommodate them.
|
||||
- Derive YAML records, Markdown summaries, and CLI counts from actual processing
|
||||
outcomes with deterministic ordering, rather than independently calculating them.
|
||||
|
||||
## Production Outputs, Placement, and Lifecycle
|
||||
|
||||
Reporting is enabled by default. The ordinary bundle is:
|
||||
|
||||
- `errors.yml`: structured error-bearing units, their underlying failures, and
|
||||
applicable run-level failures.
|
||||
- `recoveries.yml`: structured recovered operations and pertinent counts.
|
||||
- `report.md`: a concise human-facing summary of outcome, corrections, processing
|
||||
volume, errors, recoveries, and their locations in the pipeline.
|
||||
|
||||
Write all three documents, including empty event lists for uneventful completed
|
||||
runs. Use Audita-owned schemas with safe explanations and stable reason codes;
|
||||
retain run, stage-instance, chunk, operation, and validator/batch identities where
|
||||
applicable. The implementation plan specifies the versioned schemas; neither old
|
||||
report formats nor raw library structures define the new output contract.
|
||||
|
||||
Place reports beside the output transcript. Retain transcript-to-stdout support
|
||||
with an explicit report-directory selection, defaulting to the invocation working
|
||||
directory. Preserve the existing overwrite convention: a new run can replace files
|
||||
at its selected destinations. Do not add a mandatory fresh-directory restriction
|
||||
or compatibility exports.
|
||||
|
||||
Before model work, create initial versions of all three reports carrying a common
|
||||
run identifier and explicit `in_progress` status (rendered as prose in Markdown).
|
||||
Fail immediately if initialization fails. Also check the transcript destination;
|
||||
creating neighboring reports does not prove that an existing transcript file is
|
||||
writable. Preflight must not prematurely destroy an existing transcript.
|
||||
|
||||
Finalize all required outputs and check every final write. Initial write success
|
||||
cannot guarantee later success if space or permissions change. Failure to write
|
||||
the transcript or finalize a required report returns nonzero and must not erase
|
||||
an already produced transcript. Where possible, finalize available reports with
|
||||
an explicit failed status on fatal termination. Abrupt termination may leave
|
||||
`in_progress`; elaborate crash-recovery machinery is not required.
|
||||
|
||||
Provide an option to disable reporting entirely. It suppresses all three files
|
||||
(`errors.yml`, `recoveries.yml`, and `report.md`), including initialization and
|
||||
final writes, while retaining the brief CLI summary and independently selectable
|
||||
debug output. Disabled reporting must not create or require a writable report
|
||||
destination, and must not remove reports from earlier runs.
|
||||
Transcript output failures remain fatal regardless of reporting selection.
|
||||
|
||||
Ordinary reports survive independently of debug cleanup. Reporting configuration
|
||||
and finalization must not depend on retaining a debug run directory.
|
||||
|
||||
## Debug and Internal Boundaries
|
||||
|
||||
Production issue records are compact. Full responses, transcript excerpts,
|
||||
correction lists, and complete validation decisions are opt-in debug evidence.
|
||||
Use a single ConfigKit-backed debug boolean with a `--debug` CLI override. Debug
|
||||
artifacts may use JSON where useful. Preserve configured-secret redaction in every
|
||||
presentation.
|
||||
|
||||
Debug-off prevents detailed artifact persistence; writing and later deleting
|
||||
those files is insufficient. In-memory domain state needed to process candidates
|
||||
correctly is not debug output. Debug and ordinary reporting are independent
|
||||
selections and do not alter processing behavior.
|
||||
|
||||
There is no obligation to retain `--report-json`, old artifact layouts, parallel
|
||||
exports, or old retention settings. Reuse code only where useful to the new
|
||||
workflow. Enabled debug output is retained on success or failure; disabled debug
|
||||
creates no debug directory. This supersedes the retention question in the
|
||||
[runtime follow-up](runtime-behavior.md). The legacy `never` behavior is not the
|
||||
meaning of the new reporting-disable option. Optional debug-write failures produce
|
||||
a safe stderr diagnostic without changing correction decisions or the run exit
|
||||
status; ordinary report and transcript-write failures retain their fatal semantics.
|
||||
|
||||
## Deterministic Human-Facing Presentation
|
||||
|
||||
Use the CLI style:
|
||||
|
||||
> Completed: 18 corrections applied; 3 work units with errors; 7 recoveries.
|
||||
|
||||
Do not say that all error-bearing units were skipped: some apply valid corrections.
|
||||
Include report locations when reporting is enabled and the files exist. Send the
|
||||
summary to stderr so stdout remains exclusively available for transcript output.
|
||||
Disabled reporting does not suppress the summary or fatal-error messages.
|
||||
|
||||
Keep `report.md` concise and high-level. Use separate error and recovery matrices
|
||||
with stages as columns and chunks as rows. Include every chunk in fixed order;
|
||||
retain configured stage order and distinct labels for repeated module instances.
|
||||
No automatic width detection, thresholds, adaptive omission, or table splitting.
|
||||
A wide or long table is acceptable.
|
||||
|
||||
The error matrix counts error-bearing work units: each attempted cell is 0 or 1.
|
||||
Recovery cells count recovered operations and can exceed 1. An error cell may
|
||||
coexist with recoveries and successfully applied corrections in the same unit.
|
||||
Distinguish completed work with zero events from unattempted work; do not portray
|
||||
canceled/unavailable processing as clean completion. Include totals, processing
|
||||
volume, and a short reason breakdown. Detailed inspection belongs in YAML or debug.
|
||||
|
||||
## Target Acceptance Criteria
|
||||
|
||||
- Total LLM failure still returns success and the encoded normalized transcript,
|
||||
provided required output writes succeed.
|
||||
- Mixed-success runs preserve independently validated, applicable corrections
|
||||
within the same stage/chunk as a failed batch. No candidate bypasses a required
|
||||
validator; later validators handle survivors by stable proposal identity.
|
||||
- Subsequent stages and unaffected chunks execute after recoverable failures.
|
||||
Fatal errors and explicit cancellation stop and join active work appropriately.
|
||||
- Chunk boundaries and identity remain fixed, with operator-selected sizing and
|
||||
unchanged defaults; final recombination preserves ordering.
|
||||
- Valid no-change/editorial-rejection outcomes are successful. Repaired/filtered
|
||||
output and unresolved failures receive distinct classifications and counts.
|
||||
- YAML, Markdown, and CLI agree on partial outcomes and totals; error matrices
|
||||
count affected work units, recovery matrices count recovered operations.
|
||||
- With reporting enabled, all three reports are initialized before model work,
|
||||
finalized and retained, including empty lists. Initial or final required-output
|
||||
write failures produce nonzero exits without destroying completed transcript output.
|
||||
- Reporting can be disabled without requiring its destination. Debug can be enabled
|
||||
independently; neither selection changes processing outcomes. Debug-off writes
|
||||
no detailed debug artifacts, and stdout transcript output remains supported.
|
||||
- Reports use the fixed deterministic layout without adaptive presentation logic.
|
||||
- Meaningful controlled-provider tests cover exhausted recovery, failed validation
|
||||
batches with successful survivors, total failure, fatal cancellation, output
|
||||
failures, and reporting/debug selections. Preserve domain safeguards, capacity
|
||||
limits, and redaction.
|
||||
- Canonical CLI, configuration, operations, integration, internal, and architecture
|
||||
documentation describes the implemented final behavior.
|
||||
|
||||
## Implementation Reference
|
||||
|
||||
The product decisions are settled. The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/a83a518/docs/roadmap/implementation.md) defines
|
||||
the selected settings and precedence, YAML schemas, counts and ordering, interrupted
|
||||
work presentation, destination checks, and output finalization. It also owns the
|
||||
ordered implementation stages and validation work. This document remains the
|
||||
reference for the desired behavior and its acceptance criteria.
|
||||
@@ -1,18 +0,0 @@
|
||||
# Runtime Behavior Decisions
|
||||
|
||||
**Last updated:** 2026-09-24
|
||||
|
||||
The documentation migration identified existing behaviors for separate product
|
||||
consideration. The resilience implementation resolved the diagnostics-retention
|
||||
question; the input-timestamp question remains open.
|
||||
|
||||
- Resolved: legacy `auto`/`always`/`never` retention settings were removed.
|
||||
Debug is explicitly enabled with `diagnostics.debug` or `--debug`, creates no
|
||||
directory when disabled, and retains its directory on every outcome when enabled.
|
||||
- Resolved: [configuration](../config.md) now treats explicit zero as disabling
|
||||
transient retries or structural repair independently; PromptKit performs both.
|
||||
- Decide whether missing or null input timestamps should be rejected; current
|
||||
decoding is documented in the [input contract](../integrations/transcript-glossary-files.md).
|
||||
|
||||
These decisions are independent of the completed documentation migration and
|
||||
release workflow.
|
||||
@@ -1,203 +0,0 @@
|
||||
# Consistent Span Deletion
|
||||
|
||||
**Last updated:** 2026-09-25
|
||||
|
||||
**Status:** Implemented. This document owns the desired behavior and user intent.
|
||||
The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md)
|
||||
records the concrete steps, selected implementation details, and completion
|
||||
evidence.
|
||||
|
||||
## Purpose and User Intent
|
||||
|
||||
Align proposal prompts, the LLM response schema, internal validation, and
|
||||
application behavior so legitimate text-span deletions can be considered and
|
||||
applied through the ordinary correction workflow. An empty replacement should
|
||||
not trigger structural repair merely because it is empty, nor invalidate other
|
||||
usable corrections returned alongside it.
|
||||
|
||||
This is a consistency fix within Audita's current editing scope. It does not
|
||||
authorize whole-segment removal, broaden module responsibilities, or introduce
|
||||
new recovery machinery. The accepted [resilience behavior](resilience.md) remains
|
||||
in effect, including partial success, fixed chunks, and validation before mutation.
|
||||
|
||||
## Historical Baseline (`0b6cd98`)
|
||||
|
||||
The source reviewed at `0b6cd98` contains a mismatch:
|
||||
|
||||
- The [spoken-word prompt](../../internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml)
|
||||
permits removing fillers and asks for the exact affected span and replacement,
|
||||
without explaining how to represent deletion. Grammar permits punctuation and
|
||||
spacing changes, which can also naturally involve deletion.
|
||||
- The shared [correction response schema](../../internal/framework/responseschema/assets/correction_set.v1.json)
|
||||
requires `corrected_text` to have at least one character. Every built-in proposal
|
||||
module uses that schema. An empty replacement invalidates the response before
|
||||
Audita evaluates individual proposals.
|
||||
- PromptKit performs bounded structural repair. If repair does not produce a
|
||||
valid response, the proposal operation fails and yields no candidates, including
|
||||
otherwise valid proposals in the same response. The current resilience path
|
||||
records the error and continues other work.
|
||||
- [CorrectionProposal](../../internal/framework/proposals/proposal.go) already
|
||||
allows empty replacement text. The
|
||||
[preview/application logic](../../internal/framework/proposals/preview.go)
|
||||
supports literal span deletion when the resulting segment remains nonblank.
|
||||
- Domain shape checks currently reject whitespace-only original spans, although
|
||||
the response schema permits them and grammar prompts permit spacing cleanup.
|
||||
This is another restriction to align with the target contract below.
|
||||
- The validator named `non_empty_corrected_text` actually checks the resulting
|
||||
segment, not the replacement string. Both this check and application prevent
|
||||
a segment from becoming empty or whitespace-only. Application changes segment
|
||||
text; it does not remove segment records.
|
||||
|
||||
A temporary controlled-provider review reproduced whole-response rejection for
|
||||
an empty replacement alongside a valid correction, and immediate schema acceptance
|
||||
for a space-only replacement. Existing proposal tests confirm internal support for
|
||||
partial deletion and rejection of blank resulting segments. These observations
|
||||
explain a plausible production failure mechanism; they do not establish the cause
|
||||
of any particular historical incident without its payload and running version.
|
||||
|
||||
## Target Correction Contract
|
||||
|
||||
- `original_text` identifies an existing, nonempty literal span in the current
|
||||
target segment. Whitespace-only spans are eligible; an empty string identifies
|
||||
no source span and remains invalid. Matching and ambiguity checks still apply
|
||||
under the module's replacement policy: `require_unique` for grammar, spoken-word,
|
||||
and homophones; `replace_all` for glossary.
|
||||
- `corrected_text` remains a required string. An empty string means delete the
|
||||
identified span; it does not mean remove the segment record or skip validation.
|
||||
Missing fields and non-string values remain structurally invalid.
|
||||
- After applying a correction, the segment must contain non-whitespace text.
|
||||
Enforce this using the source segment and replacement result in Audita, not a
|
||||
minimum replacement length in the structural schema.
|
||||
- Every accepted deletion passes the same required validators as other edits in
|
||||
its module, including confidence, protected terms, editorial review, and meaning
|
||||
preservation where applicable. Empty replacement is neither automatic approval
|
||||
nor an automatic rejection.
|
||||
- Application preserves segment identity, speaker, timing, membership, ordering,
|
||||
and fixed chunk boundaries. Recheck applicability against text produced by
|
||||
earlier accepted corrections, retaining existing conflict handling.
|
||||
- Preserve literal replacement behavior. Prompts should select spans and
|
||||
replacements that handle adjacent whitespace and punctuation appropriately;
|
||||
this feature does not add automatic trimming or a second normalization pass.
|
||||
Neither side of a correction is rejected solely for containing only whitespace.
|
||||
For example, reducing a uniquely identified run of two spaces to one space may
|
||||
be appropriate; validity still depends on the resulting text and domain checks.
|
||||
|
||||
Preserve the intended safeguards and outcomes, not every detail of the current
|
||||
implementation. Validator ordering, internal structure, and payload construction
|
||||
may change when needed to achieve this consistent contract. Existing behavior or
|
||||
a passing test does not justify retaining a conflicting restriction.
|
||||
|
||||
Illustrative cases:
|
||||
|
||||
| Segment | Original span | Replacement | Intended handling |
|
||||
| --- | --- | --- | --- |
|
||||
| `uh hello` | `uh ` | empty string | Structurally valid; may produce `hello` after required validation. |
|
||||
| `uh hello` | `uh hello` | `hello` | Remains a valid representation of the same cleanup, subject to validation. |
|
||||
| `hello world` | two spaces | one space | Eligible spacing correction, subject to unique matching and required validation. |
|
||||
| `uh` | `uh` | empty string | Structurally valid proposal, then filtered because the resulting segment would be blank. |
|
||||
| `hello world` | `hello world` | spaces only | Filtered because the resulting segment would be blank. |
|
||||
|
||||
These examples describe eligibility and mechanical results, not unconditional
|
||||
editorial approval. An unchanged segment or valid empty correction list remains
|
||||
a successful no-change outcome.
|
||||
|
||||
## Prompt and Module Alignment
|
||||
|
||||
Review relevant prompt definitions for consistency and edit only those needing
|
||||
clarification. Where needed, explain empty-string deletion, whitespace-only spans,
|
||||
and the resulting-segment limit with a concise, module-appropriate example. The
|
||||
model should omit a correction that would leave a blank segment rather than invent
|
||||
padding or replacement content. Consistent guidance is the goal; adding deletion
|
||||
language to every prompt is not a requirement.
|
||||
|
||||
Retain each module's existing scope:
|
||||
|
||||
- Spoken-word cleanup can propose appropriate filler or dysfluency deletion while
|
||||
preserving substantive content and intentional emphasis.
|
||||
- Grammar can propose permitted punctuation or spacing deletion; allowing an
|
||||
empty replacement does not authorize word removal or spoken-word cleanup.
|
||||
- Glossary and homophone modules retain their existing correction purposes.
|
||||
Shared deletion syntax does not grant them new editorial responsibilities.
|
||||
|
||||
Review the active LLM-validator prompts and their input construction as well as
|
||||
proposal prompts. Validators must understand the before/after effect of deletion
|
||||
and assess it under their existing criteria. A correct negative editorial decision
|
||||
remains ordinary processing, not a defect that should be regenerated until approved.
|
||||
|
||||
The resulting-segment validator's name and documented identity should describe
|
||||
that responsibility accurately, rather than imply that the replacement string
|
||||
itself must be nonempty. Preserve the existing blank-result failure reason and
|
||||
safeguard while making that deliberate identity change; current chain position
|
||||
and preview implementation are not independent product requirements.
|
||||
|
||||
## Outcomes and Reporting
|
||||
|
||||
A legitimate deletion is an ordinary correction. Empty replacement alone must
|
||||
produce neither structural repair nor a recovery classification. If another
|
||||
problem required successful repair or filtering, retain that actual recovery.
|
||||
|
||||
A schema-valid proposal that would blank its stage-input segment is a defective
|
||||
proposal handled through existing safe filtering. Preserve the
|
||||
`empty_resulting_segment` reason
|
||||
and consolidated `proposals_filtered` recovery behavior; other usable proposals
|
||||
in the response continue through required validation. Ordinary editorial rejection
|
||||
remains debug evidence rather than a model error or filtering recovery.
|
||||
|
||||
A stage can propose several edits to one segment in the same response. Each edit
|
||||
is evaluated against the stage input, then approved edits are applied sequentially.
|
||||
If a proposal was applicable to that input but an earlier accepted edit makes it
|
||||
unsafe, skip it as an apply-time conflict. Do not count that conflict as a defective
|
||||
proposal, model error, or filtering recovery. For example, in `"uh um"`, deleting
|
||||
`"uh "` and deleting `"um"` each leaves text individually; after the first deletion,
|
||||
the second must be skipped to prevent a blank result. Retain the skipped decision
|
||||
in debug evidence. The existing runner already distinguishes this situation.
|
||||
|
||||
Debug evidence must accurately retain the empty replacement and the corresponding
|
||||
validation/application outcome. Ordinary reports retain their existing counts and
|
||||
purpose; no new report schema, incident category, or deletion-specific counter is
|
||||
needed. Unrelated structural defects may still exhaust PromptKit repair and produce
|
||||
an error for the operation, as today.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A controlled valid response containing an empty replacement reaches Audita's
|
||||
domain validation without structural repair solely due to that empty string.
|
||||
- A permitted span deletion can pass the required validators and reach the final
|
||||
transcript while preserving segment metadata and fixed chunk membership.
|
||||
- A mixed response can apply an approved deletion and another approved correction;
|
||||
an empty replacement does not invalidate the whole response.
|
||||
- A proposal that would blank its stage-input segment is filtered independently,
|
||||
retains the existing reason, and does not prevent unrelated valid candidates
|
||||
from succeeding. A proposal made unsafe only by an earlier accepted edit is
|
||||
skipped without adding an error or filtering recovery; a blank result is still
|
||||
prevented and the conflict remains visible in debug evidence.
|
||||
- A uniquely matching whitespace-only original span can reach validation and
|
||||
application. Empty originals remain invalid, ambiguous matches remain subject
|
||||
to module policy, and whitespace-only resulting segments remain disallowed.
|
||||
- Protected-term and meaning-changing deletions remain subject to their normal
|
||||
rejection rules. No candidate bypasses required validation.
|
||||
- A deletion needing no recovery counts as an ordinary applied correction; actual
|
||||
repair/filtering and editorial rejection retain their existing classifications.
|
||||
Debug evidence represents the empty replacement faithfully.
|
||||
- Schema and domain tests distinguish an empty string from missing/non-string
|
||||
replacement values and distinguish empty replacement from blank resulting text.
|
||||
- Prompt guidance and validator naming express the same contract as the code.
|
||||
Tests protect outcomes rather than exact prompt prose or incidental formatting.
|
||||
|
||||
Use deterministic, offline controlled responses for behavioral integration tests;
|
||||
no live-provider or paid evaluation is a prerequisite. Representative production
|
||||
examples can inform prompt quality without expanding this into an evaluation
|
||||
framework or guaranteeing that models always follow the instructions.
|
||||
|
||||
## Deferred Work
|
||||
|
||||
[Whole-segment removal](future.md#whole-segment-removal) is only potential future
|
||||
work and first requires deciding whether its benefit warrants implementation.
|
||||
Domain-aware regeneration and independent whole-pipeline traversal also remain
|
||||
in [future.md](future.md). This feature changes neither retry budgets nor the
|
||||
Audita/PromptKit ownership boundary and requires no upstream library feature.
|
||||
|
||||
Implementation sequencing and concrete names belong to the
|
||||
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md).
|
||||
The target remains consistent deletion of spans within surviving segments, with
|
||||
no new operator controls or report types.
|
||||
15
examples/configkit-inheritance.yml
Normal file
15
examples/configkit-inheritance.yml
Normal file
@@ -0,0 +1,15 @@
|
||||
version: 2
|
||||
profiles_dir: profiles/local
|
||||
backends:
|
||||
local-base:
|
||||
endpoint: http://127.0.0.1:11434/v1
|
||||
concurrency: 4
|
||||
queue_capacity: 0
|
||||
local:
|
||||
extends: local-base
|
||||
concurrency: 2
|
||||
llm:
|
||||
proposal_profile: local-proposal
|
||||
validation_profile: local-validation
|
||||
pipeline:
|
||||
modules: [grammar]
|
||||
@@ -1,4 +1,5 @@
|
||||
version: 2
|
||||
profiles_dir: profiles/local
|
||||
backends:
|
||||
local:
|
||||
endpoint: http://127.0.0.1:11434/v1
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
version: 2
|
||||
profiles_dir: profiles/service
|
||||
backends:
|
||||
service:
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
id: local-proposal
|
||||
backend: local
|
||||
model: replace-with-local-model
|
||||
temperature: 0
|
||||
max_tokens: 2048
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
@@ -1,7 +0,0 @@
|
||||
id: local-validation
|
||||
backend: local
|
||||
model: replace-with-local-model
|
||||
temperature: 0
|
||||
max_tokens: 1024
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
8
examples/profiles/local/local-proposal.yaml
Normal file
8
examples/profiles/local/local-proposal.yaml
Normal file
@@ -0,0 +1,8 @@
|
||||
profiles:
|
||||
local-proposal:
|
||||
backend: local
|
||||
model: replace-with-local-model
|
||||
temperature: 0
|
||||
max_tokens: 2048
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
8
examples/profiles/local/local-validation.yaml
Normal file
8
examples/profiles/local/local-validation.yaml
Normal file
@@ -0,0 +1,8 @@
|
||||
profiles:
|
||||
local-validation:
|
||||
backend: local
|
||||
model: replace-with-local-model
|
||||
temperature: 0
|
||||
max_tokens: 1024
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
@@ -1,7 +0,0 @@
|
||||
id: service-proposal-testing
|
||||
backend: service
|
||||
model: replace-with-testing-model
|
||||
temperature: 0
|
||||
max_tokens: 2048
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
@@ -1,7 +0,0 @@
|
||||
id: service-proposal
|
||||
backend: service
|
||||
model: replace-with-provider-model
|
||||
temperature: 0
|
||||
max_tokens: 2048
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
@@ -1,7 +0,0 @@
|
||||
id: service-validation-testing
|
||||
backend: service
|
||||
model: replace-with-testing-model
|
||||
temperature: 0
|
||||
max_tokens: 1024
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
@@ -1,7 +0,0 @@
|
||||
id: service-validation
|
||||
backend: service
|
||||
model: replace-with-provider-model
|
||||
temperature: 0
|
||||
max_tokens: 1024
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
8
examples/profiles/service/service-proposal-testing.yaml
Normal file
8
examples/profiles/service/service-proposal-testing.yaml
Normal file
@@ -0,0 +1,8 @@
|
||||
profiles:
|
||||
service-proposal-testing:
|
||||
backend: service
|
||||
model: replace-with-testing-model
|
||||
temperature: 0
|
||||
max_tokens: 2048
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
8
examples/profiles/service/service-proposal.yaml
Normal file
8
examples/profiles/service/service-proposal.yaml
Normal file
@@ -0,0 +1,8 @@
|
||||
profiles:
|
||||
service-proposal:
|
||||
backend: service
|
||||
model: replace-with-provider-model
|
||||
temperature: 0
|
||||
max_tokens: 2048
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
@@ -0,0 +1,8 @@
|
||||
profiles:
|
||||
service-validation-testing:
|
||||
backend: service
|
||||
model: replace-with-testing-model
|
||||
temperature: 0
|
||||
max_tokens: 1024
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
8
examples/profiles/service/service-validation.yaml
Normal file
8
examples/profiles/service/service-validation.yaml
Normal file
@@ -0,0 +1,8 @@
|
||||
profiles:
|
||||
service-validation:
|
||||
backend: service
|
||||
model: replace-with-provider-model
|
||||
temperature: 0
|
||||
max_tokens: 1024
|
||||
top_p: 1
|
||||
timeout_seconds: 120
|
||||
8
go.mod
8
go.mod
@@ -3,14 +3,14 @@ module gitea.maximumdirect.net/eric/audita
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2
|
||||
gitea.maximumdirect.net/eric/promptkit v0.11.2
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
|
||||
gitea.maximumdirect.net/eric/promptkit v0.12.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 // indirect
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 // indirect
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 // indirect
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
|
||||
16
go.sum
16
go.sum
@@ -1,11 +1,11 @@
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2 h1:ZMxmv8pOnLtAx2hFTf8Z+pHKUbR89xErkBhwjr4yK24=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.11.2 h1:2RqI+KtflvSh7vLQaTAQCmMaFQscEA7EPwPazyvXRoY=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.11.2/go.mod h1:onm/FBPUKQ8/NSPwTEk26Ut+dVX2iXcq8vEseB+g/0E=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 h1:wdab+R+lgMZxzYwhbAuCrW7zdSQ1cW9ovBR6UzLmZR4=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 h1:baCCIB6Ee5t0S4Zc++NTBIPcJEv5UPS6DlSrpZfykwg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.12.0 h1:oIFWEg6GPxtmX3Plqv0euP+FnK1LCk/t4xDy6oKCpP4=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.12.0/go.mod h1:pFOOy5v1zH17stEi9rmHYmfbuLjnfVMcUhfmk+4VtJY=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
|
||||
@@ -24,6 +24,7 @@ type configInput struct {
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
ReportDirSet bool
|
||||
Overrides []ck.Override
|
||||
}
|
||||
|
||||
@@ -124,6 +125,9 @@ func parseConfigInput(command string, args []string, output io.Writer) (configIn
|
||||
if f.Name == "config" {
|
||||
input.ConfigPathSet = true
|
||||
}
|
||||
if f.Name == "report-dir" {
|
||||
input.ReportDirSet = true
|
||||
}
|
||||
path, ok := settingPaths[f.Name]
|
||||
if !ok {
|
||||
return
|
||||
@@ -208,15 +212,22 @@ func configInputError(err error) string {
|
||||
return strings.Join(messages, "; ")
|
||||
}
|
||||
|
||||
func selectedReportDir(cfg config.Config, outputPath string) (string, error) {
|
||||
func selectedReportDir(cfg config.Config, outputPath string, reportDirSet bool) (string, error) {
|
||||
if !cfg.Reporting.Enabled {
|
||||
return "", nil
|
||||
}
|
||||
var reportDir string
|
||||
if outputPath != "" {
|
||||
return filepath.Dir(outputPath), nil
|
||||
if reportDirSet {
|
||||
reportDir = cfg.Reporting.StdoutDir
|
||||
} else {
|
||||
reportDir = filepath.Dir(outputPath)
|
||||
}
|
||||
} else {
|
||||
reportDir = cfg.Reporting.StdoutDir
|
||||
}
|
||||
if strings.TrimSpace(cfg.Reporting.StdoutDir) == "" {
|
||||
return "", fmt.Errorf("report directory is required when reporting is enabled and transcript output is stdout")
|
||||
if strings.TrimSpace(reportDir) == "" {
|
||||
return "", fmt.Errorf("report directory is required when reporting is enabled for the selected destination")
|
||||
}
|
||||
return cfg.Reporting.StdoutDir, nil
|
||||
return reportDir, nil
|
||||
}
|
||||
|
||||
@@ -215,22 +215,42 @@ func TestReportingAndDebugOverridesRemainIndependent(t *testing.T) {
|
||||
t.Fatalf("%s did not honor explicit false: %+v", command, selected.Result.Config)
|
||||
}
|
||||
}
|
||||
processInput, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--config", main, "--overlay", "testing"}, io.Discard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
processConfig, err := loadConfigInput(processInput, "process")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := selectedReportDir(processConfig.Result.Config, filepath.Join("out", "corrected.json"), processInput.ReportDirSet); err != nil || got != "out" {
|
||||
t.Fatalf("overlay stdout directory redirected file output: %q %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportDirectorySelectionOnlyUsesActiveDestination(t *testing.T) {
|
||||
cfg := config.Config{Reporting: config.Reporting{Enabled: true, StdoutDir: "reports"}}
|
||||
if got, err := selectedReportDir(cfg, ""); err != nil || got != "reports" {
|
||||
if got, err := selectedReportDir(cfg, "", false); err != nil || got != "reports" {
|
||||
t.Fatalf("stdout destination: %q %v", got, err)
|
||||
}
|
||||
cfg.Reporting.StdoutDir = ""
|
||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json")); err != nil || got != "out" {
|
||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), false); err != nil || got != "out" {
|
||||
t.Fatalf("file destination ignored stdout setting: %q %v", got, err)
|
||||
}
|
||||
if _, err := selectedReportDir(cfg, ""); err == nil {
|
||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err != nil || got != "reports" {
|
||||
t.Fatalf("explicit file report directory: %q %v", got, err)
|
||||
}
|
||||
cfg.Reporting.StdoutDir = ""
|
||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), false); err != nil || got != "out" {
|
||||
t.Fatalf("inactive blank configured report directory rejected for file output: %q %v", got, err)
|
||||
}
|
||||
if _, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err == nil {
|
||||
t.Fatal("active explicit blank report directory accepted for file output")
|
||||
}
|
||||
if _, err := selectedReportDir(cfg, "", false); err == nil {
|
||||
t.Fatal("enabled stdout reporting accepted blank destination")
|
||||
}
|
||||
cfg.Reporting.Enabled = false
|
||||
if got, err := selectedReportDir(cfg, ""); err != nil || got != "" {
|
||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err != nil || got != "" {
|
||||
t.Fatalf("disabled reporting probed destination: %q %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -247,4 +267,14 @@ func TestBareBooleanFlagsAndReportDirectoryOverride(t *testing.T) {
|
||||
if values[ck.Keys("reporting", "enabled").String()] != true || values[ck.Keys("diagnostics", "debug").String()] != true || values[ck.Keys("reporting", "stdout_dir").String()] != "reports" {
|
||||
t.Fatalf("incorrect visited overrides: %+v", values)
|
||||
}
|
||||
if !input.ReportDirSet {
|
||||
t.Fatal("explicit --report-dir presence was not recorded")
|
||||
}
|
||||
withoutReportDir, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml"}, io.Discard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if withoutReportDir.ReportDirSet {
|
||||
t.Fatal("absent --report-dir was recorded as explicit")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
||||
return 2
|
||||
}
|
||||
cfg := selected.Result.Config
|
||||
reportDir, err := selectedReportDir(cfg, input.OutputPath)
|
||||
reportDir, err := selectedReportDir(cfg, input.OutputPath, input.ReportDirSet)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
||||
return 2
|
||||
@@ -87,7 +87,7 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
||||
protected = append(protected, source.Path)
|
||||
}
|
||||
}
|
||||
protected = append(protected, runtime.ProfilePaths...)
|
||||
protected = append(protected, runtime.SourcePaths...)
|
||||
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{InputTranscriptPath: input.TranscriptPath, TranscriptPath: input.OutputPath,
|
||||
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
|
||||
if err != nil {
|
||||
|
||||
@@ -97,7 +97,7 @@ func runConfigValidate(ctx context.Context, args []string, stdout, stderr io.Wri
|
||||
}
|
||||
if _, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %s\n", llm.RedactError(err, selected.Result.Config))
|
||||
if llm.IsAssetError(err) {
|
||||
if ctx.Err() != nil || llm.IsAssetError(err) {
|
||||
return 1
|
||||
}
|
||||
return 2
|
||||
@@ -124,7 +124,7 @@ func runConfigPrintEffective(ctx context.Context, args []string, stdout, stderr
|
||||
llmRuntime, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %s\n", llm.RedactError(err, selected.Result.Config))
|
||||
if llm.IsAssetError(err) {
|
||||
if ctx.Err() != nil || llm.IsAssetError(err) {
|
||||
return 1
|
||||
}
|
||||
return 2
|
||||
@@ -225,7 +225,7 @@ func writeProcessUsage(w io.Writer) {
|
||||
fmt.Fprintf(w, " --%s\n", name)
|
||||
}
|
||||
fmt.Fprintln(w, " --reporting[=true|false] Write the three-file report bundle (default true)")
|
||||
fmt.Fprintln(w, " --report-dir DIR Report directory for stdout output only; ignored with --output or --reporting=false")
|
||||
fmt.Fprintln(w, " --report-dir DIR Select the report directory (stdout setting for stdout; output parent by default for files)")
|
||||
fmt.Fprintln(w, " --debug[=true|false] Persist detailed diagnostics (default false)")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Example:")
|
||||
|
||||
76
internal/cli/runtime_errors_test.go
Normal file
76
internal/cli/runtime_errors_test.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
profiles := filepath.Join(dir, "profiles")
|
||||
if err := os.Mkdir(profiles, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
configText := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n"
|
||||
if err := os.WriteFile(main, []byte(configText), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profiles, "operator.yml"), []byte(operatorDocument), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return main
|
||||
}
|
||||
|
||||
func TestConfigCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
|
||||
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
|
||||
definitions := map[string]string{
|
||||
"malformed operator YAML": "profiles:\n proposal: [\n",
|
||||
"invalid operator template": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content: '{{'\n",
|
||||
"missing operator message resource": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content_file: missing-instruction.md\n",
|
||||
}
|
||||
commands := []struct {
|
||||
name string
|
||||
args []string
|
||||
}{
|
||||
{name: "validate", args: []string{"config", "validate"}},
|
||||
{name: "print effective", args: []string{"config", "print-effective", "--view", "profiles"}},
|
||||
}
|
||||
for definition, document := range definitions {
|
||||
for _, command := range commands {
|
||||
t.Run(definition+"/"+command.name, func(t *testing.T) {
|
||||
main := writeRuntimeErrorConfig(t, document)
|
||||
args := append(append([]string(nil), command.args...), "--config", main)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := Run(context.Background(), args, &stdout, &stderr); code != 2 {
|
||||
t.Fatalf("exit=%d, stdout=%q, stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
|
||||
t.Fatalf("configured secret leaked: %q", stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigCommandsKeepCallerCancellationAsRuntimeExit(t *testing.T) {
|
||||
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
|
||||
main := writeRuntimeErrorConfig(t, "profiles:\n proposal:\n backend: offline\n model: local-model\n")
|
||||
commands := [][]string{
|
||||
{"config", "validate", "--config", main},
|
||||
{"config", "print-effective", "--config", main, "--view", "profiles"},
|
||||
}
|
||||
for _, args := range commands {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := Run(ctx, args, &stdout, &stderr); code != 1 {
|
||||
t.Fatalf("args=%v exit=%d, stdout=%q, stderr=%q", args, code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -137,9 +137,167 @@ func TestSurvivingCredentials(t *testing.T) {
|
||||
if err != nil || loaded.Config.Backends["other"].Credential != nil {
|
||||
t.Fatalf("null credential: %+v %v", loaded, err)
|
||||
}
|
||||
|
||||
inherited := `version: 2
|
||||
backends:
|
||||
parent:
|
||||
endpoint: https://parent.invalid/v1
|
||||
credential: BASE_SECRET
|
||||
local:
|
||||
extends: parent
|
||||
credential: null
|
||||
llm:
|
||||
proposal_profile: proposal
|
||||
`
|
||||
if _, err := loadText(inherited, ""); err == nil {
|
||||
t.Fatal("null child credential removed the unselected parent's surviving reference")
|
||||
}
|
||||
t.Setenv("BASE_SECRET", "synthetic-parent-secret")
|
||||
loaded, err = loadText(strings.Replace(inherited, " credential: null\n", "", 1), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := loaded.Config.Backends["local"].Credential.Value(); got != "synthetic-parent-secret" {
|
||||
t.Fatalf("inherited secret: %q", got)
|
||||
}
|
||||
if got := loaded.Metadata().IsExplicit(ck.Keys("backends", "local", "credential")); !got {
|
||||
t.Fatal("inherited credential is not explicit")
|
||||
}
|
||||
var inheritedEffective bytes.Buffer
|
||||
if err := loaded.WriteEffective(&inheritedEffective); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(inheritedEffective.String(), "synthetic-parent-secret") {
|
||||
t.Fatal("effective output leaked an inherited secret")
|
||||
}
|
||||
loaded, err = loadText(inherited, "")
|
||||
if err != nil || loaded.Config.Backends["local"].Credential != nil {
|
||||
t.Fatalf("child null credential: %+v %v", loaded, err)
|
||||
}
|
||||
if got := loaded.Config.Backends["parent"].Credential.Value(); got != "synthetic-parent-secret" {
|
||||
t.Fatalf("parent credential after child null: %q", got)
|
||||
}
|
||||
|
||||
t.Setenv("AUDITA_MODEL", "legacy-model")
|
||||
loaded, err = loadText(base, "")
|
||||
if err != nil || loaded.Config.LLM.ProposalProfile != "proposal" {
|
||||
t.Fatalf("legacy environment affected loading: %+v %v", loaded, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSiblingInheritancePreservesBackendSettings(t *testing.T) {
|
||||
text := `version: 2
|
||||
backends:
|
||||
base:
|
||||
endpoint: https://base.invalid/v1
|
||||
concurrency: 3
|
||||
queue_capacity: 0
|
||||
local:
|
||||
extends: base
|
||||
llm:
|
||||
proposal_profile: proposal
|
||||
`
|
||||
loaded, err := loadText(text, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend := loaded.Config.Backends["local"]
|
||||
if backend.Endpoint != "https://base.invalid/v1" || backend.Concurrency != 3 || backend.QueueCapacity != 0 {
|
||||
t.Fatalf("inherited backend settings: %+v", backend)
|
||||
}
|
||||
for _, path := range []ck.Path{
|
||||
ck.Keys("backends", "local", "endpoint"),
|
||||
ck.Keys("backends", "local", "concurrency"),
|
||||
ck.Keys("backends", "local", "queue_capacity"),
|
||||
} {
|
||||
if !loaded.Metadata().IsExplicit(path) {
|
||||
t.Errorf("inherited field %s is not explicit", path.String())
|
||||
}
|
||||
}
|
||||
|
||||
overridden, err := loadText(strings.Replace(text, " extends: base\n", " extends: base\n concurrency: 2\n", 1), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := overridden.Config.Backends["local"].Concurrency; got != 2 {
|
||||
t.Fatalf("child concurrency override: %d", got)
|
||||
}
|
||||
if got := overridden.Config.Backends["local"].QueueCapacity; got != 0 {
|
||||
t.Fatalf("inherited zero queue capacity: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSiblingInheritanceUsesSelectedOverlay(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
text := `version: 2
|
||||
backends:
|
||||
base:
|
||||
endpoint: https://base.invalid/v1
|
||||
concurrency: 3
|
||||
queue_capacity: 0
|
||||
local:
|
||||
extends: base
|
||||
concurrency: 2
|
||||
llm:
|
||||
proposal_profile: proposal
|
||||
`
|
||||
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("backends:\n base:\n endpoint: https://overlay.invalid/v1\n concurrency: 4\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := Load(ck.File(main), "testing")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
child := loaded.Config.Backends["local"]
|
||||
if child.Endpoint != "https://overlay.invalid/v1" || child.Concurrency != 2 || child.QueueCapacity != 0 {
|
||||
t.Fatalf("overlay inheritance and child override: %+v", child)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSiblingInheritanceDoesNotBypassBackendValidation(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
text string
|
||||
}{
|
||||
{
|
||||
name: "invalid inherited parent setting",
|
||||
text: `version: 2
|
||||
backends:
|
||||
base:
|
||||
endpoint: https://base.invalid/v1
|
||||
concurrency: 0
|
||||
local:
|
||||
extends: base
|
||||
llm:
|
||||
proposal_profile: proposal
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "invalid child override",
|
||||
text: `version: 2
|
||||
backends:
|
||||
base:
|
||||
endpoint: https://base.invalid/v1
|
||||
concurrency: 3
|
||||
local:
|
||||
extends: base
|
||||
concurrency: 0
|
||||
llm:
|
||||
proposal_profile: proposal
|
||||
`,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := loadText(tc.text, ""); err == nil {
|
||||
t.Fatal("invalid backend concurrency accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,9 +23,14 @@ import (
|
||||
)
|
||||
|
||||
func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
|
||||
t.Setenv("INHERITED_TEST_KEY", "synthetic-inherited-runtime-secret")
|
||||
t.Setenv("OPERATOR_BACKEND_KEY", "synthetic-operator-backend-secret")
|
||||
entered := make(chan struct{}, 3)
|
||||
release := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer synthetic-inherited-runtime-secret" {
|
||||
t.Errorf("inherited credential header: %q", got)
|
||||
}
|
||||
entered <- struct{}{}
|
||||
<-release
|
||||
content, _ := json.Marshal(`{"corrections":[]}`)
|
||||
@@ -37,16 +42,26 @@ func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
body := fmt.Sprintf("version: 2\nbackends:\n shared:\n endpoint: %s/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL)
|
||||
body := fmt.Sprintf("version: 2\nbackends:\n shared-base:\n endpoint: %s/v1\n credential: INHERITED_TEST_KEY\n concurrency: 1\n queue_capacity: 0\n shared:\n extends: shared-base\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL)
|
||||
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"proposal", "validation"} {
|
||||
profile := "id: " + name + "\nbackend: shared\nmodel: local\n"
|
||||
profile := "profiles:\n " + name + ":\n backend: shared\n model: local\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(profile), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
operatorBackend := `backends:
|
||||
shared:
|
||||
endpoint: https://operator.invalid/v1
|
||||
concurrency_limit: 77
|
||||
queue_capacity: 88
|
||||
api_key_env: OPERATOR_BACKEND_KEY
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "backend-tuning.yaml"), []byte(operatorBackend), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.File(main), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -171,7 +186,7 @@ func TestDistinctBackendsHaveIndependentCapacity(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, backend := range map[string]string{"proposal": "first", "validation": "second"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte("id: "+name+"\nbackend: "+backend+"\nmodel: local\n"), 0600); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte("profiles:\n "+name+":\n backend: "+backend+"\n model: local\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,22 @@ type reply struct {
|
||||
}
|
||||
|
||||
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) {
|
||||
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{
|
||||
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
|
||||
})
|
||||
}
|
||||
|
||||
func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) {
|
||||
documents := map[string]string{
|
||||
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
|
||||
}
|
||||
if operatorPrompt != "" {
|
||||
documents["operator-prompt.yaml"] = operatorPrompt
|
||||
}
|
||||
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents)
|
||||
}
|
||||
|
||||
func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) {
|
||||
t.Helper()
|
||||
var count atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -65,13 +81,21 @@ func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook fu
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
t.Setenv("ADAPTER_KEY", "adapter-secret")
|
||||
base := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\n credential: ADAPTER_KEY\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", server.URL, retries, repairs)
|
||||
dir := profileLayout(t, base, "")
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
profile := "id: base\nbackend: offline\nmodel: full-model\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil {
|
||||
dir := t.TempDir()
|
||||
profileDirectory := filepath.Join(dir, "profiles")
|
||||
if err := os.Mkdir(profileDirectory, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs)
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, content := range documents {
|
||||
if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.File(main), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -83,6 +107,10 @@ func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook fu
|
||||
return runtime, &count, dir
|
||||
}
|
||||
|
||||
func operatorPromptDocument(system, user string) string {
|
||||
return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user)
|
||||
}
|
||||
|
||||
func adapterRequest(schema responseschema.Key) structuredoutput.Request {
|
||||
id := prompts.PromptIDModuleGlossaryProposal
|
||||
vars := map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}
|
||||
@@ -286,10 +314,238 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
|
||||
if !found {
|
||||
t.Fatal("hardening policy missing")
|
||||
}
|
||||
backend := runtime.Config.Backends["offline"]
|
||||
backend.Endpoint = "http://127.0.0.1:1/v1"
|
||||
runtime.Config.Backends["offline"] = backend
|
||||
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 {
|
||||
t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
system string
|
||||
user string
|
||||
replies []reply
|
||||
repairs int
|
||||
wantSystem string
|
||||
wantUser string
|
||||
}{
|
||||
{
|
||||
name: "operator text without shared references",
|
||||
system: "Operator system instructions.",
|
||||
user: "Operator request without variables.",
|
||||
replies: []reply{{200, `{"wrong":true}`}, {200, `{"corrections":[]}`}},
|
||||
repairs: 1,
|
||||
wantSystem: "Operator system instructions.",
|
||||
wantUser: "Operator request without variables.",
|
||||
},
|
||||
{
|
||||
name: "operator text with shared references",
|
||||
system: "Policy: {{.HardeningPolicy}}",
|
||||
user: "Section: {{.SectionJSON}}",
|
||||
replies: []reply{{200, `{"corrections":[]}`}},
|
||||
wantSystem: "Policy: " + prompts.HardeningText(),
|
||||
wantUser: "Section: []",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var posted []struct {
|
||||
Messages []structuredoutput.Message `json:"messages"`
|
||||
}
|
||||
runtime, count, _ := adapterFixtureWithPrompt(t, tc.replies, 0, tc.repairs, func(r *http.Request) {
|
||||
var envelope struct {
|
||||
Messages []structuredoutput.Message `json:"messages"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
posted = append(posted, envelope)
|
||||
}, operatorPromptDocument(tc.system, tc.user))
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := adapterRequest(responseschema.CorrectionSetKey)
|
||||
if tc.name == "operator text with shared references" {
|
||||
req.Variables["HardeningPolicy"] = "attacker-controlled"
|
||||
}
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), req, &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if int(count.Load()) != len(tc.replies) || response.Execution.StructuralRepairs != tc.repairs {
|
||||
t.Fatalf("schema/repair execution = %d calls, %+v; want %d calls and %d repair", count.Load(), response.Execution, len(tc.replies), tc.repairs)
|
||||
}
|
||||
if response.Prepared == nil || len(posted) == 0 || len(posted[0].Messages) != len(response.Prepared.Messages) {
|
||||
t.Fatal("prepared operator prompt was not dispatched")
|
||||
}
|
||||
for i := range response.Prepared.Messages {
|
||||
if response.Prepared.Messages[i] != posted[0].Messages[i] {
|
||||
t.Fatal("prepared operator messages differ from the first dispatched request")
|
||||
}
|
||||
}
|
||||
if len(response.Prepared.Messages) != 2 || response.Prepared.Messages[0].Content != tc.wantSystem || response.Prepared.Messages[1].Content != tc.wantUser {
|
||||
t.Fatalf("prepared messages = %+v", response.Prepared.Messages)
|
||||
}
|
||||
if tc.name == "operator text with shared references" {
|
||||
if strings.Contains(response.Prepared.Messages[0].Content, "attacker-controlled") || !strings.Contains(response.Prepared.Messages[0].Content, prompts.HardeningText()) {
|
||||
t.Fatal("caller variables replaced the reserved shared value")
|
||||
}
|
||||
} else if strings.Contains(response.Prepared.Messages[0].Content+response.Prepared.Messages[1].Content, prompts.HardeningText()) {
|
||||
t.Fatal("shared text was injected into operator messages that omit its reference")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) {
|
||||
t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used")
|
||||
cases := []struct {
|
||||
name string
|
||||
parentTemperature string
|
||||
childTemperature string
|
||||
wantTemperature bool
|
||||
}{
|
||||
{name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true},
|
||||
{name: "omitted", wantTemperature: false},
|
||||
{name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature +
|
||||
" base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature +
|
||||
" extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n"
|
||||
operatorBackend := `backends:
|
||||
openrouter:
|
||||
endpoint: https://operator.invalid/v1
|
||||
concurrency_limit: 91
|
||||
queue_capacity: 92
|
||||
api_key_env: CATALOG_TRAP_KEY
|
||||
extra_params:
|
||||
routing:
|
||||
inherited: backend
|
||||
overridden: backend
|
||||
cleared: remove-me
|
||||
`
|
||||
documents := map[string]string{
|
||||
"unrelated-profile-file.yml": profileDocument,
|
||||
"provider-tuning.yaml": operatorBackend,
|
||||
}
|
||||
var wire map[string]json.RawMessage
|
||||
runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" {
|
||||
t.Errorf("application credential was not authoritative: %q", got)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&wire); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}, "openrouter", documents)
|
||||
profile := runtime.Profiles["proposal"]
|
||||
if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 {
|
||||
t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams)
|
||||
}
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count.Load() != 1 {
|
||||
t.Fatalf("provider calls = %d, want 1", count.Load())
|
||||
}
|
||||
temperature, hasTemperature := wire["temperature"]
|
||||
if hasTemperature != tc.wantTemperature {
|
||||
t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire)
|
||||
}
|
||||
if tc.wantTemperature {
|
||||
var value float64
|
||||
if err := json.Unmarshal(temperature, &value); err != nil || value != 0 {
|
||||
t.Fatalf("explicit zero temperature = %s (%v)", temperature, err)
|
||||
}
|
||||
}
|
||||
var routing map[string]any
|
||||
if err := json.Unmarshal(wire["routing"], &routing); err != nil {
|
||||
t.Fatalf("provider routing fields missing: %s (%v)", wire, err)
|
||||
}
|
||||
if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" {
|
||||
t.Fatalf("recursively overlaid provider fields = %#v", routing)
|
||||
}
|
||||
if value, exists := routing["cleared"]; !exists || value != nil {
|
||||
t.Fatalf("null did not replace inherited provider field: %#v", routing)
|
||||
}
|
||||
if response.Model != "full-model" || response.Prepared == nil {
|
||||
t.Fatalf("execution did not use selected profile: %+v", response)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) {
|
||||
var posted []struct {
|
||||
Model string `json:"model"`
|
||||
Messages []structuredoutput.Message `json:"messages"`
|
||||
}
|
||||
runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
|
||||
var envelope struct {
|
||||
Model string `json:"model"`
|
||||
Messages []structuredoutput.Message `json:"messages"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
posted = append(posted, envelope)
|
||||
}, operatorPromptDocument("Initial operator instructions.", "Initial payload."))
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := "profiles:\n base:\n backend: offline\n model: edited-model\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.")
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" {
|
||||
t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0])
|
||||
}
|
||||
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" {
|
||||
t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
|
||||
runtime, count, dir := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
|
||||
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -300,15 +556,6 @@ func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
|
||||
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || count.Load() != 0 {
|
||||
t.Fatalf("preparation failure dispatched: %v, %d", err, count.Load())
|
||||
}
|
||||
// PrepareExecution reloads the profile, so a changed target must be blocked.
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nendpoint: http://127.0.0.1:1/v1\nmodel: full-model\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req = adapterRequest(responseschema.CorrectionSetKey)
|
||||
_, err = client.CompleteStructured(context.Background(), req, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 0 {
|
||||
t.Fatalf("target drift dispatched: %v, %d", err, count.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
|
||||
@@ -333,7 +580,9 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
|
||||
|
||||
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
|
||||
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
|
||||
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
|
||||
runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{
|
||||
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n",
|
||||
})
|
||||
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
|
||||
@@ -6,13 +6,11 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/redaction"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
@@ -26,9 +24,9 @@ type Runtime struct {
|
||||
Config config.Config
|
||||
Engine *pk.Engine
|
||||
Profiles map[string]pk.ProfileInspection
|
||||
// ProfilePaths contains filesystem sources actually read by PromptKit.
|
||||
ProfilePaths []string
|
||||
nextCallID atomic.Uint64
|
||||
// SourcePaths contains host configuration and message files captured by PromptKit.
|
||||
SourcePaths []string
|
||||
nextCallID atomic.Uint64
|
||||
}
|
||||
|
||||
// AssetError indicates a broken embedded prompt or schema rather than an
|
||||
@@ -39,32 +37,63 @@ func (e *AssetError) Error() string { return e.Err.Error() }
|
||||
func (e *AssetError) Unwrap() error { return e.Err }
|
||||
|
||||
func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) {
|
||||
return newRuntimeWithPromptFS(ctx, c, mainPath, client, prompts.Filesystem())
|
||||
}
|
||||
|
||||
func newRuntimeWithPromptFS(ctx context.Context, c config.Config, mainPath string, client *http.Client, promptFS fs.FS) (*Runtime, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
profileDir := c.ProfilesDir
|
||||
if !filepath.IsAbs(profileDir) {
|
||||
profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
|
||||
}
|
||||
profileFS := &profileSourceFS{FS: os.DirFS(profileDir), paths: make(map[string]struct{})}
|
||||
options := []pk.Option{
|
||||
pk.WithPromptFS(prompts.Filesystem(), prompts.AssetRoot),
|
||||
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
|
||||
pk.WithProfileFS(profileFS, "."),
|
||||
}
|
||||
ids := make([]string, 0, len(c.Backends))
|
||||
for id := range c.Backends {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
backend := c.Backends[id]
|
||||
options = append(options, pk.WithBackend(pk.Backend{ID: id, Endpoint: backend.Endpoint, ConcurrencyLimit: backend.Concurrency, QueueCapacity: &backend.QueueCapacity}))
|
||||
}
|
||||
engine, err := pk.NewEngine(pk.Config{HTTPClient: client, TransientRetries: &c.LLM.TransientRetries}, options...)
|
||||
promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("assemble LLM runtime: %w", err)
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return nil, ctxErr
|
||||
}
|
||||
return nil, &AssetError{Err: fmt.Errorf("discover embedded prompts: %w", err)}
|
||||
}
|
||||
profileInputs, err := pk.DirectoryInputs(profileDir)
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return nil, ctxErr
|
||||
}
|
||||
return nil, fmt.Errorf("discover operator definitions: %w", err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
applicationBackends := make(map[string]any, len(c.Backends))
|
||||
for id := range c.Backends {
|
||||
backend := c.Backends[id]
|
||||
applicationBackends[id] = map[string]any{
|
||||
"endpoint": backend.Endpoint,
|
||||
"concurrency_limit": backend.Concurrency,
|
||||
"queue_capacity": backend.QueueCapacity,
|
||||
"api_key_env": nil,
|
||||
}
|
||||
}
|
||||
sources := pk.ConfigurationSources{
|
||||
Underlays: []ck.Underlay{
|
||||
{Inputs: promptInputs},
|
||||
{Inputs: profileInputs},
|
||||
},
|
||||
Main: &ck.Input{Source: ck.Mapping("application backends", map[string]any{"backends": applicationBackends})},
|
||||
}
|
||||
engine, err := pk.NewEngine(pk.Config{
|
||||
Sources: sources,
|
||||
HTTPClient: client,
|
||||
TransientRetries: &c.LLM.TransientRetries,
|
||||
}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
|
||||
if err != nil {
|
||||
return nil, embeddedAssetOrInputError(ctx, promptInputs, fmt.Errorf("assemble LLM runtime: %w", err))
|
||||
}
|
||||
for _, meta := range prompts.RegisteredMetadata() {
|
||||
if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil {
|
||||
return nil, &AssetError{Err: fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, err)}
|
||||
cause := fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err)
|
||||
return nil, embeddedAssetOrInputError(ctx, promptInputs, cause)
|
||||
}
|
||||
}
|
||||
runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)}
|
||||
@@ -78,6 +107,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s profile must select a registered application backend", selection.role)
|
||||
}
|
||||
if strings.TrimSpace(target.Model) == "" {
|
||||
return nil, fmt.Errorf("%s profile must select a nonblank model", selection.role)
|
||||
}
|
||||
if strings.TrimSpace(target.Endpoint) == "" {
|
||||
return nil, fmt.Errorf("%s profile must resolve to a nonblank endpoint", selection.role)
|
||||
}
|
||||
endpoint := func(value string) string { return strings.TrimRight(strings.TrimSpace(value), "/") }
|
||||
if endpoint(target.Endpoint) != endpoint(backend.Endpoint) {
|
||||
return nil, fmt.Errorf("%s profile endpoint must match its registered backend", selection.role)
|
||||
@@ -90,15 +125,39 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
|
||||
}
|
||||
runtime.Profiles[selection.role] = *profile
|
||||
}
|
||||
profileFS.mu.Lock()
|
||||
for name := range profileFS.paths {
|
||||
runtime.ProfilePaths = append(runtime.ProfilePaths, filepath.Join(profileDir, filepath.FromSlash(name)))
|
||||
runtime.SourcePaths, err = capturedSourcePaths(engine)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect captured LLM sources: %w", err)
|
||||
}
|
||||
profileFS.mu.Unlock()
|
||||
sort.Strings(runtime.ProfilePaths)
|
||||
return runtime, nil
|
||||
}
|
||||
|
||||
func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cause error) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
embeddedSources := pk.ConfigurationSources{
|
||||
Underlays: []ck.Underlay{{Inputs: promptInputs}},
|
||||
}
|
||||
diagnosticEngine, err := pk.NewEngine(pk.Config{Sources: embeddedSources},
|
||||
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
|
||||
if err == nil {
|
||||
for _, meta := range prompts.RegisteredMetadata() {
|
||||
if _, inspectErr := diagnosticEngine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); inspectErr != nil {
|
||||
err = fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, inspectErr)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
if err != nil {
|
||||
return &AssetError{Err: fmt.Errorf("validate embedded prompt assets: %w", err)}
|
||||
}
|
||||
return cause
|
||||
}
|
||||
|
||||
func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) }
|
||||
|
||||
// RedactError replaces every configured resolved backend secret before a
|
||||
@@ -109,21 +168,3 @@ func RedactError(err error, c config.Config) string {
|
||||
}
|
||||
return redaction.New(ConfiguredSecrets(&c)).String(err.Error())
|
||||
}
|
||||
|
||||
// Record successful profile reads at the library boundary, including inherited
|
||||
// and differently named sources, without duplicating PromptKit's discovery rules.
|
||||
type profileSourceFS struct {
|
||||
fs.FS
|
||||
mu sync.Mutex
|
||||
paths map[string]struct{}
|
||||
}
|
||||
|
||||
func (f *profileSourceFS) ReadFile(name string) ([]byte, error) {
|
||||
data, err := fs.ReadFile(f.FS, name)
|
||||
if err == nil {
|
||||
f.mu.Lock()
|
||||
f.paths[name] = struct{}{}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
return data, err
|
||||
}
|
||||
|
||||
257
internal/framework/llm/runtime_assembly_test.go
Normal file
257
internal/framework/llm/runtime_assembly_test.go
Normal file
@@ -0,0 +1,257 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
profileDir := filepath.Join(dir, "profiles")
|
||||
if err := os.Mkdir(profileDir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections
|
||||
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, content := range documents {
|
||||
if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.File(main), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return loaded.Config, main
|
||||
}
|
||||
|
||||
func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
|
||||
t.Helper()
|
||||
source := prompts.Filesystem()
|
||||
files := make(fstest.MapFS)
|
||||
err := fs.WalkDir(source, prompts.AssetRoot, func(name string, entry fs.DirEntry, err error) error {
|
||||
if err != nil || entry.IsDir() {
|
||||
return err
|
||||
}
|
||||
data, err := fs.ReadFile(source, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files[name] = &fstest.MapFile{Data: data}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) {
|
||||
validProposal := " backend: offline\n model: proposal-model\n"
|
||||
cases := []struct {
|
||||
name string
|
||||
selections string
|
||||
profiles string
|
||||
role string
|
||||
field string
|
||||
}{
|
||||
{
|
||||
name: "omitted proposal model with derived validation selection",
|
||||
selections: " proposal_profile: proposal\n",
|
||||
profiles: "profiles:\n proposal:\n backend: offline\n",
|
||||
role: "proposal",
|
||||
field: "model",
|
||||
},
|
||||
{
|
||||
name: "empty validation model",
|
||||
selections: " proposal_profile: proposal\n validation_profile: validation\n",
|
||||
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n",
|
||||
role: "validation",
|
||||
field: "model",
|
||||
},
|
||||
{
|
||||
name: "null proposal model",
|
||||
selections: " proposal_profile: proposal\n",
|
||||
profiles: "profiles:\n proposal:\n backend: offline\n model: null\n",
|
||||
role: "proposal",
|
||||
field: "model",
|
||||
},
|
||||
{
|
||||
name: "cleared validation endpoint",
|
||||
selections: " proposal_profile: proposal\n validation_profile: validation\n",
|
||||
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n",
|
||||
role: "validation",
|
||||
field: "endpoint",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles})
|
||||
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||
if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) {
|
||||
t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n",
|
||||
})
|
||||
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" {
|
||||
t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProfileInheritancePreservesControlsAndMergesProviderValues(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: \"child profile\"\n", map[string]string{
|
||||
"unrelated-name.yaml": `profiles:
|
||||
"parent~/shared":
|
||||
backend: offline
|
||||
temperature: 0.4
|
||||
top_p: 0.7
|
||||
extra_params:
|
||||
routing:
|
||||
retained: from-parent
|
||||
overridden: parent
|
||||
cleared: remove-me
|
||||
"child profile":
|
||||
extends: "parent~/shared"
|
||||
model: child-model
|
||||
extra_params:
|
||||
routing:
|
||||
overridden: child
|
||||
child_only: present
|
||||
cleared: null
|
||||
`,
|
||||
})
|
||||
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := runtime.Profiles["proposal"]
|
||||
target := profile.EffectiveModelParams
|
||||
if profile.ProfileID != "child profile" || target.Model != "child-model" || target.Temperature != 0.4 || target.TopP != 0.7 {
|
||||
t.Fatalf("inherited profile target = %+v", target)
|
||||
}
|
||||
routing, ok := target.ExtraParams["routing"].(map[string]any)
|
||||
if !ok || routing["retained"] != "from-parent" || routing["overridden"] != "child" || routing["child_only"] != "present" {
|
||||
t.Fatalf("recursively inherited provider values = %#v", target.ExtraParams)
|
||||
}
|
||||
if value, exists := routing["cleared"]; !exists || value != nil {
|
||||
t.Fatalf("null did not replace the inherited provider value: %#v", routing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
|
||||
})
|
||||
if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||
if err == nil || IsAssetError(err) {
|
||||
t.Fatalf("missing operator profile directory classification: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
document map[string]string
|
||||
selection string
|
||||
}{
|
||||
{
|
||||
name: "malformed unselected definition",
|
||||
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"},
|
||||
selection: " proposal_profile: proposal\n",
|
||||
},
|
||||
{
|
||||
name: "undefined selected backend",
|
||||
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"},
|
||||
selection: " proposal_profile: proposal\n",
|
||||
},
|
||||
{
|
||||
name: "conflicting directory contributions",
|
||||
document: map[string]string{
|
||||
"a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n",
|
||||
"b.yml": "profiles:\n proposal:\n model: second-model\n",
|
||||
},
|
||||
selection: " proposal_profile: proposal\n",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document)
|
||||
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||
if err == nil || IsAssetError(err) {
|
||||
t.Fatalf("operator definition classification: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
|
||||
_, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{})
|
||||
if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{})
|
||||
if !errors.Is(err, context.Canceled) || IsAssetError(err) {
|
||||
t.Fatalf("caller cancellation classification: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
|
||||
promptFS := copyEmbeddedPromptFilesystem(t)
|
||||
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
|
||||
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
|
||||
})
|
||||
_, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
|
||||
if err == nil || !IsAssetError(err) {
|
||||
t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
|
||||
promptFS := copyEmbeddedPromptFilesystem(t)
|
||||
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
|
||||
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||
"operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n",
|
||||
})
|
||||
runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
|
||||
if err != nil {
|
||||
t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err)
|
||||
}
|
||||
if runtime.Engine == nil {
|
||||
t.Fatal("runtime engine was not retained")
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type rejectingTransport struct{ t *testing.T }
|
||||
@@ -29,8 +30,8 @@ func profileLayout(t *testing.T, base, overlay string) string {
|
||||
}
|
||||
}
|
||||
for name, content := range map[string]string{
|
||||
"base": "id: base\nbackend: offline\nmodel: full-model\ntemperature: 0.3\nmax_tokens: 500\ntop_p: 0.8\ntimeout_seconds: 12\nservice_tier: priority\nreasoning_effort: medium\n",
|
||||
"testing": "id: testing\nbackend: offline\nmodel: economy-model\n",
|
||||
"base": "profiles:\n base:\n backend: offline\n model: full-model\n temperature: 0.3\n max_tokens: 500\n top_p: 0.8\n timeout_seconds: 12\n service_tier: priority\n reasoning_effort: medium\n",
|
||||
"testing": "profiles:\n testing:\n backend: offline\n model: economy-model\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -112,14 +113,111 @@ func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeViewsOmitProviderAndPromptValues(t *testing.T) {
|
||||
const providerValue = "private-provider-value"
|
||||
const promptValue = "private-operator-prompt-text"
|
||||
runtime, _, dir := adapterFixtureWithDocuments(t, nil, 0, 0, nil, "offline", map[string]string{
|
||||
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n extra_params:\n vendor_extension: " + providerValue + "\n",
|
||||
"operator-prompt.yaml": operatorPromptDocument(promptValue, promptValue),
|
||||
})
|
||||
profiles, err := runtime.SelectedProfilesYAML()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.File(filepath.Join(dir, "config.yml")), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
explain, err := ExplanationYAML(loaded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
combined := string(profiles) + string(explain)
|
||||
for _, value := range []string{providerValue, promptValue, "adapter-secret", "127.0.0.1"} {
|
||||
if strings.Contains(combined, value) {
|
||||
t.Fatalf("safe views exposed %q: %s", value, combined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplanationYAMLIncludesSafeInheritanceEvidence(t *testing.T) {
|
||||
t.Setenv("SYNTHETIC_KEY", "synthetic-inherited-secret-never-print")
|
||||
text := `version: 2
|
||||
backends:
|
||||
shared:
|
||||
endpoint: https://shared.invalid/v1
|
||||
concurrency: 3
|
||||
queue_capacity: 0
|
||||
credential: SYNTHETIC_KEY
|
||||
offline:
|
||||
extends: shared
|
||||
concurrency: 1
|
||||
llm:
|
||||
proposal_profile: base
|
||||
`
|
||||
loaded, err := config.Load(ck.Bytes("config.yml", []byte(text)), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
explanation, err := ExplanationYAML(loaded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(explanation), "synthetic-inherited-secret-never-print") {
|
||||
t.Fatal("explanation exposed a resolved inherited credential")
|
||||
}
|
||||
var records []struct {
|
||||
Path string `yaml:"path"`
|
||||
Inheritance []struct {
|
||||
From string `yaml:"from"`
|
||||
To string `yaml:"to"`
|
||||
Reference string `yaml:"reference"`
|
||||
} `yaml:"inheritance"`
|
||||
}
|
||||
if err := yaml.Unmarshal(explanation, &records); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var endpoint, credential, concurrency *struct {
|
||||
Path string `yaml:"path"`
|
||||
Inheritance []struct {
|
||||
From string `yaml:"from"`
|
||||
To string `yaml:"to"`
|
||||
Reference string `yaml:"reference"`
|
||||
} `yaml:"inheritance"`
|
||||
}
|
||||
for i := range records {
|
||||
switch records[i].Path {
|
||||
case "/backends/offline/endpoint":
|
||||
endpoint = &records[i]
|
||||
case "/backends/offline/credential":
|
||||
credential = &records[i]
|
||||
case "/backends/offline/concurrency":
|
||||
concurrency = &records[i]
|
||||
}
|
||||
}
|
||||
if endpoint == nil || len(endpoint.Inheritance) != 1 {
|
||||
t.Fatalf("inherited endpoint evidence missing: %s", explanation)
|
||||
}
|
||||
step := endpoint.Inheritance[0]
|
||||
if step.From != "/backends/shared/endpoint" || step.To != "/backends/offline/endpoint" || step.Reference != "/backends/offline/extends" {
|
||||
t.Fatalf("inherited endpoint evidence: %+v", step)
|
||||
}
|
||||
if credential == nil || len(credential.Inheritance) != 1 {
|
||||
t.Fatalf("inherited credential evidence missing: %s", explanation)
|
||||
}
|
||||
if concurrency == nil || len(concurrency.Inheritance) != 0 {
|
||||
t.Fatalf("direct child override reported parent inheritance: %+v", concurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) {
|
||||
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
|
||||
dir := profileLayout(t, profileBase, "")
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
for _, tc := range []struct{ name, profile string }{
|
||||
{"endpoint only", "id: custom\nendpoint: https://offline.invalid/v1\nmodel: model\n"},
|
||||
{"mismatched endpoint", "id: custom\nbackend: offline\nendpoint: https://elsewhere.invalid/v1\nmodel: model\n"},
|
||||
{"profile credential", "id: custom\nbackend: offline\nmodel: model\napi_key_env: OTHER_KEY\n"},
|
||||
{"endpoint only", "profiles:\n custom:\n endpoint: https://offline.invalid/v1\n model: model\n"},
|
||||
{"mismatched endpoint", "profiles:\n custom:\n backend: offline\n endpoint: https://elsewhere.invalid/v1\n model: model\n"},
|
||||
{"profile credential", "profiles:\n custom:\n backend: offline\n model: model\n api_key_env: OTHER_KEY\n"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil {
|
||||
|
||||
@@ -61,11 +61,17 @@ type derivationView struct {
|
||||
Reason string `yaml:"reason"`
|
||||
Dependencies []string `yaml:"dependencies,omitempty"`
|
||||
}
|
||||
type inheritanceStepView struct {
|
||||
From string `yaml:"from"`
|
||||
To string `yaml:"to"`
|
||||
Reference string `yaml:"reference"`
|
||||
}
|
||||
type explanationView struct {
|
||||
Path string `yaml:"path"`
|
||||
Source string `yaml:"source"`
|
||||
Layer ck.Layer `yaml:"layer"`
|
||||
Derivations []derivationView `yaml:"derivations,omitempty"`
|
||||
Path string `yaml:"path"`
|
||||
Source string `yaml:"source"`
|
||||
Layer ck.Layer `yaml:"layer"`
|
||||
Derivations []derivationView `yaml:"derivations,omitempty"`
|
||||
Inheritance []inheritanceStepView `yaml:"inheritance,omitempty"`
|
||||
}
|
||||
|
||||
// ExplanationYAML reports only origins and recorded resolution operations.
|
||||
@@ -93,6 +99,11 @@ func ExplanationYAML(result *ck.Result[config.Config]) ([]byte, error) {
|
||||
continue
|
||||
}
|
||||
record := explanationView{Path: path.String(), Layer: info.Origin.Layer, Source: string(info.Origin.Layer)}
|
||||
for _, step := range info.Origin.Inheritance {
|
||||
record.Inheritance = append(record.Inheritance, inheritanceStepView{
|
||||
From: step.From.String(), To: step.To.String(), Reference: step.Reference.String(),
|
||||
})
|
||||
}
|
||||
if len(info.Origin.Locations) > 0 {
|
||||
if label := sources[info.Origin.Locations[0].SourceID]; label != "" {
|
||||
record.Source = label
|
||||
|
||||
65
internal/framework/llm/source_paths.go
Normal file
65
internal/framework/llm/source_paths.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func capturedSourcePaths(engine *pk.Engine) ([]string, error) {
|
||||
inspection, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect configuration: %w", err)
|
||||
}
|
||||
|
||||
sources := inspection.Sources()
|
||||
byID := make(map[string]ck.SourceInfo, len(sources))
|
||||
paths := make(map[string]struct{}, len(sources))
|
||||
for _, source := range sources {
|
||||
byID[source.ID] = source
|
||||
if source.Kind == ck.SourceKindFile && source.Path != "" {
|
||||
paths[source.Path] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
metadata := inspection.Metadata()
|
||||
for _, path := range metadata.HistoricalPaths() {
|
||||
segments := strings.Split(strings.TrimPrefix(path.String(), "/"), "/")
|
||||
if len(segments) != 6 || segments[0] != "prompts" || segments[3] != "messages" || segments[5] != "content_file" {
|
||||
continue
|
||||
}
|
||||
if messageIndex, err := strconv.Atoi(segments[4]); err != nil || messageIndex < 0 {
|
||||
continue
|
||||
}
|
||||
field, ok := metadata.Lookup(path)
|
||||
if !ok || field.Null || len(field.Origin.Locations) != 1 {
|
||||
continue
|
||||
}
|
||||
source, ok := byID[field.Origin.Locations[0].SourceID]
|
||||
if !ok || source.Kind != ck.SourceKindFile || source.Path == "" {
|
||||
continue
|
||||
}
|
||||
explanation, ok := inspection.Explain(path)
|
||||
if !ok || !explanation.Effective.Present || explanation.Effective.Redacted {
|
||||
continue
|
||||
}
|
||||
var resource string
|
||||
if err := yaml.Unmarshal([]byte(explanation.Effective.YAML), &resource); err != nil {
|
||||
return nil, fmt.Errorf("decode captured message resource at %s: %w", path.String(), err)
|
||||
}
|
||||
paths[filepath.Join(filepath.Dir(source.Path), filepath.FromSlash(resource))] = struct{}{}
|
||||
}
|
||||
|
||||
result := make([]string, 0, len(paths))
|
||||
for path := range paths {
|
||||
result = append(result, path)
|
||||
}
|
||||
sort.Strings(result)
|
||||
return result, nil
|
||||
}
|
||||
61
internal/framework/llm/source_paths_test.go
Normal file
61
internal/framework/llm/source_paths_test.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRuntimeTracksCapturedConfigurationAndInheritedMessageFiles(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n validation_profile: validation\n", map[string]string{
|
||||
"selected.yml": "profiles:\n proposal:\n backend: offline\n model: proposal\n validation:\n backend: offline\n model: validation\n",
|
||||
})
|
||||
profiles := filepath.Join(filepath.Dir(main), cfg.ProfilesDir)
|
||||
parentDir := filepath.Join(profiles, "catalog")
|
||||
childDir := filepath.Join(profiles, "nested", "child")
|
||||
for _, directory := range []string{parentDir, childDir} {
|
||||
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
parent := filepath.Join(parentDir, "arbitrary-name.yaml")
|
||||
child := filepath.Join(childDir, "override.yaml")
|
||||
resource := filepath.Join(parentDir, "instruction.md")
|
||||
parentDocument := "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./instruction.md\n output:\n format: text\n validation_mode: basic\n"
|
||||
childDocument := "prompts:\n literal/~child:\n extends: source\n"
|
||||
for path, content := range map[string]string{
|
||||
parent: parentDocument,
|
||||
child: childDocument,
|
||||
resource: "Captured operator message.\n",
|
||||
} {
|
||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
filepath.Join(profiles, "nested", "child", "override.yaml"),
|
||||
filepath.Join(profiles, "catalog", "arbitrary-name.yaml"),
|
||||
filepath.Join(profiles, "selected.yml"),
|
||||
resource,
|
||||
}
|
||||
// Runtime paths are canonical paths from PromptKit's captured sources.
|
||||
for i := range want {
|
||||
want[i], err = filepath.EvalSymlinks(want[i])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sort.Strings(want)
|
||||
if !reflect.DeepEqual(runtime.SourcePaths, want) {
|
||||
t.Fatalf("captured source paths = %#v, want %#v", runtime.SourcePaths, want)
|
||||
}
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
|
||||
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nmodel: local-model\n"), 0600); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.File(main), "")
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
|
||||
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles/base.yaml"), []byte("id: base\nbackend: shared\nmodel: local\ntimeout_seconds: 30\n"), 0600); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles/base.yaml"), []byte("profiles:\n base:\n backend: shared\n model: local\n timeout_seconds: 30\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.File(main), "")
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
||||
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nmodel: local-model\n"), 0600); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.File(main), "")
|
||||
|
||||
@@ -1,37 +1,38 @@
|
||||
id: modules.glossary.proposal
|
||||
version: v1
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a careful transcript correction assistant. Identify only transcription errors that are strongly supported by the glossary. A valid correction must be acoustically plausible: the original transcript text should sound similar to the proposed correction when spoken aloud. Do not make generic grammar, spelling, capitalization, style, or filler-word edits. Do not substitute an unrelated glossary term just because it could fit the topic. Preserve speaker names, timestamps, and meaning.
|
||||
- role: user
|
||||
content: |
|
||||
Review this transcript section and return only glossary-supported corrections that should be applied.
|
||||
prompts:
|
||||
modules.glossary.proposal:
|
||||
v1:
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a careful transcript correction assistant. Identify only transcription errors that are strongly supported by the glossary. A valid correction must be acoustically plausible: the original transcript text should sound similar to the proposed correction when spoken aloud. Do not make generic grammar, spelling, capitalization, style, or filler-word edits. Do not substitute an unrelated glossary term just because it could fit the topic. Preserve speaker names, timestamps, and meaning.
|
||||
- role: user
|
||||
content: |
|
||||
Review this transcript section and return only glossary-supported corrections that should be applied.
|
||||
|
||||
Rules:
|
||||
- Correct domain-specific names, aliases, jargon, deities, locations, NPCs, players, factions, and similar terms only when both the glossary and surrounding transcript context support the correction.
|
||||
- The correction must plausibly fix a transcription error: the original words should be phonetically or acoustically similar to the corrected words in spoken English.
|
||||
- Appropriate example: correcting "gestures" to "Jesters" can be valid if "Jesters" appears in the glossary and nearby context supports that inference.
|
||||
- Inappropriate example: correcting "Lyra" to "Jesters" should be omitted because those words are not similar in spoken English, even if "Jesters" appears in the glossary.
|
||||
- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.
|
||||
- Preserve correctly identified glossary names and aliases. A recognized glossary term may still be a mishearing of another term; correct that misidentification only when spoken similarity and surrounding context support it.
|
||||
- Apart from that supported mishearing exception, do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary.
|
||||
- Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
Rules:
|
||||
- Correct domain-specific names, aliases, jargon, deities, locations, NPCs, players, factions, and similar terms only when both the glossary and surrounding transcript context support the correction.
|
||||
- The correction must plausibly fix a transcription error: the original words should be phonetically or acoustically similar to the corrected words in spoken English.
|
||||
- Appropriate example: correcting "gestures" to "Jesters" can be valid if "Jesters" appears in the glossary and nearby context supports that inference.
|
||||
- Inappropriate example: correcting "Lyra" to "Jesters" should be omitted because those words are not similar in spoken English, even if "Jesters" appears in the glossary.
|
||||
- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.
|
||||
- Preserve correctly identified glossary names and aliases. A recognized glossary term may still be a mishearing of another term; correct that misidentification only when spoken similarity and surrounding context support it.
|
||||
- Apart from that supported mishearing exception, do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary.
|
||||
- Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
|
||||
{{ .ProposalContract }}
|
||||
{{ .ProposalContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Glossary:
|
||||
{{ .GlossaryJSON }}
|
||||
{{ .TranscriptDescriptionBlock }}Glossary:
|
||||
{{ .GlossaryJSON }}
|
||||
|
||||
Transcript section:
|
||||
{{ .SectionJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: correction_set.v1.json
|
||||
Transcript section:
|
||||
{{ .SectionJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: correction_set.v1.json
|
||||
|
||||
@@ -1,40 +1,41 @@
|
||||
id: modules.grammar.proposal
|
||||
version: v1
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a conservative grammar cleanup assistant. Identify only punctuation, capitalization, and spacing cleanup that preserves the same underlying words. The only word-substitution exception is the whole-word article "a"/"an" correction described below; otherwise do not change content, substitute words, or rewrite the speaker's phrasing.
|
||||
- role: user
|
||||
content: |
|
||||
Review this transcript section and return only grammar cleanup corrections that should be applied.
|
||||
prompts:
|
||||
modules.grammar.proposal:
|
||||
v1:
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a conservative grammar cleanup assistant. Identify only punctuation, capitalization, and spacing cleanup that preserves the same underlying words. The only word-substitution exception is the whole-word article "a"/"an" correction described below; otherwise do not change content, substitute words, or rewrite the speaker's phrasing.
|
||||
- role: user
|
||||
content: |
|
||||
Review this transcript section and return only grammar cleanup corrections that should be applied.
|
||||
|
||||
Rules:
|
||||
- Allowed changes are punctuation, capitalization, spacing, and article cleanup only.
|
||||
- You may add, remove, or adjust commas, periods, quotation marks, apostrophes, dashes, ellipses, spacing, and capitalization when the underlying words stay the same.
|
||||
- You may change the whole-word article "a" to "an" or "an" to "a" when the surrounding text otherwise stays the same.
|
||||
- Homophone, spoken-form, and mistranscription corrections are outside this module's scope; do not propose them here.
|
||||
- Apart from the a/an article exception, do not make word substitutions, spelling fixes, homophone fixes, filler cleanup, repetition cleanup, paraphrases, or other semantic rewrites.
|
||||
- Do not change one written word into a different written word, except for capitalization changes to the same letters and the a/an article exception.
|
||||
- If a possible correction depends on changing a content word into a different word, omit it here rather than bundling it together with formatting cleanup.
|
||||
- Treat glossary names and aliases as protected spellings and context.
|
||||
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Choose an original_text span that appears exactly once in the current segment text.
|
||||
- A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed.
|
||||
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
Rules:
|
||||
- Allowed changes are punctuation, capitalization, spacing, and article cleanup only.
|
||||
- You may add, remove, or adjust commas, periods, quotation marks, apostrophes, dashes, ellipses, spacing, and capitalization when the underlying words stay the same.
|
||||
- You may change the whole-word article "a" to "an" or "an" to "a" when the surrounding text otherwise stays the same.
|
||||
- Homophone, spoken-form, and mistranscription corrections are outside this module's scope; do not propose them here.
|
||||
- Apart from the a/an article exception, do not make word substitutions, spelling fixes, homophone fixes, filler cleanup, repetition cleanup, paraphrases, or other semantic rewrites.
|
||||
- Do not change one written word into a different written word, except for capitalization changes to the same letters and the a/an article exception.
|
||||
- If a possible correction depends on changing a content word into a different word, omit it here rather than bundling it together with formatting cleanup.
|
||||
- Treat glossary names and aliases as protected spellings and context.
|
||||
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Choose an original_text span that appears exactly once in the current segment text.
|
||||
- A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed.
|
||||
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
|
||||
{{ .ProposalContract }}
|
||||
{{ .ProposalContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Protected glossary/context:
|
||||
{{ .GlossaryJSON }}
|
||||
{{ .TranscriptDescriptionBlock }}Protected glossary/context:
|
||||
{{ .GlossaryJSON }}
|
||||
|
||||
Transcript section:
|
||||
{{ .SectionJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: correction_set.v1.json
|
||||
Transcript section:
|
||||
{{ .SectionJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: correction_set.v1.json
|
||||
|
||||
@@ -1,37 +1,38 @@
|
||||
id: modules.homophones.proposal
|
||||
version: v1
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a conservative homophone correction assistant. Identify only transcript changes that plausibly reflect homophones, phonetic similarity, or common mistranscriptions of spoken English. Do not make punctuation, capitalization, spacing, filler-word, repetition, style, or grammar edits. Do not paraphrase, summarize, or rewrite content.
|
||||
- role: user
|
||||
content: |
|
||||
Review this transcript section and return only homophone or spoken-form corrections that should be applied.
|
||||
prompts:
|
||||
modules.homophones.proposal:
|
||||
v1:
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a conservative homophone correction assistant. Identify only transcript changes that plausibly reflect homophones, phonetic similarity, or common mistranscriptions of spoken English. Do not make punctuation, capitalization, spacing, filler-word, repetition, style, or grammar edits. Do not paraphrase, summarize, or rewrite content.
|
||||
- role: user
|
||||
content: |
|
||||
Review this transcript section and return only homophone or spoken-form corrections that should be applied.
|
||||
|
||||
Rules:
|
||||
- Approve only corrections where the original text is plausibly a mistaken homophone, phonetic rendering, or mistranscription of what was likely spoken.
|
||||
- Allow examples such as changing "dam" to "damn", "rank" to "Hrank", or "gestures" to "Jesters" when local context supports the correction.
|
||||
- Reject unrelated substitutions like changing "Lyra" to "Jesters".
|
||||
- Reject antonyms or reversals such as changing "visible" to "invisible".
|
||||
- Do not add or remove punctuation, alter capitalization only, normalize spacing, remove filler words, collapse repetitions, or make general readability edits.
|
||||
- Treat glossary names and aliases as protected spellings and context.
|
||||
- You may correct toward glossary names, aliases, or their plural forms when the correction is acoustically plausible and supported by local context.
|
||||
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Choose an original_text span that appears exactly once in the current segment text.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
Rules:
|
||||
- Approve only corrections where the original text is plausibly a mistaken homophone, phonetic rendering, or mistranscription of what was likely spoken.
|
||||
- Allow examples such as changing "dam" to "damn", "rank" to "Hrank", or "gestures" to "Jesters" when local context supports the correction.
|
||||
- Reject unrelated substitutions like changing "Lyra" to "Jesters".
|
||||
- Reject antonyms or reversals such as changing "visible" to "invisible".
|
||||
- Do not add or remove punctuation, alter capitalization only, normalize spacing, remove filler words, collapse repetitions, or make general readability edits.
|
||||
- Treat glossary names and aliases as protected spellings and context.
|
||||
- You may correct toward glossary names, aliases, or their plural forms when the correction is acoustically plausible and supported by local context.
|
||||
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Choose an original_text span that appears exactly once in the current segment text.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
|
||||
{{ .ProposalContract }}
|
||||
{{ .ProposalContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Protected glossary/context:
|
||||
{{ .GlossaryJSON }}
|
||||
{{ .TranscriptDescriptionBlock }}Protected glossary/context:
|
||||
{{ .GlossaryJSON }}
|
||||
|
||||
Transcript section:
|
||||
{{ .SectionJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: correction_set.v1.json
|
||||
Transcript section:
|
||||
{{ .SectionJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: correction_set.v1.json
|
||||
|
||||
@@ -1,42 +1,43 @@
|
||||
id: modules.spoken_word.proposal
|
||||
version: v1
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a conservative spoken-word cleanup assistant. Identify only low-risk cleanup of repeated words or short phrases, filler words, hesitation artifacts, and similar dysfluencies that commonly appear in spoken English transcripts. Preserve substantive meaning, named entities, and transcript content.
|
||||
- role: user
|
||||
content: |
|
||||
Review this transcript section and return only spoken-word cleanup corrections that should be applied.
|
||||
prompts:
|
||||
modules.spoken_word.proposal:
|
||||
v1:
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a conservative spoken-word cleanup assistant. Identify only low-risk cleanup of repeated words or short phrases, filler words, hesitation artifacts, and similar dysfluencies that commonly appear in spoken English transcripts. Preserve substantive meaning, named entities, and transcript content.
|
||||
- role: user
|
||||
content: |
|
||||
Review this transcript section and return only spoken-word cleanup corrections that should be applied.
|
||||
|
||||
Rules:
|
||||
- Approve only conservative cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar spoken dysfluencies.
|
||||
- You may collapse adjacent repetition such as "I I think" to "I think" or remove filler spans such as "you know" or "uh" when local context supports that cleanup.
|
||||
- Do not collapse repeated words or short phrases when the repetition plausibly expresses urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.
|
||||
- Phrases such as "Help! Help! Help!", "Stop! Stop! Stop!", "No! No! No!", "Yes! Yes! Yes!", and "Go! Go! Go!" are often intentional emphasis and should usually be preserved.
|
||||
- Only collapse repetition when local context supports it as accidental spoken repetition, hesitation, or verbal restart.
|
||||
- You may include low-risk punctuation, spacing, or capitalization cleanup when it is part of removing a dysfluency, such as removing ellipses or hesitation punctuation that no longer belongs after the cleanup.
|
||||
- Do not paraphrase, summarize, reorder ideas, replace content with different wording, or make substantive semantic edits.
|
||||
- Do not change clear content words just because a different phrasing reads better.
|
||||
- Do not convert uncertain statements into certain statements.
|
||||
- Treat glossary names and aliases as protected spellings and context. Accidental repetition of a protected name may be reduced when that identity remains in the segment; preserve substantive references and intentional emphasis.
|
||||
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Choose an original_text span that appears exactly once in the current segment text.
|
||||
- An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result.
|
||||
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
Rules:
|
||||
- Approve only conservative cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar spoken dysfluencies.
|
||||
- You may collapse adjacent repetition such as "I I think" to "I think" or remove filler spans such as "you know" or "uh" when local context supports that cleanup.
|
||||
- Do not collapse repeated words or short phrases when the repetition plausibly expresses urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.
|
||||
- Phrases such as "Help! Help! Help!", "Stop! Stop! Stop!", "No! No! No!", "Yes! Yes! Yes!", and "Go! Go! Go!" are often intentional emphasis and should usually be preserved.
|
||||
- Only collapse repetition when local context supports it as accidental spoken repetition, hesitation, or verbal restart.
|
||||
- You may include low-risk punctuation, spacing, or capitalization cleanup when it is part of removing a dysfluency, such as removing ellipses or hesitation punctuation that no longer belongs after the cleanup.
|
||||
- Do not paraphrase, summarize, reorder ideas, replace content with different wording, or make substantive semantic edits.
|
||||
- Do not change clear content words just because a different phrasing reads better.
|
||||
- Do not convert uncertain statements into certain statements.
|
||||
- Treat glossary names and aliases as protected spellings and context. Accidental repetition of a protected name may be reduced when that identity remains in the segment; preserve substantive references and intentional emphasis.
|
||||
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Choose an original_text span that appears exactly once in the current segment text.
|
||||
- An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result.
|
||||
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
|
||||
{{ .ProposalContract }}
|
||||
{{ .ProposalContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Protected glossary/context:
|
||||
{{ .GlossaryJSON }}
|
||||
{{ .TranscriptDescriptionBlock }}Protected glossary/context:
|
||||
{{ .GlossaryJSON }}
|
||||
|
||||
Transcript section:
|
||||
{{ .SectionJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: correction_set.v1.json
|
||||
Transcript section:
|
||||
{{ .SectionJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: correction_set.v1.json
|
||||
|
||||
@@ -1,34 +1,35 @@
|
||||
id: validators.editorial_review
|
||||
version: v1
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a conservative editorial validation assistant. Evaluate whether each proposed correction is editorial in nature and preserves the segment's underlying meaning. Editorial revisions may include dysfluency cleanup, punctuation changes, capitalization changes, homophone or mistranscription corrections, and similar low-risk editorial cleanup.
|
||||
- role: user
|
||||
content: |
|
||||
Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.
|
||||
prompts:
|
||||
validators.editorial_review:
|
||||
v1:
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a conservative editorial validation assistant. Evaluate whether each proposed correction is editorial in nature and preserves the segment's underlying meaning. Editorial revisions may include dysfluency cleanup, punctuation changes, capitalization changes, homophone or mistranscription corrections, and similar low-risk editorial cleanup.
|
||||
- role: user
|
||||
content: |
|
||||
Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.
|
||||
|
||||
Rules:
|
||||
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}.
|
||||
- Approve editorial revisions that preserve the underlying meaning of the segment.
|
||||
- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.
|
||||
- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.
|
||||
- Approve low-risk homophone or mistranscription corrections when they are contextually well supported.
|
||||
- Approve conservative combinations of these editorial changes when the overall revision remains meaning-preserving.
|
||||
- Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.
|
||||
- Phrases such as "Help! Help! Help!", "Stop! Stop! Stop!", "No! No! No!", "Yes! Yes! Yes!", and "Go! Go! Go!" are often intentional emphasis and should usually be preserved.
|
||||
- Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart.
|
||||
- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.
|
||||
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
|
||||
- An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty.
|
||||
Rules:
|
||||
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}.
|
||||
- Approve editorial revisions that preserve the underlying meaning of the segment.
|
||||
- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.
|
||||
- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.
|
||||
- Approve low-risk homophone or mistranscription corrections when they are contextually well supported.
|
||||
- Approve conservative combinations of these editorial changes when the overall revision remains meaning-preserving.
|
||||
- Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.
|
||||
- Phrases such as "Help! Help! Help!", "Stop! Stop! Stop!", "No! No! No!", "Yes! Yes! Yes!", and "Go! Go! Go!" are often intentional emphasis and should usually be preserved.
|
||||
- Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart.
|
||||
- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.
|
||||
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
|
||||
- An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty.
|
||||
|
||||
{{ .ValidationContract }}
|
||||
{{ .ValidationContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Corrections to validate:
|
||||
{{ .PayloadJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: validator_decision_set.v1.json
|
||||
{{ .TranscriptDescriptionBlock }}Corrections to validate:
|
||||
{{ .PayloadJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: validator_decision_set.v1.json
|
||||
|
||||
@@ -1,30 +1,31 @@
|
||||
id: validators.meaning_reversal_review
|
||||
version: v1
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a narrow semantic-reversal validation assistant. Evaluate whether each proposed correction changes a word to its antonym or otherwise reverses the meaning of the full segment. Do not treat every word substitution as a problem; focus specifically on antonyms and meaning reversals.
|
||||
- role: user
|
||||
content: |
|
||||
Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.
|
||||
prompts:
|
||||
validators.meaning_reversal_review:
|
||||
v1:
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a narrow semantic-reversal validation assistant. Evaluate whether each proposed correction changes a word to its antonym or otherwise reverses the meaning of the full segment. Do not treat every word substitution as a problem; focus specifically on antonyms and meaning reversals.
|
||||
- role: user
|
||||
content: |
|
||||
Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.
|
||||
|
||||
Rules:
|
||||
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}.
|
||||
- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.
|
||||
- Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning.
|
||||
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
|
||||
- An empty corrected_text deletes the identified span; decide from the full before/after segment whether that deletion reverses meaning, not from the empty string alone.
|
||||
- For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone.
|
||||
- Do not reject a correction merely because the literal written word changes.
|
||||
- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.
|
||||
Rules:
|
||||
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}.
|
||||
- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.
|
||||
- Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning.
|
||||
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
|
||||
- An empty corrected_text deletes the identified span; decide from the full before/after segment whether that deletion reverses meaning, not from the empty string alone.
|
||||
- For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone.
|
||||
- Do not reject a correction merely because the literal written word changes.
|
||||
- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.
|
||||
|
||||
{{ .ValidationContract }}
|
||||
{{ .ValidationContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Corrections to validate:
|
||||
{{ .PayloadJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: validator_decision_set.v1.json
|
||||
{{ .TranscriptDescriptionBlock }}Corrections to validate:
|
||||
{{ .PayloadJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: validator_decision_set.v1.json
|
||||
|
||||
@@ -1,29 +1,30 @@
|
||||
id: validators.spoken_form_plausibility
|
||||
version: v1
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a conservative spoken-form validation assistant. Evaluate whether each proposed correction is plausibly explained by a homophone, phonetic similarity, or a common mistranscription of spoken English. Your job is not to improve style or readability. Approve only when the corrected text is a plausible recovery of the words that were likely spoken.
|
||||
- role: user
|
||||
content: |
|
||||
Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.
|
||||
prompts:
|
||||
validators.spoken_form_plausibility:
|
||||
v1:
|
||||
messages:
|
||||
- role: system
|
||||
content: |
|
||||
You are Audita, a conservative spoken-form validation assistant. Evaluate whether each proposed correction is plausibly explained by a homophone, phonetic similarity, or a common mistranscription of spoken English. Your job is not to improve style or readability. Approve only when the corrected text is a plausible recovery of the words that were likely spoken.
|
||||
- role: user
|
||||
content: |
|
||||
Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.
|
||||
|
||||
Rules:
|
||||
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}.
|
||||
- Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.
|
||||
- Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn".
|
||||
- Reject unrelated substitutions like changing "Lyra" to "Jesters".
|
||||
- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.
|
||||
- For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription.
|
||||
- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.
|
||||
Rules:
|
||||
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}.
|
||||
- Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.
|
||||
- Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn".
|
||||
- Reject unrelated substitutions like changing "Lyra" to "Jesters".
|
||||
- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.
|
||||
- For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription.
|
||||
- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.
|
||||
|
||||
{{ .ValidationContract }}
|
||||
{{ .ValidationContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Corrections to validate:
|
||||
{{ .PayloadJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: validator_decision_set.v1.json
|
||||
{{ .TranscriptDescriptionBlock }}Corrections to validate:
|
||||
{{ .PayloadJSON }}
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: validator_decision_set.v1.json
|
||||
|
||||
@@ -5,15 +5,28 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
|
||||
engine, err := pk.NewEngine(pk.Config{},
|
||||
pk.WithPromptFS(Filesystem(), AssetRoot),
|
||||
promptInputs, err := pk.FilesystemInputs("embedded prompts", Filesystem(), AssetRoot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoint := "https://example.invalid/v1"
|
||||
model := "fixture-model"
|
||||
sources := pk.ConfigurationSources{
|
||||
Underlays: []ck.Underlay{
|
||||
{Inputs: promptInputs},
|
||||
{Inputs: []ck.Input{{Source: pk.ProfileSource("registry fixture profile", map[string]pk.ProfileConfig{
|
||||
"fixture": {Endpoint: &endpoint, Model: &model},
|
||||
})}}},
|
||||
},
|
||||
}
|
||||
engine, err := pk.NewEngine(pk.Config{Sources: sources},
|
||||
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
|
||||
pk.WithProfiles(pk.Profile{ID: "fixture", Endpoint: "https://example.invalid/v1", Model: "fixture-model"}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
Reference in New Issue
Block a user