Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3b4933256e | |||
| 6f7136724f | |||
| 9bcf18331e | |||
| 5525fe875e | |||
| b81bd28865 | |||
| 9b82eb7da2 | |||
| f8d00e2f20 | |||
| f5fa607c35 | |||
| 43a68e97ee | |||
| b1b0fc7455 | |||
| afcf58331f | |||
| b31955fc97 | |||
| 1777821e8b | |||
| 79d38606fe | |||
| 8cba751835 | |||
| 8175ed87d0 | |||
| b64010e729 | |||
| 1cfe0c5a01 | |||
| 0e2c0b0b82 | |||
| cd9ee78359 | |||
| 23e1df4cfb | |||
| 2023c91ec3 | |||
| 67519051f1 | |||
| 6ae1d7cd5f | |||
| ddfc3dd4c7 | |||
| 990ca0b71a | |||
| 61bc467a03 | |||
| 0bd116b6b7 |
@@ -15,7 +15,7 @@ go build -o ./bin/audita ./cmd/audita
|
|||||||
Processing requires a reachable [compatible LLM endpoint](docs/integrations/openai-compatible-llm.md)
|
Processing requires a reachable [compatible LLM endpoint](docs/integrations/openai-compatible-llm.md)
|
||||||
and access to a model you select. The [minimal config](examples/minimal-config.yml)
|
and access to a model you select. The [minimal config](examples/minimal-config.yml)
|
||||||
targets an unauthenticated local endpoint; edit its endpoint and
|
targets an unauthenticated local endpoint; edit its endpoint and
|
||||||
[local profiles](examples/profiles/) to match your service and models. The
|
[local profiles](examples/profiles/local/) to match your service and models. The
|
||||||
[service example](examples/production-config.yml) demonstrates a named credential
|
[service example](examples/production-config.yml) demonstrates a named credential
|
||||||
reference and a shared backend. Set its referenced environment variable before
|
reference and a shared backend. Set its referenced environment variable before
|
||||||
validating it. The [testing overlay](examples/overlays/testing.yml) selects both
|
validating it. The [testing overlay](examples/overlays/testing.yml) selects both
|
||||||
|
|||||||
@@ -237,7 +237,7 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
for _, id := range []string{"proposal", "validation"} {
|
for _, id := range []string{"proposal", "validation"} {
|
||||||
profile := fmt.Sprintf("id: %s\nbackend: local\nmodel: local\ntimeout_seconds: 5\n", id)
|
profile := fmt.Sprintf("profiles:\n %s:\n backend: local\n model: local\n timeout_seconds: 5\n", id)
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -293,6 +293,25 @@ func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (
|
|||||||
return code, stdout.String(), stderr.String()
|
return code, stdout.String(), stderr.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f processFixture) runConfigView(t *testing.T, view string) (int, string, string) {
|
||||||
|
t.Helper()
|
||||||
|
command := exec.Command(os.Args[0], "-test.run=^TestProcessSubprocessHelper$", "--", "config", "print-effective", "--config", f.config, "--view", view)
|
||||||
|
command.Dir = f.dir
|
||||||
|
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
command.Stdout, command.Stderr = &stdout, &stderr
|
||||||
|
err := command.Run()
|
||||||
|
code := 0
|
||||||
|
if err != nil {
|
||||||
|
if exit, ok := err.(*exec.ExitError); ok {
|
||||||
|
code = exit.ExitCode()
|
||||||
|
} else {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return code, stdout.String(), stderr.String()
|
||||||
|
}
|
||||||
|
|
||||||
func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument {
|
func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml"))
|
raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml"))
|
||||||
@@ -799,15 +818,70 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
transcriptDir := filepath.Join(f.dir, "transcripts")
|
||||||
|
reportDir := filepath.Join(f.dir, "reports")
|
||||||
|
debugDir := filepath.Join(f.dir, "debug")
|
||||||
|
for _, dir := range []string{transcriptDir, reportDir} {
|
||||||
|
if err := os.Mkdir(dir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.output = filepath.Join(transcriptDir, "corrected.json")
|
||||||
|
code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir)
|
||||||
|
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
||||||
|
t.Fatalf("transcript: %s %v", output, err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
||||||
|
if _, err := os.Stat(filepath.Join(reportDir, name)); err != nil {
|
||||||
|
t.Fatalf("report %s: %v", name, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(transcriptDir, name)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("report %s appeared beside transcript: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dirs, err := filepath.Glob(filepath.Join(debugDir, "run-*"))
|
||||||
|
if err != nil || len(dirs) != 1 {
|
||||||
|
t.Fatalf("debug runs: %v %v", dirs, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dirs[0], "correction-ledger.json")); err != nil {
|
||||||
|
t.Fatalf("debug capture: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
||||||
var expectedOutput, expectedSummary string
|
var expectedOutput, expectedSummary string
|
||||||
for _, selection := range []struct{ reporting, debug bool }{
|
for _, selection := range []struct{ reporting, debug bool }{
|
||||||
{false, false}, {false, true}, {true, false}, {true, true},
|
{false, false}, {false, true}, {true, false}, {true, true},
|
||||||
} {
|
} {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
extra := []string{"--reporting=false", "--report-dir", filepath.Join(f.dir, "missing")}
|
if selection.reporting && selection.debug {
|
||||||
|
original, err := os.ReadFile(f.config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
old := fmt.Sprintf("backends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY", f.server.URL)
|
||||||
|
inherited := fmt.Sprintf("backends:\n local-base:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\n queue_capacity: 0\n local:\n extends: local-base\n concurrency: 1", f.server.URL)
|
||||||
|
updated := strings.Replace(string(original), old, inherited, 1)
|
||||||
|
if updated == string(original) {
|
||||||
|
t.Fatal("process fixture backend configuration was not found")
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reportDir := filepath.Join(f.dir, "missing")
|
||||||
|
extra := []string{"--reporting=false", "--report-dir", reportDir}
|
||||||
if selection.reporting {
|
if selection.reporting {
|
||||||
extra = []string{"--reporting=true", "--report-dir", filepath.Join(f.dir, "missing")}
|
reportDir = filepath.Join(f.dir, "selected-reports")
|
||||||
|
if err := os.Mkdir(reportDir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
extra = []string{"--reporting=true", "--report-dir", reportDir}
|
||||||
}
|
}
|
||||||
if selection.debug {
|
if selection.debug {
|
||||||
extra = append(extra, "--debug")
|
extra = append(extra, "--debug")
|
||||||
@@ -827,7 +901,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
|||||||
t.Fatalf("selection changed processing: %+v summary=%q output=%s", selection, summary, output)
|
t.Fatalf("selection changed processing: %+v summary=%q output=%s", selection, summary, output)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
||||||
_, err := os.Stat(filepath.Join(f.dir, name))
|
_, err := os.Stat(filepath.Join(reportDir, name))
|
||||||
if selection.reporting && err != nil || !selection.reporting && !os.IsNotExist(err) {
|
if selection.reporting && err != nil || !selection.reporting && !os.IsNotExist(err) {
|
||||||
t.Fatalf("selection=%+v report %s: %v", selection, name, err)
|
t.Fatalf("selection=%+v report %s: %v", selection, name, err)
|
||||||
}
|
}
|
||||||
@@ -842,6 +916,31 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if selection.reporting {
|
||||||
|
effective, err := os.ReadFile(filepath.Join(dirs[0], "effective-config.redacted.yaml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(effective), f.server.URL+"/v1") || !strings.Contains(string(effective), "concurrency: 1") || !strings.Contains(string(effective), "queue_capacity: 0") || strings.Contains(string(effective), f.provider.secret) {
|
||||||
|
t.Fatalf("effective inherited settings: %s", effective)
|
||||||
|
}
|
||||||
|
origins, err := os.ReadFile(filepath.Join(dirs[0], "config-origins.yaml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"from: /backends/local-base/endpoint", "to: /backends/local/endpoint", "reference: /backends/local/extends"} {
|
||||||
|
if !strings.Contains(string(origins), want) {
|
||||||
|
t.Fatalf("configuration origins missing %q: %s", want, origins)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(string(origins), f.provider.secret) {
|
||||||
|
t.Fatal("configuration origins exposed the resolved inherited credential")
|
||||||
|
}
|
||||||
|
code, explain, stderr := f.runConfigView(t, "explain")
|
||||||
|
if code != 0 || stderr != "" || !strings.Contains(explain, "from: /backends/local-base/endpoint") || strings.Contains(explain, f.provider.secret) {
|
||||||
|
t.Fatalf("inheritance explanation: code=%d stdout=%s stderr=%s", code, explain, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1011,6 +1110,63 @@ func TestSubprocessInvalidInputLeavesFailedReports(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
original, err := os.ReadFile(f.config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
updated := strings.Replace(string(original), " local:\n endpoint:", " local:\n extends: missing\n endpoint:", 1)
|
||||||
|
if updated == string(original) {
|
||||||
|
t.Fatal("process fixture backend configuration was not found")
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, stdout, stderr := f.run(t, true)
|
||||||
|
if code != 2 || stdout != "" || f.provider.callCount() != 0 || !strings.Contains(stderr, "referenced sibling does not exist") {
|
||||||
|
t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportDir := filepath.Join(f.dir, "failed-reports")
|
||||||
|
if err := os.Mkdir(reportDir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, _, _ := f.run(t, true, "--report-dir", reportDir)
|
||||||
|
if code != 1 || f.provider.callCount() != 0 {
|
||||||
|
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(reportDir, "errors.yml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var document reporting.ErrorsDocument
|
||||||
|
if err := yaml.Unmarshal(raw, &document); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
|
||||||
|
t.Fatalf("failed report: %+v", document)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(reportDir, "recoveries.yml")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if raw, err := os.ReadFile(filepath.Join(reportDir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
|
||||||
|
t.Fatalf("Markdown status: %s %v", raw, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("failed report appeared in default directory: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("failed transcript published: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
|
func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
f.output = filepath.Join(f.dir, "missing", "output.json")
|
f.output = filepath.Join(f.dir, "missing", "output.json")
|
||||||
@@ -1243,4 +1399,47 @@ func TestSubprocessProfileSources(t *testing.T) {
|
|||||||
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
|
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
|
||||||
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
|
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
parentDir := filepath.Join(f.dir, "profiles", "catalog")
|
||||||
|
childDir := filepath.Join(f.dir, "profiles", "nested", "child")
|
||||||
|
for _, directory := range []string{parentDir, childDir} {
|
||||||
|
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parent := filepath.Join(parentDir, "arbitrary-name.yaml")
|
||||||
|
child := filepath.Join(childDir, "override.yaml")
|
||||||
|
resource := filepath.Join(parentDir, "report.md")
|
||||||
|
for path, content := range map[string]string{
|
||||||
|
parent: "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./report.md\n output:\n format: text\n validation_mode: basic\n",
|
||||||
|
child: "prompts:\n literal/~child:\n extends: source\n",
|
||||||
|
resource: "Captured operator message.\n",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resourceBefore, err := os.ReadFile(resource)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.output = resource
|
||||||
|
code, _, stderr = f.run(t, true, "--reporting=false")
|
||||||
|
resourceAfter, err := os.ReadFile(resource)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
|
||||||
|
t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
f.output = filepath.Join(f.dir, "output.json")
|
||||||
|
code, _, stderr = f.run(t, true, "--report-dir", parentDir)
|
||||||
|
resourceAfter, err = os.ReadFile(resource)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
|
||||||
|
t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
28
docs/cli.md
28
docs/cli.md
@@ -48,12 +48,17 @@ validation. The shared setting flags are:
|
|||||||
An omitted or empty output value selects stdout. Nonempty output filenames are
|
An omitted or empty output value selects stdout. Nonempty output filenames are
|
||||||
used literally, including whitespace.
|
used literally, including whitespace.
|
||||||
Reporting defaults to enabled. `--reporting=false` disables all three ordinary
|
Reporting defaults to enabled. `--reporting=false` disables all three ordinary
|
||||||
reports; an explicit false overrides a true file or overlay setting. With file
|
reports; an explicit false overrides a true file or overlay setting. For stdout
|
||||||
output, reports go beside the transcript and `--report-dir` has no effect. With
|
output, reports use the effective `reporting.stdout_dir`, which defaults to the
|
||||||
stdout output, `--report-dir` selects the report directory, defaulting to the
|
invocation working directory. For file output, they go beside the transcript
|
||||||
invocation working directory. `--debug` separately enables retained detailed
|
unless `--report-dir` is explicitly supplied. The flag selects the report
|
||||||
artifacts under `--work-dir`. Configuration inspection does not create or probe
|
directory for either transcript destination; a config-file or overlay value for
|
||||||
these output directories.
|
`reporting.stdout_dir` alone affects stdout output only. A selected report
|
||||||
|
directory must be nonblank when reporting is enabled. With reporting disabled,
|
||||||
|
the report destination is inactive and is not probed. `--debug` separately
|
||||||
|
enables retained detailed artifacts under `--work-dir`; selecting `--work-dir`
|
||||||
|
alone does not enable capture. Configuration inspection does not create or
|
||||||
|
probe these output directories.
|
||||||
Per-role model, base URL, key, timeout, retry, and concurrency flags have been
|
Per-role model, base URL, key, timeout, retry, and concurrency flags have been
|
||||||
removed. Models and generation controls belong to PromptKit profiles, backend
|
removed. Models and generation controls belong to PromptKit profiles, backend
|
||||||
credentials and capacity to configuration, and retry/deadline settings to the
|
credentials and capacity to configuration, and retry/deadline settings to the
|
||||||
@@ -69,9 +74,14 @@ YAML summary of selected ID, backend, model, sampling controls, per-attempt
|
|||||||
timeout, service tier, and reasoning effort for each role. It omits credentials,
|
timeout, service tier, and reasoning effort for each role. It omits credentials,
|
||||||
endpoints, and arbitrary extra provider parameters. Explain writes a YAML
|
endpoints, and arbitrary extra provider parameters. Explain writes a YAML
|
||||||
sequence of field paths, source origins, and recorded derivations without
|
sequence of field paths, source origins, and recorded derivations without
|
||||||
resolved setting values. These views share the same settings and profile
|
resolved setting values. An inherited field may also include an `inheritance`
|
||||||
validation path as `process`. Unknown views are usage errors. The effective
|
sequence; each step identifies `from`, `to`, and `reference` paths as JSON
|
||||||
settings YAML is for inspection and is not guaranteed reloadable.
|
pointers. These fields describe the effective inheritance chain without
|
||||||
|
including configuration values. A direct child override reports its own source
|
||||||
|
instead. Debug configuration origins use the same explanation shape. These views
|
||||||
|
share the same settings and profile validation path as `process`. Unknown views
|
||||||
|
are usage errors. The effective settings YAML is for inspection and is not
|
||||||
|
guaranteed reloadable.
|
||||||
|
|
||||||
For `process`, `--output` writes the corrected transcript to a file and leaves
|
For `process`, `--output` writes the corrected transcript to a file and leaves
|
||||||
stdout empty on success. Without it, stdout carries transcript JSON only on
|
stdout empty on success. Without it, stdout carries transcript JSON only on
|
||||||
|
|||||||
136
docs/config.md
136
docs/config.md
@@ -2,8 +2,12 @@
|
|||||||
|
|
||||||
This reference describes the version-2 configuration loader in `internal/core/config`.
|
This reference describes the version-2 configuration loader in `internal/core/config`.
|
||||||
Use the maintained [local](../examples/minimal-config.yml) and
|
Use the maintained [local](../examples/minimal-config.yml) and
|
||||||
[service](../examples/production-config.yml) examples with their PromptKit
|
[service](../examples/production-config.yml) examples with their separately
|
||||||
[profiles](../examples/profiles/) and optional [testing overlay](../examples/overlays/testing.yml).
|
selected [local profiles](../examples/profiles/local/) and
|
||||||
|
[service profiles](../examples/profiles/service/), plus the optional
|
||||||
|
[testing overlay](../examples/overlays/testing.yml).
|
||||||
|
The [ConfigKit inheritance example](../examples/configkit-inheritance.yml)
|
||||||
|
shows a local child backend reusing and overriding sibling settings.
|
||||||
Choose real models for your endpoint; a testing profile makes no cost or quality
|
Choose real models for your endpoint; a testing profile makes no cost or quality
|
||||||
guarantee. See the [migration guide](migrations/v2.md) for retired inputs.
|
guarantee. See the [migration guide](migrations/v2.md) for retired inputs.
|
||||||
|
|
||||||
@@ -20,26 +24,123 @@ and secret-directory bootstrap are disabled. Unknown fields and missing, null,
|
|||||||
or non-integer `version: 2` are errors. Version-1 files require migration; no
|
or non-integer `version: 2` are errors. Version-1 files require migration; no
|
||||||
compatibility loader or implicit environment configuration remains.
|
compatibility loader or implicit environment configuration remains.
|
||||||
|
|
||||||
|
After overlay and explicit CLI override precedence is applied, ConfigKit resolves
|
||||||
|
native sibling inheritance before typed decoding. Any input mapping may use
|
||||||
|
`extends: <sibling-name>`; the child keeps its key, inherits parent settings, and
|
||||||
|
overrides them with its own settings. Selected overlay and CLI values therefore
|
||||||
|
participate in inheritance at their normal precedence. The generic syntax and
|
||||||
|
resolution rules are defined by ConfigKit v0.5.0's
|
||||||
|
[sibling inheritance contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.5.0/docs/config.md#sibling-inheritance).
|
||||||
|
|
||||||
|
`extends` is reserved structural input in base files, imports, overlays, and
|
||||||
|
other application input mappings, including mapping values supplied by explicit
|
||||||
|
CLI setting overrides. ConfigKit consumes it before typed decoding. A previously
|
||||||
|
valid backend whose ID is literally `extends` cannot be represented under this
|
||||||
|
rule. Rename that backend to a non-conflicting ID and update every PromptKit
|
||||||
|
profile that selects it. This compatibility impact means the upgrade does not
|
||||||
|
promise that every previously valid configuration remains valid.
|
||||||
|
|
||||||
|
Inherited values count as explicit input, including zero, false, empty, and
|
||||||
|
null. Existing field rules still apply: for example, inherited backend
|
||||||
|
concurrency must be positive, while an inherited queue capacity of zero is
|
||||||
|
valid. Parent backends undergo the same decoding and application validation as
|
||||||
|
other backends, whether or not a selected profile uses them. A child
|
||||||
|
`credential: null` removes the child's inherited credential; it does not remove
|
||||||
|
the parent's credential reference, which remains subject to ordinary secret
|
||||||
|
resolution.
|
||||||
|
|
||||||
Validation runs after overlays, explicit overrides, and derived settings have
|
Validation runs after overlays, explicit overrides, and derived settings have
|
||||||
resolved. An override can therefore repair an invalid lower-precedence setting
|
resolved. An override can therefore repair an invalid lower-precedence setting
|
||||||
when that setting can be decoded. ConfigKit retains field origins and derivations
|
when that setting can be decoded. ConfigKit retains field origins and derivations
|
||||||
for effective-configuration inspection. Its effective YAML redacts resolved
|
for effective-configuration inspection. Its effective YAML redacts resolved
|
||||||
credentials.
|
credentials.
|
||||||
|
|
||||||
Profile files are resolved against the selected main file's directory, or may
|
### PromptKit profiles and operator prompts
|
||||||
use an absolute `profiles_dir`. Both selected profiles must exist and select a
|
|
||||||
registered backend. Their effective endpoints must match that backend's endpoint
|
`profiles_dir` selects a PromptKit configuration root, resolved against the
|
||||||
after surrounding whitespace and trailing slashes are removed. Profiles cannot
|
selected main file's directory unless it is absolute. The root may contain
|
||||||
supply a competing `api_key_env` lookup. The backend credential supplies the
|
keyed `profiles`, `backends`, and `prompts` definitions; it is not limited to
|
||||||
runtime key when required; inspection makes no provider call. The
|
profile-only files. IDs come from literal YAML mapping keys, never filenames.
|
||||||
[CLI inspection views](cli.md#streams-and-exit-status) show selected controls and
|
Multiple definitions may share a document, and a document may contribute to any
|
||||||
field provenance separately from ConfigKit's redacted settings YAML.
|
of the supported mappings. Directory discovery is recursive, but discovery
|
||||||
|
order does not establish field precedence. Contributions within each source
|
||||||
|
group must compose without conflicting values.
|
||||||
|
|
||||||
|
PromptKit assembles sources in this order: its selected catalogs and owned
|
||||||
|
defaults, Audita's embedded prompt/schema inputs, operator documents under
|
||||||
|
`profiles_dir`, then the application backend mapping. The application mapping
|
||||||
|
owns each configured endpoint, concurrency limit, queue capacity, and credential
|
||||||
|
environment lookup. Other supported provider fields can be contributed by the
|
||||||
|
operator documents. Imports remain disabled. The current maintained examples
|
||||||
|
select separate roots: [local](../examples/profiles/local/) and
|
||||||
|
[service](../examples/profiles/service/), so definitions for one backend do not
|
||||||
|
enter the other example's final configuration.
|
||||||
|
|
||||||
|
Both selected roles must resolve to an existing profile, registered backend,
|
||||||
|
nonblank effective model, and nonblank endpoint. The normalized effective
|
||||||
|
endpoint must match the application backend endpoint. A selected profile cannot
|
||||||
|
introduce a competing `api_key_env` lookup; the application backend credential
|
||||||
|
supplies the runtime key when required. Other unselected profiles may be
|
||||||
|
partial, but every final backend, profile, and prompt definition is validated
|
||||||
|
when the engine is constructed. A malformed or conflicting unused definition
|
||||||
|
can therefore make the selected root invalid. A missing selected root is an
|
||||||
|
operator configuration error; an empty root may contribute no definitions.
|
||||||
|
|
||||||
|
Profile IDs and parent IDs are exact, literal keys. PromptKit profiles can use
|
||||||
|
native `extends` inheritance; child fields override inherited fields, while
|
||||||
|
unmentioned controls survive changes to model or backend. For example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
profiles:
|
||||||
|
"shared~base":
|
||||||
|
backend: service
|
||||||
|
model: example-model
|
||||||
|
temperature: 0.4
|
||||||
|
top_p: 0.8
|
||||||
|
"proposal fast":
|
||||||
|
extends: "shared~base"
|
||||||
|
model: faster-model
|
||||||
|
temperature: 0
|
||||||
|
```
|
||||||
|
|
||||||
|
Presence matters. Explicit `temperature: 0` is sent as zero; omission leaves it
|
||||||
|
unspecified; null suppresses an inherited temperature. Other fields have their
|
||||||
|
own native null and omission rules. Provider `extra_params` maps merge
|
||||||
|
recursively, and a null provider value is sent as JSON null. See the pinned
|
||||||
|
[PromptKit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
|
||||||
|
for the complete field semantics.
|
||||||
|
|
||||||
|
Operator documents can also override an embedded prompt/version, including its
|
||||||
|
messages and definition-level output settings. Shared variables supplied by the
|
||||||
|
adapter are rendered only when an effective message references them; Audita does
|
||||||
|
not append omitted shared text. Audita's per-call schemas, structural repair
|
||||||
|
budget, and downstream domain validators remain in force. The engine captures
|
||||||
|
effective definitions and message-file contents when constructed, so edits are
|
||||||
|
observed only by a newly constructed runtime.
|
||||||
|
|
||||||
|
### Migrating PromptKit profile documents
|
||||||
|
|
||||||
|
For the PromptKit v0.11 format, wrap each profile's fields under
|
||||||
|
`profiles: {<literal-id>: ...}`, remove inner `id`, and replace
|
||||||
|
`base_profile` with `extends`. Profile and prompt IDs are mapping keys, so they
|
||||||
|
need not match source filenames. Review zeros, omitted values, and nulls against
|
||||||
|
the native presence rules before deploying. Select a root whose final
|
||||||
|
definitions have valid backend references; eager validation applies even to
|
||||||
|
unselected definitions. See the complete maintained [local](../examples/minimal-config.yml)
|
||||||
|
and [service](../examples/production-config.yml) examples and the pinned
|
||||||
|
[consumer migration guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api).
|
||||||
|
This incompatible profile-format migration is intended for an Audita major
|
||||||
|
release; no automatic file conversion is provided.
|
||||||
|
|
||||||
|
The [CLI inspection views](cli.md#streams-and-exit-status) show selected
|
||||||
|
controls and field provenance separately from ConfigKit's redacted settings
|
||||||
|
YAML. Inspection resolves both roles without provider calls.
|
||||||
|
|
||||||
## Settings
|
## Settings
|
||||||
|
|
||||||
| Field | Default | Rule |
|
| Field | Default | Rule |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `profiles_dir` | `profiles` | Nonempty; profile files are resolved relative to the selected main file. |
|
| `profiles_dir` | `profiles` | Nonempty PromptKit document root; resolved relative to the selected main file, or absolute. |
|
||||||
| `backends.<id>.endpoint` | None | Required nonempty endpoint for each backend. At least one backend is required. |
|
| `backends.<id>.endpoint` | None | Required nonempty endpoint for each backend. At least one backend is required. |
|
||||||
| `backends.<id>.concurrency` | `1` | Positive finite capacity; shared by both roles using that backend. |
|
| `backends.<id>.concurrency` | `1` | Positive finite capacity; shared by both roles using that backend. |
|
||||||
| `backends.<id>.queue_capacity` | `1024` | Nonnegative; zero disables waiting admission. |
|
| `backends.<id>.queue_capacity` | `1024` | Nonnegative; zero disables waiting admission. |
|
||||||
@@ -62,17 +163,20 @@ field provenance separately from ConfigKit's redacted settings YAML.
|
|||||||
| `thresholds.glossary`, `.homophones`, `.spoken_word`, `.grammar` | `0.8` each | Confidence thresholds in `[0, 1]`. |
|
| `thresholds.glossary`, `.homophones`, `.spoken_word`, `.grammar` | `0.8` each | Confidence thresholds in `[0, 1]`. |
|
||||||
| `context.description` | Empty | Trimmed prompt context, at most 500 bytes. |
|
| `context.description` | Empty | Trimmed prompt context, at most 500 bytes. |
|
||||||
| `reporting.enabled` | `true` | Write the ordinary three-file report bundle when processing. |
|
| `reporting.enabled` | `true` | Write the ordinary three-file report bundle when processing. |
|
||||||
| `reporting.stdout_dir` | `.` | Report directory for stdout transcript output; ignored for file output or disabled reporting. |
|
| `reporting.stdout_dir` | `.` | Report directory for stdout transcript output; a `--report-dir` flag can select reports for either transcript destination. |
|
||||||
| `diagnostics.debug` | `false` | Persist detailed debug artifacts when processing. |
|
| `diagnostics.debug` | `false` | Persist detailed debug artifacts when processing. |
|
||||||
| `diagnostics.work_dir` | `/tmp/audita` | Parent of the retained debug run directory; required only when debug is enabled. |
|
| `diagnostics.work_dir` | `/tmp/audita` | Parent of the retained debug run directory; required only when debug is enabled. |
|
||||||
|
|
||||||
Module keys select correction modules; validator chains remain built into them.
|
Module keys select correction modules; validator chains remain built into them.
|
||||||
The loader preserves module order and repetitions. Normalization time settings
|
The loader preserves module order and repetitions. Normalization time settings
|
||||||
accept numeric seconds only. The operation timeout uses a duration string.
|
accept numeric seconds only. The operation timeout uses a duration string.
|
||||||
`reporting.stdout_dir` must be nonblank only for reporting-enabled stdout runs;
|
`reporting.stdout_dir` must be nonblank when it supplies the active report
|
||||||
`diagnostics.work_dir` must be nonblank only with debug enabled. These path
|
destination: for stdout output, or for file output when `--report-dir` is
|
||||||
requirements are checked when choosing process destinations, not during
|
explicitly supplied. File output without that flag uses the transcript's parent,
|
||||||
configuration inspection.
|
so the configured value is inactive. Reporting-disabled runs do not select a
|
||||||
|
report directory. `diagnostics.work_dir` must be nonblank only with debug
|
||||||
|
enabled. These path requirements are checked when choosing process destinations,
|
||||||
|
not during configuration inspection.
|
||||||
|
|
||||||
## Credentials
|
## Credentials
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ failures; explicit zero makes one attempt. `llm.repair_attempts` counts
|
|||||||
additional generations for schema-invalid model output. The operation timeout
|
additional generations for schema-invalid model output. The operation timeout
|
||||||
covers preparation, capacity waiting, retries, generation, and repair. Caller
|
covers preparation, capacity waiting, retries, generation, and repair. Caller
|
||||||
cancellation or an earlier deadline also applies. See [configuration](../config.md#settings)
|
cancellation or an earlier deadline also applies. See [configuration](../config.md#settings)
|
||||||
for defaults and [PromptKit's consumer contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/consumers/pkg-promptkit.md)
|
for defaults and [PromptKit v0.12.0's consumer contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md)
|
||||||
for the pinned protocol and retry details.
|
for the pinned protocol and retry details.
|
||||||
|
|
||||||
A model operation that exhausts structural repair, a live-parent local operation
|
A model operation that exhausts structural repair, a live-parent local operation
|
||||||
|
|||||||
@@ -9,11 +9,14 @@ The [CLI reference](../cli.md) owns command syntax, streams, and exit semantics;
|
|||||||
Use an argument array with the transcript path and required glossary argument,
|
Use an argument array with the transcript path and required glossary argument,
|
||||||
following the [process examples](../cli.md#commands-and-help). For machine workflows,
|
following the [process examples](../cli.md#commands-and-help). For machine workflows,
|
||||||
select an explicit transcript file destination and a configuration path. Reports
|
select an explicit transcript file destination and a configuration path. Reports
|
||||||
then go beside the transcript. For stdout output, select `--report-dir` or use
|
go beside the transcript unless an explicit `--report-dir` selects another
|
||||||
the invocation working directory. Debug is independent and off by default.
|
directory. For stdout output, reports use the effective `reporting.stdout_dir`
|
||||||
Supplying a configuration path also
|
(defaulting to the invocation working directory), including a `--report-dir`
|
||||||
makes file selection predictable; only explicitly referenced credential variables
|
override. A config-file or overlay value for `reporting.stdout_dir` alone does
|
||||||
are read from the environment.
|
not redirect file-output reports. `--work-dir` selects a debug root, while
|
||||||
|
configuration or `--debug` enables capture; the path flag alone does not.
|
||||||
|
Supplying a configuration path also makes file selection predictable; only
|
||||||
|
explicitly referenced credential variables are read from the environment.
|
||||||
|
|
||||||
Resolve the executable deliberately and set the child's working directory so
|
Resolve the executable deliberately and set the child's working directory so
|
||||||
relative input/output paths and reported artifact paths have a known base.
|
relative input/output paths and reported artifact paths have a known base.
|
||||||
@@ -28,7 +31,8 @@ Consume transcript/report data through the
|
|||||||
[output-file contract](output-files.md), and treat stderr as human-readable detail.
|
[output-file contract](output-files.md), and treat stderr as human-readable detail.
|
||||||
The three reports may be absent after early configuration or destination
|
The three reports may be absent after early configuration or destination
|
||||||
failures. After initialization, failed reports can describe invalid input or
|
failures. After initialization, failed reports can describe invalid input or
|
||||||
partial processing. Exit `0` can include recoverable model errors and applied
|
partial processing and use the same selected report directory as successful
|
||||||
|
runs. Exit `0` can include recoverable model errors and applied
|
||||||
corrections; inspect `errors.yml` and `recoveries.yml` for those outcomes. A
|
corrections; inspect `errors.yml` and `recoveries.yml` for those outcomes. A
|
||||||
transcript may have been written before a later required report failure, so
|
transcript may have been written before a later required report failure, so
|
||||||
the file alone does not prove completion.
|
the file alone does not prove completion.
|
||||||
|
|||||||
@@ -35,11 +35,18 @@ From the repository root, validate the maintained configuration examples:
|
|||||||
|
|
||||||
```sh
|
```sh
|
||||||
go run ./cmd/audita config validate --config examples/minimal-config.yml
|
go run ./cmd/audita config validate --config examples/minimal-config.yml
|
||||||
|
go run ./cmd/audita config validate --config examples/configkit-inheritance.yml
|
||||||
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml
|
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml
|
||||||
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml --overlay testing
|
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml --overlay testing
|
||||||
go run ./cmd/audita config print-effective --config examples/minimal-config.yml --view profiles
|
go run ./cmd/audita config print-effective --config examples/minimal-config.yml --view profiles
|
||||||
|
go run ./cmd/audita config print-effective --config examples/configkit-inheritance.yml --view explain
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The local and ConfigKit inheritance files select `profiles/local`; the service
|
||||||
|
file selects `profiles/service`, including the service testing profiles. Keep
|
||||||
|
these roots separate because PromptKit validates every final definition in a
|
||||||
|
selected root, including definitions that are not selected for a role.
|
||||||
|
|
||||||
Only referenced credentials read environment values; use a controlled synthetic
|
Only referenced credentials read environment values; use a controlled synthetic
|
||||||
value when validating the service example. Check affected CLI examples against the
|
value when validating the service example. Check affected CLI examples against the
|
||||||
parser and use schema tests for maintained transcript/glossary inputs. Example
|
parser and use schema tests for maintained transcript/glossary inputs. Example
|
||||||
|
|||||||
@@ -2,11 +2,28 @@
|
|||||||
|
|
||||||
## Client Boundary
|
## Client Boundary
|
||||||
|
|
||||||
`llm.NewRuntime` assembles one PromptKit engine for both roles. It loads embedded
|
`llm.NewRuntime` assembles one PromptKit engine for both roles. PromptKit's
|
||||||
prompt and schema assets, registers application backends, and inspects the
|
catalogs and defaults are followed by Audita's embedded prompt and schema
|
||||||
selected proposal and validation profiles offline. A role-bound `llm.RoleClient`
|
inputs, operator documents from `profiles_dir`, and the highest-priority
|
||||||
implements `structuredoutput.Client`; proposal and validator contracts alias that
|
application backend mapping. Audita owns backend endpoints, capacity settings,
|
||||||
interface. Both roles share engine-local backend capacity.
|
and resolved credentials; native PromptKit layering owns profile inheritance,
|
||||||
|
prompt overrides, and provider-specific fields. Imports are disabled. The
|
||||||
|
runtime inspects both selected roles offline, requiring complete model and
|
||||||
|
endpoint values and rejecting competing profile credential lookups. A
|
||||||
|
role-bound `llm.RoleClient` implements `structuredoutput.Client`; proposal and
|
||||||
|
validator contracts alias that interface. Both roles share engine-local backend
|
||||||
|
capacity.
|
||||||
|
|
||||||
|
Successful construction captures the final definitions and referenced message
|
||||||
|
contents. The runtime does not reload operator documents or message files during
|
||||||
|
calls. If final engine construction fails, an embedded-only diagnostic
|
||||||
|
construction distinguishes shipped asset failures from invalid operator input.
|
||||||
|
An asset failure remains an asset error; when embedded construction succeeds,
|
||||||
|
the original failure remains a configuration error. The diagnostic engine
|
||||||
|
never replaces or executes the requested configuration, and caller cancellation
|
||||||
|
skips this path. The captured inspection also supplies an internal source
|
||||||
|
ledger: all loaded host YAML plus effective, source-relative host
|
||||||
|
`content_file` resources are tracked for output-collision checks.
|
||||||
|
|
||||||
For each logical call, the adapter starts an operation context and a diagnostic
|
For each logical call, the adapter starts an operation context and a diagnostic
|
||||||
record before preparation. It supplies the reserved hardening text, the selected
|
record before preparation. It supplies the reserved hardening text, the selected
|
||||||
@@ -24,6 +41,10 @@ Counts distinguish provider attempts, transient retries, and structural
|
|||||||
repairs, including on errors. Secrets from all configured backends, including unselected
|
repairs, including on errors. Secrets from all configured backends, including unselected
|
||||||
ones, are available to the diagnostics redaction boundary.
|
ones, are available to the diagnostics redaction boundary.
|
||||||
|
|
||||||
|
CLI explain output and debug configuration origins share a value-free projection
|
||||||
|
of effective field origins. The [CLI contract](../cli.md) defines its optional
|
||||||
|
inheritance evidence fields.
|
||||||
|
|
||||||
## Orchestration
|
## Orchestration
|
||||||
|
|
||||||
CLI processing supplies two role clients from the same engine to the runner.
|
CLI processing supplies two role clients from the same engine to the runner.
|
||||||
|
|||||||
@@ -11,18 +11,27 @@
|
|||||||
- `validators.meaning_reversal_review` (`v1`)
|
- `validators.meaning_reversal_review` (`v1`)
|
||||||
- `validators.editorial_review` (`v1`)
|
- `validators.editorial_review` (`v1`)
|
||||||
|
|
||||||
PromptKit reads the embedded definitions and renders their system and user
|
PromptKit reads the embedded definitions as shipped defaults and renders their
|
||||||
messages from named string variables. The role-bound adapter supplies
|
messages from named string variables. Operator documents in `profiles_dir` may
|
||||||
|
contribute native keyed prompt definitions and override an embedded
|
||||||
|
prompt/version, including its messages and definition-level output settings.
|
||||||
|
The effective operator text is captured when the runtime is constructed; the
|
||||||
|
embedded assets remain the source of Audita's shipped task instructions. The
|
||||||
|
role-bound adapter supplies
|
||||||
`HardeningPolicy` alongside `ProposalContract` and `ValidationContract` from
|
`HardeningPolicy` alongside `ProposalContract` and `ValidationContract` from
|
||||||
embedded fragments under `assets/shared`, after caller variables so callers cannot replace those
|
embedded fragments under `assets/shared`, after caller variables so callers
|
||||||
instructions. Common literal-span and identity guidance lives in the proposal
|
cannot replace those instructions. Common literal-span and identity guidance
|
||||||
fragment; shared candidate-reference, context, and confidence guidance lives in
|
lives in the proposal fragment; shared candidate-reference, context, and
|
||||||
the validation fragment. Module-specific policies and examples remain in YAML.
|
confidence guidance lives in the validation fragment. Module-specific policies
|
||||||
Proposal prompts receive the section JSON, glossary JSON where used, and transcript description block.
|
and examples remain in YAML. Proposal prompts receive the section JSON, glossary
|
||||||
Validator prompts receive the correction payload and description block.
|
JSON where used, and transcript description block. Validator prompts receive
|
||||||
The prompt assets retain the prior task instructions and output-schema
|
the correction payload and description block.
|
||||||
association. The role-bound runtime supplies the complete JSON Schema output contract
|
The embedded prompt assets retain the shipped task instructions and
|
||||||
and repair count; the definition does not own an operator-facing repair default.
|
output-schema association. An operator override controls which template
|
||||||
|
variables its messages reference: supplying a shared variable does not append
|
||||||
|
its text when the effective message omits that reference. The role-bound runtime
|
||||||
|
supplies the complete JSON Schema output contract and repair count; the
|
||||||
|
definition does not own an operator-facing repair default.
|
||||||
The correction-set schema requires a string replacement and accepts an empty
|
The correction-set schema requires a string replacement and accepts an empty
|
||||||
string. Proposal guidance uses it for eligible span deletion and permits literal
|
string. Proposal guidance uses it for eligible span deletion and permits literal
|
||||||
spacing edits; validator guidance evaluates the full before/after segment under
|
spacing edits; validator guidance evaluates the full before/after segment under
|
||||||
|
|||||||
@@ -17,23 +17,32 @@ settings, and the [output contract](integrations/output-files.md) owns schemas.
|
|||||||
## Required Output Files
|
## Required Output Files
|
||||||
|
|
||||||
Reporting is enabled by default. For file transcript output, `errors.yml`,
|
Reporting is enabled by default. For file transcript output, `errors.yml`,
|
||||||
`recoveries.yml`, and `report.md` go in the transcript output's parent directory.
|
`recoveries.yml`, and `report.md` go in the transcript output's parent directory
|
||||||
For stdout transcript output, they go in `reporting.stdout_dir` (default `.`),
|
unless `--report-dir` is explicitly supplied. For stdout transcript output, they
|
||||||
overridden by `--report-dir`. `--reporting=false` suppresses all three reports
|
go in the effective `reporting.stdout_dir` (default `.`), including a
|
||||||
and removes their path requirements. It does not suppress the stderr summary or
|
`--report-dir` override. A config-file or overlay value for
|
||||||
|
`reporting.stdout_dir` alone does not redirect file-output reports. An active
|
||||||
|
selected report directory must be nonblank. `--reporting=false` suppresses all
|
||||||
|
three reports and leaves their destination inactive, so it is not probed merely
|
||||||
|
because `--report-dir` was supplied. It does not suppress the stderr summary or
|
||||||
fatal errors. Stdout contains only transcript JSON; file output leaves it empty.
|
fatal errors. Stdout contains only transcript JSON; file output leaves it empty.
|
||||||
|
|
||||||
Existing report directories and parents for regular transcript files must be
|
Existing report directories and parents for regular transcript files must be
|
||||||
writable. Audita prevents report writes from overwriting transcript, glossary,
|
writable. Audita prevents report writes from overwriting transcript, glossary,
|
||||||
loaded configuration, or loaded profile sources, and rejects conflicting output
|
loaded application configuration, host YAML discovered under `profiles_dir`,
|
||||||
destinations. The final transcript may intentionally replace its own input after
|
or host message files referenced by effective PromptKit `content_file` fields,
|
||||||
successful processing; other input/configuration/profile sources remain protected.
|
and rejects conflicting output destinations. Source paths come from the
|
||||||
|
runtime's captured configuration; unrelated files in the selected directory
|
||||||
|
remain eligible output destinations. The final transcript may intentionally
|
||||||
|
replace its own input after successful processing; other input/configuration/
|
||||||
|
profile sources remain protected.
|
||||||
Preparation never truncates an existing transcript. Device and pipe destinations
|
Preparation never truncates an existing transcript. Device and pipe destinations
|
||||||
(such as `/dev/null` and named pipes) receive ordinary writes at publication time;
|
(such as `/dev/null` and named pipes) receive ordinary writes at publication time;
|
||||||
they are not opened during preflight or replaced with regular files. A pipe write
|
they are not opened during preflight or replaced with regular files. A pipe write
|
||||||
can wait for its reader, and stream write errors are fatal. With reporting enabled,
|
can wait for its reader, and stream write errors are fatal. With reporting enabled,
|
||||||
reports still use the output path's parent; disable reporting when that placement
|
reports still use the output path's parent unless an explicit `--report-dir`
|
||||||
is unsuitable for a stream destination.
|
selects another directory; disable reporting when that placement is unsuitable
|
||||||
|
for a stream destination.
|
||||||
|
|
||||||
With reporting enabled, all three reports are initialized as `in_progress` before
|
With reporting enabled, all three reports are initialized as `in_progress` before
|
||||||
input processing. Output is encoded fully before publication. Each regular output
|
input processing. Output is encoded fully before publication. Each regular output
|
||||||
@@ -48,10 +57,12 @@ finalized where possible. An uncatchable termination can leave `in_progress`.
|
|||||||
|
|
||||||
## Diagnostics Directory
|
## Diagnostics Directory
|
||||||
|
|
||||||
`--debug` enables detailed evidence independently of reporting. One directory
|
`--debug` or `diagnostics.debug` enables detailed evidence independently of
|
||||||
named `run-<id>` is created under `diagnostics.work_dir` (default `/tmp/audita`)
|
reporting. One directory named `run-<id>` is created under
|
||||||
and retained on success or failure. Without debug, no directory or payload
|
`diagnostics.work_dir` (default `/tmp/audita`), which `--work-dir` can override,
|
||||||
files are created. The parent is created when needed only for debug. Failure to
|
and retained on success or failure. Selecting `--work-dir` alone does not enable
|
||||||
|
capture. Without debug, no directory or payload files are created. The parent
|
||||||
|
is created when needed only for debug. Failure to
|
||||||
write optional debug evidence prints a redacted stderr diagnostic but does not
|
write optional debug evidence prints a redacted stderr diagnostic but does not
|
||||||
change correction decisions or the exit status.
|
change correction decisions or the exit status.
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
|||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.4.2`.
|
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -24,9 +24,9 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|||||||
|
|
||||||
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
[GPL v3](licenses/GPL-3.0.txt) |
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f/COPYRIGHT).
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f).
|
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
|
||||||
|
|
||||||
## Go runtime and standard library
|
## Go runtime and standard library
|
||||||
|
|
||||||
@@ -142,7 +142,7 @@ License and notices: [Apache 2.0](licenses/Apache-2.0.txt).
|
|||||||
|
|
||||||
## PromptKit
|
## PromptKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.11.2`.
|
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.12.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -150,37 +150,37 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|||||||
|
|
||||||
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
[GPL v3](licenses/GPL-3.0.txt) |
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2/COPYRIGHT).
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3/COPYRIGHT).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2).
|
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3).
|
||||||
|
|
||||||
## Promptkit OpenRouter Catalog
|
## Promptkit OpenRouter Catalog
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2` at `v2.20260926.0`.
|
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3` at `v3.20260930.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
|
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3).
|
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
|
||||||
|
|
||||||
## Promptkit Rakestrawhome Catalog
|
## Promptkit Rakestrawhome Catalog
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2` at `v2.20260926.0`.
|
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3` at `v3.20260930.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
|
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596).
|
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
|
||||||
|
|
||||||
## yaml.v3
|
## yaml.v3
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
# Release Notes
|
# Release Notes
|
||||||
|
|
||||||
|
- [Audita v3.0.0](v3.0.0.md)
|
||||||
- [Audita v2.0.1](v2.0.1.md)
|
- [Audita v2.0.1](v2.0.1.md)
|
||||||
- [Audita v2.0.0](v2.0.0.md)
|
- [Audita v2.0.0](v2.0.0.md)
|
||||||
- [Audita v1.1.0](v1.1.0.md)
|
- [Audita v1.1.0](v1.1.0.md)
|
||||||
|
|||||||
59
docs/releases/v3.0.0.md
Normal file
59
docs/releases/v3.0.0.md
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# Audita v3.0.0
|
||||||
|
|
||||||
|
Audita v3 adopts native configuration inheritance and layered PromptKit
|
||||||
|
definitions, and allows transcript and report destinations to be selected
|
||||||
|
independently. This is an intentional breaking release: existing PromptKit
|
||||||
|
profile files require migration from the v2 release format.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- ConfigKit is updated from v0.4.2 to v0.5.0. Application configuration supports
|
||||||
|
native sibling inheritance, and configuration explanations include value-free
|
||||||
|
inheritance provenance.
|
||||||
|
- PromptKit is updated from v0.11.2 to v0.12.0, with both backend catalogs at
|
||||||
|
`v3.20260930.0`. Native keyed definitions support profile inheritance and
|
||||||
|
operator prompt overrides. The selected `profiles_dir` can contain profiles,
|
||||||
|
backends, and prompts; application backend endpoints, capacity, and credentials
|
||||||
|
remain authoritative.
|
||||||
|
- Effective definitions and referenced prompt message contents are captured at
|
||||||
|
runtime construction. Consumed host configuration and message files are
|
||||||
|
protected from output collisions. Audita retains its per-call schemas, domain
|
||||||
|
validators, retry and repair budgets, and resilient processing behavior.
|
||||||
|
- An explicit `--report-dir` now selects reports for either file or stdout
|
||||||
|
transcript output. File output without that flag still places reports beside
|
||||||
|
the transcript. This supports Narratio's independent output destinations;
|
||||||
|
debug capture remains independently selectable.
|
||||||
|
|
||||||
|
## Upgrading from v2
|
||||||
|
|
||||||
|
1. Convert each old PromptKit profile to a keyed definition under
|
||||||
|
`profiles: {<profile-id>: ...}`, remove its inner `id`, and replace
|
||||||
|
`base_profile` with `extends`. Profile IDs come from mapping keys rather than
|
||||||
|
filenames. No compatibility loader or automatic conversion is provided.
|
||||||
|
2. Review explicit zero, omitted, and null generation controls. For example,
|
||||||
|
`temperature: 0` now sends an explicit zero, omission leaves the control
|
||||||
|
unspecified, and null suppresses an inherited control. A zero profile timeout
|
||||||
|
disables PromptKit's generation deadline; Audita's operation deadline still
|
||||||
|
applies.
|
||||||
|
3. Select a configuration root whose final definitions are valid. PromptKit now
|
||||||
|
validates unused definitions too, including backend references, and conflicting
|
||||||
|
values within a directory source group are errors. The maintained examples
|
||||||
|
use separate [local](../../examples/profiles/local/) and
|
||||||
|
[service](../../examples/profiles/service/) roots. Runtime source edits take
|
||||||
|
effect on the next invocation.
|
||||||
|
4. Rename any application backend literally named `extends` and update its
|
||||||
|
profile references. ConfigKit now reserves that key for inheritance.
|
||||||
|
5. Validate the migrated installation with
|
||||||
|
`audita config validate --config /path/to/config.yml`, including any selected
|
||||||
|
testing overlay. Application configuration continues to use `version: 2`.
|
||||||
|
|
||||||
|
The [configuration reference](../config.md#promptkit-profiles-and-operator-prompts)
|
||||||
|
contains the migrated format and presence rules. Start from the maintained
|
||||||
|
[local](../../examples/minimal-config.yml),
|
||||||
|
[service](../../examples/production-config.yml), or
|
||||||
|
[application inheritance](../../examples/configkit-inheritance.yml) examples.
|
||||||
|
The [CLI reference](../cli.md) describes report selection, and the
|
||||||
|
[release procedure](../release.md) describes source publication and asynchronous
|
||||||
|
Linux binary delivery.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/docs/oss/attribution.md)
|
||||||
@@ -1,171 +1,133 @@
|
|||||||
# ConfigKit Adoption Roadmap
|
# ConfigKit v0.5.0 Adoption Roadmap
|
||||||
|
|
||||||
**Last updated:** 2026-09-24
|
**Last updated:** 2026-10-03
|
||||||
|
|
||||||
**Status:** Production implementation complete. The selected configuration and
|
**Status:** Complete. The implementation and tests are committed through
|
||||||
overlay model is implemented; [implementation evidence](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md)
|
`8175ed8`. Final documentation and example changes were included in the full
|
||||||
records verification. No release has been published.
|
build, test, example-validation, and local-link checks before the documentation
|
||||||
|
completion commit. The [implementation plan](implementation.md) records the
|
||||||
|
ordered work and verification.
|
||||||
|
|
||||||
## Purpose and Direction
|
## Purpose and Baseline
|
||||||
|
|
||||||
Adopt ConfigKit to harmonize Audita's configuration policy with the other
|
Adopt ConfigKit v0.5.0 as an independently reviewable step before PromptKit
|
||||||
applications in the suite. Consistent configuration conventions are the primary
|
v0.12.0. Use the library's configuration and inheritance conventions while
|
||||||
benefit; reducing application-owned configuration infrastructure is a secondary
|
preserving Audita's domain ownership and existing operator workflows.
|
||||||
benefit and a useful measure of integration quality.
|
|
||||||
|
|
||||||
This is an intentional breaking migration. There is no goal of backward
|
At the reviewed Audita revision `6ae1d7cd5f548b26de5787372d3f2249eaa4757d`,
|
||||||
compatibility with existing installations, configuration files, CLI flags,
|
[go.mod](../../go.mod) selects ConfigKit v0.4.2, PromptKit v0.11.2, and Go
|
||||||
environment variables, or configuration-inspection output. Provide a migration
|
1.25.5. ConfigKit already owns loading, presence, secret resolution, and
|
||||||
guide instead of legacy loaders, aliases, dual operating modes, or an ongoing
|
configuration inspection. Audita's `internal/core/config` supplies typed
|
||||||
compatibility layer. Retain existing syntax only when it independently suits the
|
settings, defaults, derived values, and domain validation; `internal/cli`
|
||||||
new model, not because compatibility requires it.
|
selects files, overlays, and explicit overrides. The
|
||||||
|
[configuration contract](../config.md) and [CLI contract](../cli.md) own the
|
||||||
|
implemented behavior.
|
||||||
|
|
||||||
The initial review and prototype used ConfigKit v0.3.1, then verified v0.4.0
|
The reviewed upstream release is
|
||||||
selected overlays. Those findings remain historical; the prototype was removed
|
[ConfigKit v0.5.0](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.5.0/docs/releases/v0.5.0.md),
|
||||||
after production integration. The application depends on a versioned module,
|
commit `dc09959cca4e105f49ec316c6d4ba585bd2340bd`. It adds layered underlays,
|
||||||
without a sibling checkout or local module replacement.
|
caller-owned filesystem sources, mapping sources, source resource contexts, and
|
||||||
|
built-in sibling inheritance. The Go minimum and YAML dependency do not change.
|
||||||
|
Ordinary `Load` remains available; adopting `LoadLayered` is optional.
|
||||||
|
|
||||||
The shared boundary/prototype unit is complete. Its [selected contract, inventory,
|
This roadmap supersedes the completed original adoption plan at this path.
|
||||||
and evidence](library-boundary-findings.md) define the detailed target alongside
|
The [historical roadmap](https://gitea.maximumdirect.net/eric/audita/src/commit/6ae1d7cd5f548b26de5787372d3f2249eaa4757d/docs/roadmap/configkit.md)
|
||||||
this roadmap. Production configuration migration is complete.
|
records that earlier migration. Its coordinated breaking-refactor instructions
|
||||||
|
and release target do not govern this incremental upgrade.
|
||||||
|
|
||||||
## Target End State
|
## Target End State
|
||||||
|
|
||||||
- ConfigKit owns strict YAML loading, input presence, default application, explicit
|
- Audita uses the released ConfigKit v0.5.0 module, with no local replacement.
|
||||||
override mechanics, recorded resolution, secret references, and safe inspection.
|
PromptKit stays at v0.11.2 throughout this adoption unit.
|
||||||
- Audita owns its configuration types, field meanings, defaults, schema-version
|
- Configuration retains the current file-selection, overlay, explicit-override,
|
||||||
policy, file selection, CLI parsing, domain validation, and runtime initialization.
|
resolution, and validation flow, subject to the reserved-key compatibility
|
||||||
- Adopt ConfigKit's precedence and validation lifecycle rather than reproduce
|
impact described below. Retain the application schema version, CLI interface,
|
||||||
Audita's current per-layer validation and environment-overlay behavior.
|
and main-file-relative resource rules defined in the current contracts.
|
||||||
- General configuration environment overrides are retired. Credentials use typed
|
- Application configuration can use ConfigKit's native sibling `extends`
|
||||||
environment-name secret references; supplied references must resolve according
|
relationships. Audita does not implement a second inheritance resolver,
|
||||||
to ConfigKit's contract. File selection follows the application-owned
|
special backend-only syntax, or an opt-in switch around the library feature.
|
||||||
discovery rules in the selected boundary.
|
- Inherited values participate in presence-aware defaults, domain validation,
|
||||||
- Standardize duration representations, explicit empty/null/zero handling, and
|
secret resolution, and safe inspection just like other effective input.
|
||||||
inheritance on the shared conventions. Express genuine Audita domain rules
|
In particular, inherited backend settings must not be overwritten merely
|
||||||
through the supported resolution and validation APIs.
|
because they were not written directly on the child.
|
||||||
- Retained domain-setting flags and proposal/validation profile flags become
|
- Effective settings and explanations remain useful and secret-safe when
|
||||||
explicit ConfigKit overrides according to the migration inventory. Remove old
|
values are inherited. Preserve existing inspection fields and add compact,
|
||||||
provider/key/retry/concurrency flags and aliases. Credentials use secret references.
|
value-free inheritance evidence from ConfigKit's metadata rather than
|
||||||
- Use ConfigKit's redacted effective output and provide source/explanation access
|
reconstructing provenance from YAML.
|
||||||
through the selected views below. Legacy JSON inspection is replaced.
|
- Canonical documentation and a maintained example explain the adopted
|
||||||
Diagnostic configuration snapshots use the same new model.
|
inheritance behavior. Attribution identifies the exact new upstream source.
|
||||||
- Delete superseded loading and override machinery. Do not hide the old model
|
|
||||||
behind adapters that defeat the adoption's consistency benefit.
|
|
||||||
|
|
||||||
Keep Audita's domain capabilities, such as ordered and repeated correction modules,
|
The upstream [configuration contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.5.0/docs/config.md#sibling-inheritance)
|
||||||
expressible in the new model. Their existing configuration spelling is not binding.
|
owns generic inheritance mechanics. The integration must account for three
|
||||||
Transcript/glossary formats, processing algorithms, and output schemas are outside
|
consequences: `extends` is reserved automatically throughout input mappings;
|
||||||
this migration unless later planning identifies a concrete dependency or an
|
relationships resolve after source precedence and before typed decoding; and
|
||||||
explicitly coordinated change.
|
inherited zero, false, empty, or null values count as explicit input. An inherited
|
||||||
|
value still has to satisfy Audita's field-specific rules. Parent definitions
|
||||||
|
also undergo ordinary application validation; inheritance does not introduce
|
||||||
|
abstract, validation-exempt backend entries.
|
||||||
|
|
||||||
|
## Scope and Adoption Boundary
|
||||||
|
|
||||||
|
Keep the existing `Load` integration unless implementation reveals a concrete
|
||||||
|
need to change it. Source-backed defaults, additional operator underlay tiers,
|
||||||
|
imports, and source-relative reinterpretation of application paths are outside
|
||||||
|
this feature. Their availability upstream does not require expanding Audita's
|
||||||
|
configuration surface. Imports and secret-directory bootstrap retain their
|
||||||
|
current disabled state.
|
||||||
|
|
||||||
|
ConfigKit inheritance applies to Audita configuration in this step. PromptKit
|
||||||
|
profile formats, profile inheritance, engine construction, embedded prompts,
|
||||||
|
and backend catalogs remain governed by the existing PromptKit integration.
|
||||||
|
There is no application-owned translation of profile documents in this work.
|
||||||
|
|
||||||
|
This is an independent ConfigKit adoption ahead of the planned PromptKit
|
||||||
|
migration. A separate public release between the two units is optional.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- The released ConfigKit v0.5.0 dependency and exact source attribution are
|
||||||
|
adopted; PromptKit remains at v0.11.2 with its existing catalogs.
|
||||||
|
- Native inheritance preserves effective endpoints, capacity, explicit zero
|
||||||
|
queue capacity, child overrides, and surviving credential references through
|
||||||
|
Audita's loading, resolution, and validation boundary.
|
||||||
|
- Selected overlays and explicit overrides retain their precedence; omitted
|
||||||
|
validation-role selection still derives from the final proposal selection.
|
||||||
|
- Null can remove a child's credential while surviving references on parents
|
||||||
|
and other unselected backends still follow the existing credential contract.
|
||||||
|
- Selected roles use inherited backend settings and share its configured
|
||||||
|
capacity under the existing PromptKit runtime contract.
|
||||||
|
- Effective settings remain redacted, and both CLI explanations and debug
|
||||||
|
configuration origins expose compact, value-free inheritance evidence while
|
||||||
|
retaining existing fields and application-derived explanations.
|
||||||
|
- A maintained complete example and canonical documentation describe native
|
||||||
|
inheritance, its field-validation implications, and the reserved-key
|
||||||
|
compatibility impact. Existing workflows and applicable offline checks pass.
|
||||||
|
|
||||||
|
## Completion and Release Treatment
|
||||||
|
|
||||||
|
Adoption is complete when the released dependency is pinned, existing workflows
|
||||||
|
pass, representative inheritance works through loading and runtime wiring,
|
||||||
|
inspection includes value-free inheritance evidence and remains secret-safe,
|
||||||
|
and the example, canonical docs, and attribution match the implementation. No
|
||||||
|
PromptKit migration or provider behavior change is required to close this unit.
|
||||||
|
|
||||||
|
Release classification remains provisional. A minor release would be appropriate
|
||||||
|
for a backward-compatible addition, but the newly reserved `extends` key can
|
||||||
|
invalidate currently valid application configuration. For example, Audita
|
||||||
|
currently permits a backend literally named `extends`; ConfigKit v0.5.0 treats
|
||||||
|
that mapping key as an inheritance directive rather than a backend ID. Do not
|
||||||
|
promise blanket compatibility or settle on a minor release until the final
|
||||||
|
compatibility review resolves this impact under the
|
||||||
|
[release policy](../policy/release.md). Describe any required migration in the
|
||||||
|
canonical documentation and release notes when the upgrade ships. Do not select
|
||||||
|
or publish a release as part of roadmap preparation. A later PromptKit
|
||||||
|
migration's profile-format compatibility decision belongs to that separate
|
||||||
|
feature.
|
||||||
|
|
||||||
## Selected Overlays for Testing
|
## Selected Overlays for Testing
|
||||||
|
|
||||||
Enable ConfigKit's optional selected-overlay feature in the migration. A testing
|
This heading preserves links from historical adoption documents. The original
|
||||||
configuration that selects cheaper, faster LLMs is the initial use case. ConfigKit
|
[overlay plan](https://gitea.maximumdirect.net/eric/audita/src/commit/6ae1d7cd5f548b26de5787372d3f2249eaa4757d/docs/roadmap/configkit.md#selected-overlays-for-testing)
|
||||||
**overlays** modify application settings; PromptKit **execution profiles** still
|
is historical; current overlay behavior belongs to [configuration](../config.md).
|
||||||
own model and generation settings. Keep `profiles_dir`, the PromptKit `profiles/`
|
This upgrade preserves that workflow and adds native ConfigKit inheritance
|
||||||
directory, and proposal/validation profile names unchanged by this terminology
|
without reviving the earlier combined migration plan.
|
||||||
change.
|
|
||||||
|
|
||||||
The planned interface is `--overlay <name>` on `process`, `config validate`, and
|
|
||||||
`config print-effective`. Treat selection as invocation state passed to
|
|
||||||
`Options.Overlay`, not as another persistent setting or environment overlay.
|
|
||||||
Follow ConfigKit v0.4.0's selection contract: omission, an empty name, or `default`
|
|
||||||
selects no overlay; a named selection loads `overlays/<name>.yml` beneath the selected
|
|
||||||
configuration root. A missing selected overlay is an error. Use the library's
|
|
||||||
single-overlay semantics without adding stacking, inheritance, or legacy-directory
|
|
||||||
fallback.
|
|
||||||
|
|
||||||
Precedence is application defaults, main configuration, selected overlay, then
|
|
||||||
explicit setting flags. Perform Audita's role inheritance and semantic validation
|
|
||||||
on the resolved configuration. ConfigKit's base parsing and schema-version checks
|
|
||||||
still precede overlay application; overlays cannot replace the schema version or
|
|
||||||
contain ConfigKit directives. General environment-variable overrides remain retired.
|
|
||||||
Imports and secret-directory bootstrap remain disabled.
|
|
||||||
|
|
||||||
Provide a maintained `overlays/testing.yml` example that selects testing execution
|
|
||||||
profiles through `llm.proposal_profile` and `llm.validation_profile`. Model names and
|
|
||||||
generation controls belong in the corresponding PromptKit profile files. Set both
|
|
||||||
role selections in this example so a base configuration's explicit validation
|
|
||||||
profile cannot accidentally retain the production model. Normal omission-based
|
|
||||||
validation-profile inheritance continues to apply to the final effective settings.
|
|
||||||
The example must make its backend/credential requirements clear and use no real
|
|
||||||
secrets. Model choice is operator-controlled; “testing” does not promise a price,
|
|
||||||
performance level, or equivalent correction quality.
|
|
||||||
|
|
||||||
Overlays may modify other ordinary application settings supported by the schema;
|
|
||||||
they are not restricted to profile selection. The resulting configuration must
|
|
||||||
still satisfy shared backend capacity, credential, and domain rules. Validation,
|
|
||||||
effective-config inspection, selected-profile inspection, and processing must all
|
|
||||||
use the same selected overlay. Preserve overlay provenance in configuration
|
|
||||||
explanations and diagnostics while retaining secret redaction.
|
|
||||||
|
|
||||||
The [recorded overlay results](library-boundary-findings.md#selected-overlay-extension-2026-09-24)
|
|
||||||
prove the representative load, inspection, and HTTP path. Production acceptance
|
|
||||||
requires the same selection to reach both actual LLM roles, with explicit flags
|
|
||||||
winning over overlays and surviving secret references resolved/redacted correctly.
|
|
||||||
The isolated prototype is evidence, not a second supported runtime.
|
|
||||||
|
|
||||||
## Inspection and Invocation Contract
|
|
||||||
|
|
||||||
`process`, `config validate`, and `config print-effective` share file/overlay
|
|
||||||
selection and retained setting overrides. Invocation I/O paths remain process-only.
|
|
||||||
Only explicitly supplied flags override settings. Validation requires an explicit
|
|
||||||
config file; processing and inspection use the selected discovery order. Help is
|
|
||||||
available before configuration loading. All three commands validate selected
|
|
||||||
profiles without requiring a provider call during setup.
|
|
||||||
|
|
||||||
`config print-effective --view settings|profiles|explain` exposes three YAML views:
|
|
||||||
|
|
||||||
- `settings` (default): ConfigKit's redacted effective configuration.
|
|
||||||
- `profiles`: an Audita-owned summary of each selected role's profile, backend,
|
|
||||||
model, temperature, token limit, top-p, timeout, service tier, and reasoning
|
|
||||||
effort where present. Arbitrary extra provider parameters and credentials are
|
|
||||||
omitted; this is a selected-controls view, not every execution input.
|
|
||||||
- `explain`: field paths, sources, and derivation records from ConfigKit, without
|
|
||||||
resolved secret values.
|
|
||||||
|
|
||||||
A ConfigKit snapshot alone does not describe profile-owned controls. Profile
|
|
||||||
inspection must enforce the registered-backend and credential boundary. Diagnostic
|
|
||||||
snapshots use these safe views and preserve overlay/explicit-override provenance.
|
|
||||||
The migration guide explains the intentional replacement of legacy inspection JSON.
|
|
||||||
|
|
||||||
## Integration and Completion Criteria
|
|
||||||
|
|
||||||
ConfigKit and PromptKit form one intentional breaking refactor. Intermediate
|
|
||||||
commits need not build or run. The target contains no temporary old-runtime adapter,
|
|
||||||
compatibility shim, processing gate, dual loader, or automatic legacy conversion.
|
|
||||||
The [combined feature roadmap](library-adoption.md) owns execution/reporting intent;
|
|
||||||
[implementation.md](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) alone owns stage order and implementation tasks.
|
|
||||||
|
|
||||||
The final application loads all retained domain settings, assembles both selected
|
|
||||||
profiles, and processes through PromptKit. Controlled HTTP evidence must connect
|
|
||||||
configuration selection to actual requests, domain outcomes, safe diagnostics,
|
|
||||||
and shared finite backend capacity. Maintained examples and one migration guide
|
|
||||||
cover the complete file/flag/environment transition. Obsolete application-owned
|
|
||||||
loading, transport, rendering, and scheduler machinery is removed.
|
|
||||||
|
|
||||||
The combined target is Audita v2.0.0. Release compliance and publication remain
|
|
||||||
last and separate from implementing the refactor. Retention and transcript timestamp
|
|
||||||
changes are not included; the selected retry/repair contract resolves the retry
|
|
||||||
follow-up in [runtime behavior](runtime-behavior.md).
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- Pin ConfigKit v0.4.0 in production and align Go/toolchain requirements. ConfigKit requires
|
|
||||||
Go 1.25; the combined PromptKit prototype requires Go 1.25.5. Adopting these libraries
|
|
||||||
does not automatically remove Audita's existing YAML dependency used for glossary parsing.
|
|
||||||
- Implement the selected configuration and final PromptKit runtime directly,
|
|
||||||
without temporary runtime adapters, compatibility shims, or process gates.
|
|
||||||
- Enable selected overlays as specified above. Keep imports and secret-directory
|
|
||||||
bootstrap disabled unless a separate concrete requirement justifies them.
|
|
||||||
- Validation and inspection resolve surviving credential references and fail on
|
|
||||||
missing or empty values; no credential-free bypass is planned.
|
|
||||||
- Inspection and diagnostics use the safe selected-profile view and ConfigKit
|
|
||||||
provenance described above. Changed formats are covered by the migration guide.
|
|
||||||
|
|
||||||
The eventual completion test is that Audita follows the suite's ConfigKit
|
|
||||||
conventions while retaining its intended domain capabilities, with a clear
|
|
||||||
migration guide and no ongoing obligation to support the previous configuration
|
|
||||||
model.
|
|
||||||
|
|||||||
@@ -1,162 +0,0 @@
|
|||||||
# Partial Success for Validator Decision Sets
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-25
|
|
||||||
|
|
||||||
**Status:** Implemented. This document owns the target behavior and user intent.
|
|
||||||
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/91d9be68a7bb3a102900253fdfd8438f0a43f666/docs/roadmap/implementation.md) owns concrete decisions, ordered work,
|
|
||||||
and verification.
|
|
||||||
|
|
||||||
## Purpose and User Intent
|
|
||||||
|
|
||||||
Preserve usable validator decisions when other entries in the same model response
|
|
||||||
have missing, duplicate, or unexpected correction indices. One defective entry
|
|
||||||
should not prevent unrelated candidates from completing their required validation.
|
|
||||||
This extends Audita's existing partial-success approach without another model call.
|
|
||||||
|
|
||||||
Implement this simpler improvement before reconsidering guided domain recovery.
|
|
||||||
Missing validation must never authorize applying a correction, but neither should
|
|
||||||
it invalidate a different candidate's unambiguous approval or editorial rejection.
|
|
||||||
The existing [resilience design](resilience.md) remains the foundation.
|
|
||||||
|
|
||||||
## Historical Baseline
|
|
||||||
|
|
||||||
The reviewed application baseline is `d2a6fcf`:
|
|
||||||
|
|
||||||
- PromptKit validates the complete model response against Audita's embedded JSON
|
|
||||||
Schema and performs its configured structural repairs. Only structurally valid,
|
|
||||||
decoded responses reach domain decision mapping.
|
|
||||||
- `mapLLMResponseToDecisions` in
|
|
||||||
[`llm_validators.go`](../../internal/framework/validators/llm_validators.go)
|
|
||||||
requires exactly one decision for every requested correction index. An empty
|
|
||||||
decision list, missing index, duplicate index, or unexpected index returns an
|
|
||||||
error instead of retaining usable decisions.
|
|
||||||
- `LLMBackedValidator.Validate` consequently marks the entire batch unavailable
|
|
||||||
with `validation_decisions_invalid`. Other batches continue, but usable decisions
|
|
||||||
within that batch are lost.
|
|
||||||
- The runner's cardinality checks enforce an internal contract: each candidate
|
|
||||||
receives one outcome from each validator it reaches. These checks apply to
|
|
||||||
Audita-owned results, not directly to raw model entries.
|
|
||||||
- Deterministic validators already filter individual proposals. Unpreviewable
|
|
||||||
candidates and oversized validation inputs are isolated; recoverable provider
|
|
||||||
failures remain batch-local; application skips individual conflicts.
|
|
||||||
- Ordinary reports distinguish failed operations from successful operations with
|
|
||||||
recoveries. Several failed operations in one stage/chunk share one work-unit
|
|
||||||
error record. The [output contract](../integrations/output-files.md) owns the
|
|
||||||
current counts and serialized representation.
|
|
||||||
|
|
||||||
The defect addressed here is the all-or-nothing domain mapping boundary, not
|
|
||||||
validation itself or the runner's internal consistency checks.
|
|
||||||
|
|
||||||
## Target Decision Semantics
|
|
||||||
|
|
||||||
Reconcile each structurally valid response against the indices actually requested
|
|
||||||
in that particular validator batch. Correction indices, rather than segment IDs
|
|
||||||
or positions in the response array, identify candidates. Several corrections may
|
|
||||||
legitimately target the same segment.
|
|
||||||
|
|
||||||
| Response condition | Target behavior |
|
|
||||||
| --- | --- |
|
|
||||||
| Exactly one valid decision for a requested index | Preserve its approval or rejection and associated decision evidence. |
|
|
||||||
| Requested index omitted | Produce an explicit validation-unavailable outcome for that candidate. |
|
|
||||||
| Requested index appears more than once | Treat that candidate's validation as unavailable; preserve unrelated decisions. |
|
|
||||||
| Index was not requested in this batch | Ignore the extraneous entry and record the handled defect. |
|
|
||||||
| Empty decision list for a nonempty request | Mark all requested candidates unavailable; do not fail the run. |
|
|
||||||
| Decisions appear in a different order | Match by identity and retain deterministic internal results. |
|
|
||||||
|
|
||||||
Duplicate handling does not choose the first or last answer, nor infer a verdict
|
|
||||||
by voting. Even identical duplicates follow the agreed duplicate-index handling;
|
|
||||||
accepting them specially is not needed for this feature. Duplicate extraneous
|
|
||||||
indices remain extraneous entries, not reasons to invalidate requested candidates.
|
|
||||||
|
|
||||||
An unavailable candidate remains unapplied and does not proceed as approved to
|
|
||||||
later validators. It is distinct from a valid `approved: false` decision, which
|
|
||||||
is ordinary editorial processing. Valid approvals continue through the remaining
|
|
||||||
required validators and apply-time checks; this feature grants no bypass.
|
|
||||||
|
|
||||||
The mapper returns one internal outcome for every requested candidate and no
|
|
||||||
outcome for an extraneous index. Preserve strict runner cardinality checks so
|
|
||||||
application bugs are still detected. Do not convert genuine internal invariant
|
|
||||||
failures, cancellation, or fatal setup failures into tolerated model defects.
|
|
||||||
|
|
||||||
## Outcomes and Reporting
|
|
||||||
|
|
||||||
Keep existing work-unit aggregation and operation accounting:
|
|
||||||
|
|
||||||
- If any requested candidate lacks an unambiguous usable decision, record a
|
|
||||||
validation-operation error identifying the affected requested indices. Other
|
|
||||||
candidates' usable decisions from that operation remain effective. An operation
|
|
||||||
error does not mean every candidate in its batch failed.
|
|
||||||
- If all requested candidates have usable decisions and extra entries were
|
|
||||||
ignored, record a successful operation with a recovery for that handled defect.
|
|
||||||
This remains a recovery even if all legitimate decisions are editorial rejections.
|
|
||||||
- If missing/duplicate requested decisions and ignored extras coexist, retain the
|
|
||||||
operation error. Follow the existing rule that failed operations do not also
|
|
||||||
produce recovery records; retain extra-entry details in debug evidence.
|
|
||||||
- A valid complete response with no handled defect adds neither error nor recovery.
|
|
||||||
Valid editorial rejection alone never adds either category.
|
|
||||||
- Preserve actual structural repair and service-retry counts. On successful
|
|
||||||
operations, consolidate applicable recovery reasons in one recovery record.
|
|
||||||
Do not count reconciliation as another LLM operation or structural repair.
|
|
||||||
|
|
||||||
Error attribution must distinguish affected candidates from the rest of the batch.
|
|
||||||
Debug evidence should preserve supplied decisions, ignored entries, and the reason
|
|
||||||
an expected candidate became unavailable. Do not fabricate candidate ledger entries
|
|
||||||
for unknown indices or mislabel missing decisions as deliberate editorial rejection.
|
|
||||||
Ordinary reports must retain safe metadata without raw transcript or response text.
|
|
||||||
|
|
||||||
Use a decision-specific recovery reason rather than treating ignored validator
|
|
||||||
entries as defective correction proposals. Concrete reason names and minimal data
|
|
||||||
model changes belong in the implementation plan. Keep YAML, Markdown, and CLI
|
|
||||||
counts consistent, with the existing reporting-disable and independent debug options.
|
|
||||||
|
|
||||||
## Ownership and Target Architecture
|
|
||||||
|
|
||||||
Audita's validator layer reconciles structurally valid model responses into
|
|
||||||
complete internal candidate outcomes. The runner preserves strict internal
|
|
||||||
cardinality and advances only approved candidates. Reporting consumes operation
|
|
||||||
facts without treating partial validation failure as loss of every decision.
|
|
||||||
Debug evidence distinguishes unavailable validation from editorial rejection.
|
|
||||||
|
|
||||||
The schema/transport boundary remains with PromptKit; reconciliation needs no
|
|
||||||
additional model call. Concrete helper structures, reason names, mapping changes,
|
|
||||||
and verification steps belong in the [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/91d9be68a7bb3a102900253fdfd8438f0a43f666/docs/roadmap/implementation.md).
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
PromptKit's whole-response structural validation and repair remain unchanged.
|
|
||||||
A malformed entry, including an extraneous entry that violates the schema, may
|
|
||||||
still invalidate the response and exhaust structural repairs before reconciliation
|
|
||||||
can run. Ignoring schema-valid extras does not promise salvage from malformed JSON,
|
|
||||||
missing required fields, wrong types, or out-of-range values.
|
|
||||||
|
|
||||||
General per-entry structural salvage is a separate design question, not part of
|
|
||||||
this fix. Likewise, guided regeneration remains deferred for reconsideration after
|
|
||||||
this change; no new retries, feedback prompts, budgets, upstream dependency changes,
|
|
||||||
or operator controls are required here.
|
|
||||||
|
|
||||||
Preserve module scope, literal replacement behavior, fixed chunks, stage barriers,
|
|
||||||
span-deletion safeguards, backend capacity limits, and fatal cancellation behavior.
|
|
||||||
No additional restrictions on valid corrections or guarantees about model judgment
|
|
||||||
are introduced. This work does not itself authorize tagging or publishing v2.0.0.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- A schema-valid response can retain usable decisions despite missing, duplicate,
|
|
||||||
or unexpected indices elsewhere in the same batch.
|
|
||||||
- Only the affected requested candidates become unavailable; extras never create
|
|
||||||
candidates or attach decisions to a different batch's candidates.
|
|
||||||
- Valid negative decisions remain editorial outcomes, and every applied correction
|
|
||||||
completes all required validation.
|
|
||||||
- Internal validator results satisfy existing strict cardinality checks.
|
|
||||||
- Missing/duplicate requested decisions yield accurately attributed errors without
|
|
||||||
losing valid work. Extra-only defects yield a consolidated recovery; mixed
|
|
||||||
defects follow existing failed-operation reporting semantics.
|
|
||||||
- Final transcript, debug evidence, YAML reports, Markdown counts, and CLI summary
|
|
||||||
agree about applied work and failures/recoveries.
|
|
||||||
- Reconciliation makes no additional provider calls. Existing structural failures,
|
|
||||||
provider failures, and cancellation retain their documented handling.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
No unresolved product-policy questions block implementation. Guided recovery and
|
|
||||||
structural salvage remain separate future decisions.
|
|
||||||
@@ -1,15 +1,15 @@
|
|||||||
# Deferred Feature Work
|
# Deferred Feature Work
|
||||||
|
|
||||||
**Last updated:** 2026-09-25
|
**Last updated:** 2026-10-03
|
||||||
|
|
||||||
**Status:** Deferred or exploratory work; not implemented or scheduled. These
|
**Status:** Deferred or exploratory work; not implemented or scheduled. These
|
||||||
entries preserve scope outside the [resilience roadmap](resilience.md) and
|
entries preserve work beyond the completed resilience and span-deletion
|
||||||
[span-deletion consistency roadmap](span-deletion.md). Each entry states its
|
features. Each entry states its intent; inclusion does not commit us to
|
||||||
intent; inclusion does not commit us to implementing every capability.
|
implementing every capability.
|
||||||
|
|
||||||
## Domain-Aware Response Recovery
|
## Domain-Aware Response Recovery
|
||||||
|
|
||||||
The [partial-success decision reconciliation](decision-set-reconciliation.md)
|
The implemented [partial-success decision reconciliation](../internal/validators.md#llm-validator-batching)
|
||||||
preserves usable decisions without another model call. Guided recovery remains
|
preserves usable decisions without another model call. Guided recovery remains
|
||||||
deferred pending evidence from failures that remain. General per-entry
|
deferred pending evidence from failures that remain. General per-entry
|
||||||
salvage from structurally invalid responses is also a separate design question;
|
salvage from structurally invalid responses is also a separate design question;
|
||||||
@@ -59,7 +59,7 @@ No new scheduler, global limiter, or execution mode is selected by this entry.
|
|||||||
|
|
||||||
Potential future work, subject first to evaluating whether it is worth implementing
|
Potential future work, subject first to evaluating whether it is worth implementing
|
||||||
at all. Empty replacement text and deletion of a span within a surviving segment
|
at all. Empty replacement text and deletion of a span within a surviving segment
|
||||||
are covered by the accepted [span-deletion consistency roadmap](span-deletion.md).
|
are covered by the implemented [span-deletion behavior](../internal/modules.md#proposal-generation-ownership).
|
||||||
That feature does not remove segment records or permit blank resulting segments.
|
That feature does not remove segment records or permit blank resulting segments.
|
||||||
|
|
||||||
Before designing segment removal, review representative production examples to
|
Before designing segment removal, review representative production examples to
|
||||||
|
|||||||
@@ -1,313 +1,523 @@
|
|||||||
# Implementation Plan: Standard Policy Alignment
|
# Implementation Plan: PromptKit v0.12.0 Adoption
|
||||||
|
|
||||||
**Last updated:** 2026-09-28
|
**Last updated:** 2026-10-04
|
||||||
|
**Status:** Complete; Stages 1–9 completed.
|
||||||
**Status:** Complete; Stages 1–4 implemented.
|
**Audience:** Luna coding agent.
|
||||||
**Audience:** gpt-5.6-terra coding agent.
|
|
||||||
**Audita planning baseline:** `91d9be68a7bb3a102900253fdfd8438f0a43f666`.
|
|
||||||
**Selected upstream revision:** `d84a38ebab26f048cafbe59a2246700c35d29843`.
|
|
||||||
|
|
||||||
## Scope and Execution
|
## Scope and Execution
|
||||||
|
|
||||||
Implement the accepted [Standard Policy Alignment roadmap](policy.md). It owns
|
Implement the accepted target in the [PromptKit roadmap](promptkit.md). That
|
||||||
the intended end state and reasons for local variations; this plan owns ordered
|
document owns feature intent and policy choices; this plan owns the ordered
|
||||||
implementation and verification. Complete Stage 1 through Stage 4 in order, each
|
implementation work. The planning baseline is Audita commit
|
||||||
as one focused agent task. Record completion evidence under each stage and commit
|
`b31955fc97adec707b4e5f9a1b0c43c4cbc66750`: ConfigKit v0.5.0 is adopted,
|
||||||
completed units with short, plain English messages under the agent policy.
|
PromptKit is v0.11.2, and the finalized roadmap includes the selected-profile,
|
||||||
|
operator-override, example-layout, and major-release clarifications.
|
||||||
|
|
||||||
Begin with [development guidance](../development.md) and the
|
This replaces the completed ConfigKit plan at this path. Its
|
||||||
[agent policy](../policy/agents.md). Read the policies governing each stage;
|
[completed implementation record](https://gitea.maximumdirect.net/eric/audita/src/commit/b31955fc97adec707b4e5f9a1b0c43c4cbc66750/docs/roadmap/implementation.md)
|
||||||
reuse unchanged policy context. For policy edits, read that policy and affected
|
remains available for readers following the ConfigKit completion roadmap.
|
||||||
cross-references. Follow the adopted selective reading guide once installed.
|
|
||||||
Use graph-first discovery for application code; document and shell/configuration
|
|
||||||
inspection does not need a graph lookup.
|
|
||||||
|
|
||||||
Use the public upstream repository at
|
Execute Stage 1 through Stage 9 in order. Each is sized as one bounded Luna
|
||||||
`https://gitea.maximumdirect.net/eric/policy` and check out the exact revision above
|
implementation prompt. Commit completed units with short, plain English messages
|
||||||
in a temporary directory outside Audita. Verify its full commit before copying.
|
under the [agent policy](../policy/agents.md); no approval round is needed between
|
||||||
Read its `docs/adoption.md`, relevant `docs/policy/` files, `COPYRIGHT`, and
|
stages. Record actual checks and any remaining expected migration failures.
|
||||||
referenced assets. Do not replace the selected revision with current main or
|
Stages 1 and 2 change formats before Stage 3 changes their consumer. Intermediate
|
||||||
reconstruct prior adoption baselines. The public repository and its attribution
|
commits need not be runnable or release-ready. Do not add temporary readers,
|
||||||
are now explicitly approved.
|
compatibility shims, fallback engines for execution, or scaffolding to make them
|
||||||
|
so. Later stages complete the same migration rather than separate product modes.
|
||||||
|
|
||||||
This plan replaces the completed decision-set plan, preserved at
|
Start with [development guidance](../development.md) and the applicable agent,
|
||||||
[the planning baseline](https://gitea.maximumdirect.net/eric/audita/src/commit/91d9be68a7bb3a102900253fdfd8438f0a43f666/docs/roadmap/implementation.md).
|
[architecture](../policy/architecture.md), [testing](../policy/testing.md),
|
||||||
Repair historical roadmap references in Stage 4; do not reopen those features.
|
[documentation](../policy/documentation.md), and [OSS](../policy/oss.md)
|
||||||
|
policies. Use graph-first discovery. Read the current
|
||||||
|
[configuration](../config.md), [LLM runtime](../internal/llm-runtime.md),
|
||||||
|
[prompt registry](../internal/prompts.md), and
|
||||||
|
[LLM integration](../integrations/openai-compatible-llm.md) contracts for the
|
||||||
|
established behavior this migration preserves.
|
||||||
|
|
||||||
Implement directly toward the final state. Intermediate commits need no temporary
|
Target the released PromptKit v0.12.0 at commit
|
||||||
policy files, compatibility machinery, or complete dependency inventory. Keep
|
`4b8169da286917fde7875425bb32b799b12c6af3`, ConfigKit v0.5.0, and both
|
||||||
partial work truthful and finish cross-document consistency by Stage 4. Do not
|
maintained `/v3` catalogs at `v3.20260930.0`. Read the pinned
|
||||||
release, tag, push, rewrite published material, change dependency versions, or
|
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
|
||||||
refactor application code as part of this work. No extra approval round is needed
|
and [migration guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api).
|
||||||
between stages. If factual verification uncovers a concrete unresolved rights or
|
The sibling checkout is reference material, not a dependency replacement.
|
||||||
publication issue, report it rather than inventing an obligation or claiming it
|
|
||||||
resolved.
|
|
||||||
|
|
||||||
## Common Adoption Decisions
|
### Selected Implementation Decisions
|
||||||
|
|
||||||
- Start each policy from the selected upstream file. Preserve shared substantive
|
- Keep Go 1.25.5, application `version: 2`, current flags, role selection,
|
||||||
wording verbatim. Remove its `Base Policy Template` section after fulfilling
|
application overlays, and main-file-relative path rules. Retain the
|
||||||
the instructions; include only the project customizations selected below.
|
`profiles_dir` name and default; broaden its accepted documents as agreed.
|
||||||
- Retain the upstream base version and set `Last Updated` to the actual adoption
|
- Use PromptKit's catalog and owned defaults, then caller underlays for embedded
|
||||||
date. Add `**Upstream Source:**` with a Markdown link to that same file at the
|
Audita prompts and operator directory inputs, then `Sources.Main` for
|
||||||
full selected commit, using the URL prefix
|
application backend settings. Directory inputs compose strictly within their
|
||||||
`https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/`.
|
group; discovery order supplies no field precedence. Keep imports disabled.
|
||||||
Do not claim this was the source of old local documents.
|
- Use one ConfigKit mapping for the final application source. For each
|
||||||
- Keep discovery URLs in `meta.md` pointed to the public repository and
|
application backend, supply its endpoint, `concurrency_limit`,
|
||||||
`/src/branch/main/docs/adoption.md`. Pin example links in the adopted OSS policy
|
`queue_capacity`, and explicit `api_key_env: null`. The null suppresses
|
||||||
to this selected commit, after checking their paths. Preserve examples upstream.
|
lower-tier credential lookup. Leave other supported provider fields available
|
||||||
- Explain substantive local variations briefly where they apply. No separate
|
to native layering. Do not put resolved credential values in this mapping.
|
||||||
exception register, synchronization system, policy snapshot tests, or legal
|
- Keep selected-role registration, endpoint-match, competing-credential, and
|
||||||
approval workflow is needed.
|
required-key checks. Add explicit nonblank effective model and endpoint checks
|
||||||
- Keep policy credit in the single canonical attribution entry. Describe only
|
for both selected roles. Valid unselected partial profiles remain permitted.
|
||||||
material adopted so far, expanding it as later stages copy more material.
|
- Preserve the adapter's complete per-call schemas, configured repairs/retries,
|
||||||
Metadata and template removal mean an adopted file can truthfully be described
|
operation context, shared variables, observation, and prepare/check/run flow.
|
||||||
as adapted even when its substantive rules are verbatim.
|
Operator messages can omit shared-variable references; do not append the
|
||||||
- Ordinary link/text review is sufficient for documentation changes. Do not run
|
shared text unconditionally or require particular message counts or roles
|
||||||
application tests simply because a policy document changed. Each stage ends
|
beyond PromptKit's own format rules.
|
||||||
with `git diff --check` and review of its staged diff before committing.
|
- Use captured source inspection for consumed host configuration and message
|
||||||
|
paths. Keep source-relative resource resolution with PromptKit and ConfigKit;
|
||||||
|
Audita projects paths for its existing output collision checks.
|
||||||
|
- Replace the old profile-reload test expectation with snapshot behavior.
|
||||||
|
Successful construction uses only final effective definitions. A failed
|
||||||
|
construction may make an embedded-only diagnostic construction to distinguish
|
||||||
|
asset defects from operator errors; it never executes or substitutes for the
|
||||||
|
requested configuration.
|
||||||
|
- Maintained example roots are `examples/profiles/local/` and
|
||||||
|
`examples/profiles/service/`. Service testing profiles stay in the service
|
||||||
|
root. Keep the literal `temperature: 0` in examples as an explicit outbound
|
||||||
|
setting, and use native presence semantics without a compatibility translator.
|
||||||
|
|
||||||
## Stage 1 — Establish Provenance and Shared Agent/Testing Policies
|
This unit is an intentional major-release migration. Exact version selection,
|
||||||
|
release notes, tag/push, and publication timing remain release preparation under
|
||||||
|
the [release policy](../policy/release.md). Guided recovery, new report formats,
|
||||||
|
hot reload, extra configuration commands, and scheduling changes remain outside
|
||||||
|
this plan. Tests should protect changed integration behavior and established
|
||||||
|
contracts, not reproduce the libraries' entire validation matrices.
|
||||||
|
|
||||||
**Completed:** 2026-09-28. Adopted meta 1.0, agents 1.3, and testing 1.3 from
|
## Stage 1 — Convert Embedded Prompt Definitions
|
||||||
the verified upstream commit, preserving Audita's graph-first discovery rule.
|
|
||||||
The initial attribution and scoped copyright notice identify only these three
|
|
||||||
adapted policies; the CC-BY-SA text matches upstream byte-for-byte. Substantive
|
|
||||||
policy text, pinned source paths, local links, and `git diff --check` were checked.
|
|
||||||
|
|
||||||
**Files:** `docs/policy/meta.md`, `docs/policy/agents.md`,
|
**Primary files:** The seven YAML files under
|
||||||
`docs/policy/testing.md`, `docs/oss/attribution.md`,
|
`internal/prompts/assets/prompts/`.
|
||||||
`docs/oss/licenses/CC-BY-SA-4.0.txt`, and `COPYRIGHT`.
|
|
||||||
|
|
||||||
1. Adopt meta 1.0, agents 1.3, and testing 1.3 with the common metadata and
|
1. Wrap each existing definition beneath `prompts`, its exact current ID, and
|
||||||
template-removal rules. Preserve the existing Codebase Discovery section as
|
the string version `v1`. Remove the old inner `id` and `version` fields.
|
||||||
the one local addition to agents: Audita deliberately uses graph-first
|
Keep filenames and Audita registry identifiers unchanged; filenames are
|
||||||
discovery. Adopt the remaining agent rules and the testing rules verbatim.
|
asset locations rather than PromptKit identities.
|
||||||
2. Create attribution with `# Open Source Attribution Notices`, followed directly
|
2. Preserve messages, block-scalar text, variable references, task examples,
|
||||||
by the canonical entry from `examples/licensing/policy-attribution.md`.
|
output settings, and schema paths. Preserve the applicable references to
|
||||||
Substitute the selected commit in its COPYRIGHT, LICENSE, and Source URLs.
|
`HardeningPolicy`, `ProposalContract`, and `ValidationContract`. Do not edit
|
||||||
Initially identify the three newly adapted policy files in its `Material:`
|
shared fragments, response schemas, or domain policies to accommodate the
|
||||||
sentence. Do not claim the remaining files have already been updated.
|
new container format.
|
||||||
3. Copy the selected revision's `docs/oss/licenses/CC-BY-SA-4.0.txt` byte-for-byte.
|
3. Review each before/after definition structurally, including YAML indentation
|
||||||
Keep the supplied upstream copyright and links to its grant/disclaimer.
|
inside block scalars. Do not introduce default profile selections or new
|
||||||
4. Add the adoption guide's scoped CC-BY-SA notice to Audita's `COPYRIGHT`, with
|
repair counts. The adapter continues to own the runtime output contract.
|
||||||
affected material identified through attribution. Preserve Audita's name,
|
|
||||||
copyright, GPL-3.0-or-later application grant, warranty statement, and third-party
|
|
||||||
statement. Leave the root GPLv3 `LICENSE` unchanged.
|
|
||||||
5. Keep metadata local and truthful. `meta.md` points to the local attribution
|
|
||||||
and developer guide even though selective reading guidance is added next.
|
|
||||||
|
|
||||||
**Validation and completion:** Compare the substantive agent/testing text directly
|
**Verification and completion:** Check all seven migrated identities and
|
||||||
with the selected upstream documents, allowing only template removal, metadata,
|
version keys against `RegisteredMetadata`, and run `git diff --check`. Registry
|
||||||
and the identified discovery addition. Verify each new source link's exact file
|
preparation tests become runnable with the new constructor in Stage 3. Commit
|
||||||
and revision, local attribution links, copied license bytes, and narrow grant
|
the asset conversion; do not introduce an old-format reader to test it here.
|
||||||
scope. No application test or publication is required.
|
|
||||||
|
|
||||||
## Stage 2 — Customize Remaining Policies and Install Selective Orientation
|
**Completion marker:** Stage 1 complete. All seven registered prompt assets
|
||||||
|
use keyed prompt/version mappings; parsed definitions were structurally checked
|
||||||
|
against their prior content, and `git diff --check` passed.
|
||||||
|
|
||||||
**Completed:** 2026-09-28. Adopted architecture 1.3, documentation 1.11,
|
## Stage 2 — Migrate Operator Profiles, Example Roots, and YAML Fixtures
|
||||||
release 1.4, and OSS 1.8 with the selected Audita variations. Installed the
|
|
||||||
upstream root orientation and verbatim Policy Reading Guide, expanded the
|
|
||||||
scoped attribution, and repaired the release documentation heading link.
|
|
||||||
Checked versions, provenance and example destinations, local links, policy
|
|
||||||
differences, and `git diff --check`.
|
|
||||||
|
|
||||||
**Files:** `docs/policy/architecture.md`, `docs/policy/documentation.md`,
|
**Primary files:** `examples/profiles/`, the three maintained configuration
|
||||||
`docs/policy/release.md`, `docs/policy/oss.md`, `AGENTS.md`,
|
examples, and profile-producing test fixtures in
|
||||||
`docs/development.md`, `docs/oss/attribution.md`, and `COPYRIGHT` as needed for
|
`internal/framework/llm/{runtime_inspection,adapter,adapter_capacity}_test.go`,
|
||||||
accurate copied-guidance scope.
|
`internal/framework/proposal_generation/adapter_integration_test.go`,
|
||||||
|
`internal/framework/validators/llm_integration_test.go`,
|
||||||
|
`internal/framework/runner/cancellation_test.go`, and
|
||||||
|
`cmd/audita/main_integration_test.go`.
|
||||||
|
|
||||||
1. Adopt architecture 1.3. Retain the upstream general Purpose And Ownership,
|
1. Move local proposal/validation examples beneath `profiles/local/`; move the
|
||||||
Architectural Approach, and Adopted Constraints And Implemented Structure
|
four service proposal/validation and testing files beneath
|
||||||
guidance. Replace Application-Specific Topics with Audita's established
|
`profiles/service/`. Convert each to `profiles: {<ID>: <fields>}` in readable
|
||||||
purpose, ownership/dependency boundaries, correction/concurrency rules,
|
YAML. Keep IDs, models, positive timeouts, and controls, including explicit
|
||||||
recovery, state/compatibility, secret handling, and selected library ownership.
|
zero temperature. No example needs a real credential or live model.
|
||||||
Reuse existing local decisions; avoid duplicating general upstream prose.
|
2. Set `profiles_dir: profiles/local` in `examples/minimal-config.yml` and
|
||||||
Replace example threat discussion with the existing secret/input boundaries
|
`examples/configkit-inheritance.yml`; set `profiles_dir: profiles/service`
|
||||||
and canonical links where useful. Do not infer new threat exclusions,
|
in `examples/production-config.yml`. Preserve the testing overlay's role IDs
|
||||||
filesystem guarantees, interfaces, or package restructuring from the template.
|
and the existing ConfigKit inheritance example. Select the leaf root, since
|
||||||
2. Adopt documentation 1.11. Keep applicable upstream ownership rows verbatim;
|
recursive discovery at their common parent would still load both groups.
|
||||||
omit public HTTP and public Go API rows. Add the existing concrete ownership
|
3. Convert profile YAML emitted by the listed fixtures to keyed definitions.
|
||||||
of `docs/integrations/output-files.md` and contributor change recipes, and
|
Search for remaining old profile construction strings and removed
|
||||||
retain the deliberate retirement of standalone troubleshooting documentation.
|
`base_profile` syntax, and migrate any genuine profile fixture found.
|
||||||
Account for the retained agent discovery/commit responsibilities in its owner
|
Do not rewrite transcript/proposal `id` fields or domain schema fixtures.
|
||||||
row. Keep reading triggers in development guidance, not a second policy table.
|
Keep test source names independent of profile IDs where already exercised.
|
||||||
3. Adopt release 1.4. Retain upstream completion, validation, guard, immutability,
|
4. Keep negative tests negative for their selected behavioral reason. Give
|
||||||
recovery, and preparation language. Preserve the existing Supported Platforms
|
cases clean temporary layouts or replace their contribution completely
|
||||||
table and unsupported-platform statement as Audita's support commitment.
|
rather than leaving an unrelated invalid definition behind for later cases.
|
||||||
Replace the upstream discretionary human-gate paragraph with Audita's selected
|
Eager validation now reads unselected final definitions too. Do not remove
|
||||||
rule against manual approvals, sign-offs, and routine confirmations as release
|
such definitions merely to simulate the old lazy loader where the test is
|
||||||
gates. Remove the obsolete list describing now-shared upstream rules as local
|
intended to exercise eager validation.
|
||||||
deviations. Keep concrete atomic-push mechanics in the runbook.
|
|
||||||
4. Adopt OSS 1.8. Add a concise Audita context paragraph identifying the existing
|
|
||||||
GPL-3.0-or-later grant and linking to COPYRIGHT and the release procedure for
|
|
||||||
Gitea distribution. Use the upstream House Format and interpretation language
|
|
||||||
verbatim. Pin application/library COPYRIGHT and attribution examples to the
|
|
||||||
selected upstream commit; remove template-only setup text. No license change
|
|
||||||
to application code or blanket dependency compatibility claim is authorized.
|
|
||||||
5. Replace root orientation with upstream `agents/AGENTS.md` wording while
|
|
||||||
retaining any existing applicable repository instructions. Keep detailed
|
|
||||||
discovery rules in their current agent-policy owner rather than duplicating
|
|
||||||
them in a newly created block.
|
|
||||||
6. Insert the adoption guide's Policy Reading Guide and accompanying reading rules
|
|
||||||
into `docs/development.md` verbatim. Remove the blanket architecture-reading
|
|
||||||
instruction; preserve contributor commands and component-specific task rows.
|
|
||||||
Adjust cross-links that would otherwise contradict selective reading.
|
|
||||||
7. Expand the single policy attribution entry to identify all `docs/policy/`
|
|
||||||
documents, adapted root orientation, and the copied Policy Reading Guide in
|
|
||||||
`docs/development.md`. Scope the COPYRIGHT CC-BY-SA notice to those materials,
|
|
||||||
not unrelated contributor prose or application documentation. Keep the material
|
|
||||||
sentence factual and concise; retain the pinned upstream credit links.
|
|
||||||
8. Find and repair local links to policy headings renamed by adoption, including
|
|
||||||
the release correction heading. Do not edit published historical release
|
|
||||||
notes to retrofit new headings or licensing conventions.
|
|
||||||
|
|
||||||
**Validation and completion:** Compare all seven policies to the pinned source;
|
**Verification and completion:** Review the six profiles, explicit root
|
||||||
review every substantive difference against the roadmap's justified variations.
|
selections, and changed fixture text; run `git diff --check` and format touched
|
||||||
Check versions (1.3, 1.3, 1.11, 1.3, 1.4, 1.0, 1.8 for agents, architecture,
|
Go test files. These formats become consumable in Stage 3. Commit this unit.
|
||||||
documentation, testing, release, meta, OSS respectively), file-specific source
|
Current documentation links and recipes are updated together in Stage 8.
|
||||||
links, removed template sections, example destinations, ownership consistency,
|
|
||||||
and selective reading. Temporary pending release/attribution implementation is
|
|
||||||
completed by the following stages; do not create scaffolding for it.
|
|
||||||
|
|
||||||
## Stage 3 — Complete Component Attribution and README Navigation
|
**Completion marker:** Stage 2 complete. Six maintained profile documents are
|
||||||
|
keyed and split into separately selected local/service roots; the listed Go
|
||||||
|
fixtures use keyed definitions. Touched Go files were formatted and
|
||||||
|
`git diff --check` passed. Runtime tests are deferred until Stage 3 replaces
|
||||||
|
the current consumer, as the plan allows intermediate commits not to run.
|
||||||
|
|
||||||
**Completed:** 2026-09-28. Both CGO-disabled Linux target dependency lists
|
## Stage 3 — Adopt the Released Source API and Selected-Role Checks
|
||||||
selected the same eight production modules. Inspected their selected source
|
|
||||||
notices and copied assets, excluded test-only dependencies, and resolved the
|
|
||||||
Go 1.25.5 source tag and its four incorporated vendored modules. Added 14
|
|
||||||
House Format entries including policy provenance, complete applicable license
|
|
||||||
texts, and the README footer. Checked 33 pinned source and notice destinations,
|
|
||||||
license bytes, local links, entry shape, and `git diff --check`.
|
|
||||||
|
|
||||||
**Files:** `docs/oss/attribution.md`, applicable texts under
|
**Primary files:** `go.mod`, `go.sum`,
|
||||||
`docs/oss/licenses/`, and `README.md`. Add factual exception treatment to the OSS
|
`internal/framework/llm/runtime.go`, and `internal/prompts/registry_test.go`.
|
||||||
policy only if the actual inventory uncovers a concrete need.
|
|
||||||
|
|
||||||
1. Inventory actual incorporated material using the selected modules, production
|
1. Upgrade with `go get gitea.maximumdirect.net/eric/promptkit@v0.12.0` and
|
||||||
package dependencies for both Linux targets, source/assets, and shipped Go
|
reconcile the module files. Verify ConfigKit stays v0.5.0, the catalog
|
||||||
runtime. Use `go list -deps` for `./cmd/audita` with `CGO_ENABLED=0` and each
|
dependencies become `/v3` at `v3.20260930.0`, and obsolete `/v2` requirements
|
||||||
target architecture, plus source inspection for copied material. Inspect test
|
are removed when unused. Retain the Go minimum and unrelated selections;
|
||||||
dependencies where relevant to included material; a go.mod entry or tool use
|
introduce no local replacements.
|
||||||
alone is not sufficient reason for attribution. Do not change go.mod/go.sum.
|
2. Resolve `profiles_dir` exactly as the current runtime does. Obtain embedded
|
||||||
2. At the planning baseline, production dependencies include ConfigKit v0.4.1,
|
inputs with `pk.FilesystemInputs` and operator inputs with `pk.DirectoryInputs`.
|
||||||
PromptKit v0.11.1, both catalogs v2.20260926.0, jsonschema/v6 v6.0.2,
|
An unavailable selected root is an operator error; an empty directory can
|
||||||
yaml/v4 v4.0.0-rc.6, x/text v0.14.0, and yaml.v3 v3.0.1. Start with the four
|
contribute no definitions. Do not require one profile per file, IDs matching
|
||||||
verified source commits recorded in the feature roadmap; resolve remaining
|
filenames, or profile-only top-level content.
|
||||||
module versions to exact source commits and inspect their COPYRIGHT, LICENSE,
|
3. Construct the agreed `pk.ConfigurationSources`: two ordered caller underlays
|
||||||
NOTICE, PATENTS, and relevant bundled material as applicable. Account for
|
for embedded and operator groups, and one highest-priority application
|
||||||
additional material discovered in that inspection rather than treating this
|
`ck.Input` whose source is the mapping specified above. Retain
|
||||||
starting list as an exhaustive guarantee.
|
`pk.WithSchemaFS`, the supplied HTTP client, and configured transient retries.
|
||||||
3. For the Go runtime, use the release build's configured Go 1.25.5 source tag
|
Remove old prompt/profile/backend registration options.
|
||||||
and resolve it to its full source commit. Do not silently substitute the
|
4. Inspect registered prompt ID/version selections, then both selected roles.
|
||||||
inspecting agent's installed toolchain (Go 1.26.8 during planning). Preserve
|
Reject a selected role whose effective model or endpoint is empty or
|
||||||
actual runtime/standard-library notices, including relevant bundled third-party
|
whitespace-only. Retain backend registration, normalized endpoint equality,
|
||||||
material. Document maintenance of this entry when the distributed toolchain
|
nonblank effective `APIKeyEnv` rejection, and required-credential checks.
|
||||||
changes; do not pin new toolchain behavior or reject other supported local Go
|
Preserve literal IDs; do not trim/re-key identities or require all profiles
|
||||||
versions merely to simplify attribution.
|
to be complete. Populate the same role inspection map for both clients.
|
||||||
4. Add entries following OSS 1.8 exactly, with selected module/version or material
|
5. Migrate the registry test engine to `FilesystemInputs` and a programmatic
|
||||||
scope where useful. Preserve supplied copyrights and accurate license
|
`ProfileSource` fixture in its own tier, retaining its schema filesystem.
|
||||||
expressions; use the prescribed link labels and `[Source](COMMIT_URL).`
|
A complete endpoint/model fixture without an application backend is valid
|
||||||
spelling. Keep policy attribution first. Order other entries consistently by
|
for this direct library test; Audita's selected-role guard belongs in its
|
||||||
project name; ordering is presentation, not a new public contract.
|
runtime tests. Keep the embedded prompt preparation assertions scoped to
|
||||||
5. Copy applicable complete standard texts from verified sources, using the
|
embedded prompts.
|
||||||
pinned reference collection where applicable. Share identical standard texts
|
6. Remove the old `profileSourceFS` wrapper once unused. Source accounting is
|
||||||
and include required companions (LGPLv3 and GPLv3 for the two libraries).
|
completed in Stage 5. Preserve the adapter boundary; change other production
|
||||||
Include upstream commit links when a file contains project-specific material
|
callers only to resolve actual API incompatibilities. Search for remaining
|
||||||
in addition to standard wording. Preserve existing incorporated notices.
|
uses of removed source options and old `pk.Profile`/`pk.Backend` values.
|
||||||
6. Apply the adopted house interpretation for project-specific notice links.
|
Run `go mod tidy` after migrating construction callers and review its diff.
|
||||||
Put concrete exception reasoning in OSS policy or an appropriate decision
|
|
||||||
record, never in attribution. Do not add missing-copyright filler, research
|
|
||||||
narratives, inferred licensing choices, or legal conclusions to entries.
|
|
||||||
7. End README with the two OSS statements, identifying Audita as GPL v3.0 or later
|
|
||||||
and linking to `COPYRIGHT` and `docs/oss/attribution.md`. The linked grant owns
|
|
||||||
the separately licensed policy scope; no duplicated notice list is needed.
|
|
||||||
|
|
||||||
**Validation and completion:** Inspect entries against selected source evidence,
|
**Verification and completion:** Run `go test ./internal/prompts` and the
|
||||||
resolve every source/notice commit and path, compare standard license bytes, and
|
existing offline profile assembly, unsafe-selection, and absolute-directory
|
||||||
check local links. Confirm that actual distributed material is accounted for
|
tests in `internal/framework/llm`. Run `git diff --check` and review the module
|
||||||
without blindly including dependency graph or development-tool entries. Record
|
diff. The obsolete file-reload assertion is intentionally replaced in Stage 6;
|
||||||
concise inventory/check evidence under this stage; no new SBOM, scanner, durable
|
do not restore reload behavior to make that assertion pass. Commit the runtime
|
||||||
inventory generator, or test suite is required.
|
API migration.
|
||||||
|
|
||||||
## Stage 4 — Integrate Release Notes and Finish Cross-Document Validation
|
**Completion marker:** Stage 3 complete. Runtime construction now uses the
|
||||||
|
released PromptKit source API with the selected application mapping; module
|
||||||
|
selections match PromptKit v0.12.0 and both `/v3` catalogs at
|
||||||
|
`v3.20260930.0`. `go mod tidy`, prompt registry tests, offline profile
|
||||||
|
assembly, unsafe-selection, absolute-directory tests, and `git diff --check`
|
||||||
|
passed. Captured source accounting remains in Stage 5.
|
||||||
|
|
||||||
**Completed:** 2026-09-28. Wired the checked-in release note into the hosted
|
## Stage 4 — Preserve Startup Failure Classification and Validate Assembly
|
||||||
release plugin, documented immutable-tag notice links and release preparation,
|
|
||||||
repaired the specified historical roadmap references, and marked the feature
|
|
||||||
roadmap implemented. Parsed the YAML, substituted `v2.0.0` to a tracked note,
|
|
||||||
checked non-overwrite settings and candidate footer paths, reviewed all seven
|
|
||||||
policies against the pinned source, and checked repository-local Markdown links
|
|
||||||
and `git diff --check`. Hosted execution awaits the next real release.
|
|
||||||
|
|
||||||
**Files:** `.woodpecker/release.yml`, `docs/release.md`, affected documentation
|
**Primary files:** `internal/framework/llm/runtime.go`,
|
||||||
links, `docs/roadmap/policy.md`, and this plan. Repair the historical references
|
`internal/framework/llm/runtime_inspection_test.go`, and configuration-command
|
||||||
specified below; do not retire unrelated roadmaps in this sprint.
|
tests in `internal/cli`. Update `internal/cli/run.go` only where the changed
|
||||||
|
startup path needs existing exit behavior preserved.
|
||||||
|
|
||||||
1. Add this setting to the existing `publish-release` step:
|
1. Treat embedded discovery failures as `AssetError`; treat operator-directory
|
||||||
|
discovery failures as configuration input errors. Preserve wrapped causes
|
||||||
|
for `errors.Is`/`errors.As` and retain configured-secret redaction at CLI
|
||||||
|
boundaries. Caller cancellation remains a runtime exit, not an operator
|
||||||
|
mistake, and should not trigger further diagnostic loading.
|
||||||
|
2. PromptKit wraps both definition-loading and template/content failures in
|
||||||
|
`ErrInvalidConfig`; some materialization errors have no public source/path
|
||||||
|
type. Do not classify by parsing error strings or by that sentinel alone.
|
||||||
|
On final engine-construction failure, perform an embedded-only diagnostic
|
||||||
|
construction with the same embedded prompt/schema sources and library
|
||||||
|
defaults, excluding operator inputs and application backend mappings. If
|
||||||
|
that also fails, surface an `AssetError` with the asset cause; otherwise
|
||||||
|
return the original final-construction error as an operator/configuration
|
||||||
|
error. The diagnostic engine does no provider work and is never retained
|
||||||
|
for execution. Keep a small unexported assembly seam accepting an embedded
|
||||||
|
filesystem so tests can exercise asset failures without modifying shipped
|
||||||
|
files or creating a global test hook.
|
||||||
|
3. Do not run this diagnostic construction as a successful-path preflight.
|
||||||
|
A valid final snapshot remains accepted even if an operator contribution
|
||||||
|
repairs an otherwise invalid lower-tier prompt. Verify that property with
|
||||||
|
the same small assembly test seam. This avoids strengthening upstream
|
||||||
|
final-value validation into mandatory validation of displaced definitions.
|
||||||
|
4. Cover incomplete selected profiles using omitted, empty, and null model
|
||||||
|
contributions and a cleared selected endpoint. Exercise proposal and
|
||||||
|
validation selection, including omission-derived validation selection, with
|
||||||
|
a rejecting HTTP transport. Confirm valid unselected partial profiles are
|
||||||
|
accepted; malformed unselected definitions, undefined backend references,
|
||||||
|
and conflicting directory contributions still fail under native rules.
|
||||||
|
5. Add focused boundary coverage that invalid operator YAML/template/resource
|
||||||
|
input produces configuration exit `2`, and test the asset classification
|
||||||
|
directly at the runtime seam. Reuse existing CLI handling of `AssetError`
|
||||||
|
for exit `1`; no damaged-binary subprocess fixture is needed. Cover the
|
||||||
|
shared `NewRuntime` validation route and preserve cancellation exit `1` in
|
||||||
|
configuration commands if the new error path would otherwise classify it
|
||||||
|
as `2`. Avoid exact complete diagnostic wording assertions.
|
||||||
|
|
||||||
```yaml
|
**Verification and completion:** Run targeted startup/inspection tests and
|
||||||
note: docs/releases/${CI_COMMIT_TAG}.md
|
`go test ./internal/cli`; check whitespace and commit. Preserve the original
|
||||||
|
failure cause where available rather than flattening it to a generic error.
|
||||||
|
|
||||||
|
**Completion marker:** Stage 4 complete. Embedded discovery and construction
|
||||||
|
failures are distinguished from operator inputs through an embedded-only
|
||||||
|
diagnostic construction; cancellation skips that diagnostic. Selected-profile
|
||||||
|
completeness and native directory validation are covered, and both config
|
||||||
|
commands preserve exit 1 for cancellation/assets and exit 2 for invalid
|
||||||
|
operator YAML, templates, and resources. Targeted runtime tests,
|
||||||
|
`go test ./internal/cli`, and `git diff --check` passed.
|
||||||
|
|
||||||
|
## Stage 5 — Protect Consumed Configuration and Message Files
|
||||||
|
|
||||||
|
**Primary files:** `internal/framework/llm/runtime.go` or a focused source-path
|
||||||
|
helper in that package, `internal/cli/process.go`, runtime source-accounting
|
||||||
|
tests, and `cmd/audita/main_integration_test.go`.
|
||||||
|
|
||||||
|
1. Rename the internal `Runtime.ProfilePaths` field to `SourcePaths` and update
|
||||||
|
its process caller. It now covers consumed host YAML and message resources,
|
||||||
|
rather than profiles alone. Populate it from the successfully captured final
|
||||||
|
engine; do not expose source metadata as a new public output.
|
||||||
|
2. Call `InspectConfiguration`. Index `Sources()` by source ID, and collect
|
||||||
|
paths with `Kind == ck.SourceKindFile`; embedded filesystem and synthetic
|
||||||
|
sources are not host paths. All loaded host YAML counts, including unused
|
||||||
|
definitions and files whose contributions were later overlaid. Deduplicate
|
||||||
|
paths and sort for a stable internal projection.
|
||||||
|
3. For message resources, enumerate `Metadata().HistoricalPaths()` and retain
|
||||||
|
effective, nonnull paths shaped as
|
||||||
|
`/prompts/<id>/<version>/messages/<index>/content_file`. Use `Lookup` and the
|
||||||
|
effective field's origin, not an inherited destination's guessed source.
|
||||||
|
PromptKit successfully captures these fields only with a unique usable
|
||||||
|
source context. Obtain the effective scalar from `Explain(path).Effective`
|
||||||
|
and decode that scalar YAML into a string with the existing YAML dependency.
|
||||||
|
This decodes captured inspection data; it is not a second configuration
|
||||||
|
loader. When the contributing source is a host file, join the untrimmed
|
||||||
|
resource name against that source file's directory and add it to the ledger.
|
||||||
|
Ignore displaced references and non-host resources. Do not reread operator
|
||||||
|
YAML, infer IDs from filenames, or reimplement resource-root validation.
|
||||||
|
4. Pass `SourcePaths` through existing `diagnostics.PrepareOutput` protection.
|
||||||
|
Leave canonicalization, symlink/hardlink checks, permitted in-place transcript
|
||||||
|
handling, and output initialization with their current owner. Do not protect
|
||||||
|
every unrelated file in the selected directory or add filesystem mutation
|
||||||
|
guarantees beyond the existing collision contract.
|
||||||
|
5. Extend `TestSubprocessProfileSources` and a focused runtime case to cover an
|
||||||
|
arbitrarily named YAML contribution, a consumed source-relative message
|
||||||
|
file, and conflicting transcript/report destinations. Use a non-YAML message
|
||||||
|
filename, such as `instruction.md`, so directory discovery does not mistake
|
||||||
|
plain prompt text for a configuration document. Include an inherited message
|
||||||
|
reference from a different directory and a prompt ID/version with a JSON
|
||||||
|
Pointer escape character so source provenance and literal keys determine
|
||||||
|
the path. Check rejection before provider dispatch and unchanged input bytes.
|
||||||
|
Preserve acceptance of an unused alternate output filename; do not turn the
|
||||||
|
whole directory into a forbidden output location.
|
||||||
|
|
||||||
|
**Verification and completion:** Run focused source-ledger tests and subprocess
|
||||||
|
collision cases, plus `git diff --check`, then commit. Check that resource
|
||||||
|
accounting uses the frozen snapshot and never changes the engine's source or
|
||||||
|
resource-resolution behavior to facilitate tracking.
|
||||||
|
|
||||||
|
**Completion marker:** Stage 5 complete. `Runtime.SourcePaths` now projects
|
||||||
|
host YAML and effective source-relative message files from PromptKit's captured
|
||||||
|
configuration inspection, using field provenance for inherited references.
|
||||||
|
Process output protection consumes that ledger. Focused source-accounting and
|
||||||
|
subprocess collision tests passed, including unchanged input bytes, no provider
|
||||||
|
dispatch, JSON Pointer-escaped keys, and acceptance of an unused alternate
|
||||||
|
output; `git diff --check` passed.
|
||||||
|
|
||||||
|
## Stage 6 — Verify Operator Prompt Overrides and Captured Execution
|
||||||
|
|
||||||
|
**Primary files:** `internal/framework/llm/adapter_test.go` and focused runtime
|
||||||
|
tests. Change `adapter.go` only for an actual integration defect.
|
||||||
|
|
||||||
|
1. Add a native keyed operator contribution overriding messages for an active
|
||||||
|
prompt/version. Use the real runtime and controlled HTTP responses to prove
|
||||||
|
the operator text reaches both the prepared snapshot and dispatched request.
|
||||||
|
One case deliberately omits shared-variable references and still succeeds;
|
||||||
|
do not require Audita to inject their text. A reference-bearing case renders
|
||||||
|
the adapter-supplied value and retains existing protection against caller
|
||||||
|
variables replacing the reserved shared variables.
|
||||||
|
2. Override the prompt's definition-level output contract with another valid
|
||||||
|
upstream contract. Prove that Audita's per-call JSON schema and configured
|
||||||
|
repair budget still govern execution: a structurally invalid response is
|
||||||
|
repaired or exhausted as the controlled budget dictates. Domain validators
|
||||||
|
remain downstream. Reuse existing validator/proposal integration coverage
|
||||||
|
for their rules rather than weakening them to fit an operator prompt.
|
||||||
|
3. Replace the reload portion of
|
||||||
|
`TestPreparedAdapterRejectsTargetDriftAndPreparationErrors`. After successful
|
||||||
|
construction, change a profile or message file and verify the existing
|
||||||
|
runtime still uses its captured settings/text. Construct a new runtime to
|
||||||
|
observe a valid edit or reject a newly mismatched endpoint. Keep the missing
|
||||||
|
prompt preparation failure case.
|
||||||
|
4. Retain a focused test of the prepared-target check independently of reload:
|
||||||
|
deliberately change the test runtime's application backend endpoint after
|
||||||
|
construction and verify the inconsistency is blocked before dispatch. This
|
||||||
|
tests the existing guard, not a supported live mutation/reload feature.
|
||||||
|
Continue executing the same prepared handle whose details were inspected;
|
||||||
|
do not add another preparation or an application-side retry loop.
|
||||||
|
|
||||||
|
**Verification and completion:** Run the prompt override, snapshot, and
|
||||||
|
prepared-adapter tests. Compare semantic messages and selected targets, not old
|
||||||
|
hash values or exact YAML formatting. Check whitespace and commit.
|
||||||
|
|
||||||
|
**Completion marker:** Stage 6 complete. Native operator messages reach both
|
||||||
|
the prepared snapshot and provider request, with and without shared-variable
|
||||||
|
references. The adapter's per-call schema and repair budget remain effective
|
||||||
|
under a valid operator output-contract override. Existing runtimes retain
|
||||||
|
captured prompt/profile values after source edits, and a new runtime observes
|
||||||
|
valid edits; the target guard and preparation failure remain covered. The
|
||||||
|
`internal/framework/llm` tests and `git diff --check` passed.
|
||||||
|
|
||||||
|
## Stage 7 — Verify Native Profiles, Backend Authority, and Resilience
|
||||||
|
|
||||||
|
**Primary files:** Runtime inspection and adapter tests,
|
||||||
|
`internal/framework/llm/adapter_capacity_test.go`, and existing framework/CLI
|
||||||
|
integration tests where their established behavior is affected.
|
||||||
|
|
||||||
|
1. Add a compact native profile inheritance fixture with literal keyed IDs
|
||||||
|
unrelated to filenames, multiple definitions in one file, a partial parent,
|
||||||
|
and a complete selected child. Verify inherited controls survive a child
|
||||||
|
model change and child overrides win. Reuse the metadata-facing literal-key
|
||||||
|
coverage from Stage 5 rather than repeating it in profile tests.
|
||||||
|
2. Use controlled HTTP request bodies to distinguish explicit zero temperature
|
||||||
|
from omission and inherited-control suppression by null. Check a
|
||||||
|
representative recursively overlaid provider parameter and null clearing;
|
||||||
|
do not promise that all providers accept every valid configured zero.
|
||||||
|
Reuse existing deadline/cancellation coverage for unchanged outer deadlines;
|
||||||
|
explicit zero profile timeout must not disable the operation context.
|
||||||
|
3. Verify an operator backend contribution can add supported provider fields,
|
||||||
|
while a conflicting endpoint/concurrency/queue/environment contribution
|
||||||
|
cannot displace the final application fields. Preserve an explicit profile
|
||||||
|
environment lookup as a selected-role error rather than hiding it with an
|
||||||
|
adapter override. Include an application backend using a catalog ID to show
|
||||||
|
it is allowed and its catalog credential lookup is suppressed. Use synthetic
|
||||||
|
environment values and local HTTP fixtures, not provider catalog endpoints.
|
||||||
|
4. Reuse `TestRolesShareBackendCapacityAndReleaseAfterCancellation` and
|
||||||
|
`TestDistinctBackendsHaveIndependentCapacity`; verify the final mapped
|
||||||
|
limits, including zero queue capacity, still coordinate both roles and
|
||||||
|
remain backend-local. Strengthen a fixture with conflicting lower-tier
|
||||||
|
capacity only where needed to prove the new mapping wins. Keep existing
|
||||||
|
synchronization; do not infer admission with sleeps or add another limiter.
|
||||||
|
5. Preserve offline selected-profile inspection and value-free application
|
||||||
|
explanations. Check configured secrets, endpoint strings, arbitrary provider
|
||||||
|
fields, and prompt text remain outside the current safe profile summary.
|
||||||
|
Do not replace it with a full upstream dump. Its numeric projection is not
|
||||||
|
proof that an omitted provider setting is sent; wire presence belongs to
|
||||||
|
the HTTP coverage above.
|
||||||
|
6. Run the existing transient retry, structural repair, attempt-count,
|
||||||
|
observation, partial-validation, and all-model-failure subprocess cases
|
||||||
|
after fixture migration. Preserve their error/recovery/debug classifications,
|
||||||
|
cancellation, and continued output. Fix proven integration regressions;
|
||||||
|
do not rewrite domain outcomes or reduce budgets to obtain a passing suite.
|
||||||
|
|
||||||
|
**Verification and completion:** Run
|
||||||
|
`go test ./internal/framework/llm ./internal/framework/proposal_generation ./internal/framework/validators ./internal/framework/runner ./internal/cli ./cmd/audita`
|
||||||
|
and `git diff --check`. Review whether each new test addresses a distinct changed
|
||||||
|
boundary. Commit the native-profile and runtime verification unit.
|
||||||
|
|
||||||
|
**Completion marker:** Stage 7 complete. Native profile inheritance, control
|
||||||
|
presence, provider-map overlays, application backend authority, selected
|
||||||
|
credential rejection, catalog-ID handling, shared backend capacity, and safe
|
||||||
|
inspection behavior are covered. The six targeted packages passed, along with
|
||||||
|
focused wire, timeout, capacity, credential, and safe-view tests; `git diff
|
||||||
|
--check` passed.
|
||||||
|
|
||||||
|
## Stage 8 — Document the Migration and Update Attribution
|
||||||
|
|
||||||
|
**Primary files:** `docs/config.md`, `docs/internal/llm-runtime.md`,
|
||||||
|
`docs/internal/prompts.md`, `docs/integrations/openai-compatible-llm.md`,
|
||||||
|
`docs/operations.md`, `docs/internal/cli-config.md`, `README.md`, and
|
||||||
|
`docs/oss/attribution.md`.
|
||||||
|
|
||||||
|
1. Update the configuration contract for unified `profiles_dir` content, exact
|
||||||
|
keyed identities, directory composition, agreed tier precedence, profile
|
||||||
|
inheritance/presence, selected-role completeness, eager validation of all
|
||||||
|
final definitions, selected-root failures, and construction-time capture.
|
||||||
|
Replace the obsolete statement that PromptKit profiles do not gain ConfigKit
|
||||||
|
inheritance. Retain application overlays, defaults, and credential ownership.
|
||||||
|
2. Include concise migration instructions in the profile section: wrap old
|
||||||
|
profile fields beneath `profiles/<id>`, remove inner `id`, change
|
||||||
|
`base_profile` to `extends`, review explicit values/nulls, and select roots
|
||||||
|
whose final definitions have valid backend references. Show a small keyed
|
||||||
|
inheritance/override snippet and link complete maintained examples. Explain
|
||||||
|
explicit zero temperature versus omitted or null controls and link upstream
|
||||||
|
for the full field table. State that this ships as an Audita major migration;
|
||||||
|
do not invent an exact release version or automatic file conversion.
|
||||||
|
3. Document intentional prompt overrides and optional shared-variable references.
|
||||||
|
Supplying shared variables does not inject omitted text, while per-call
|
||||||
|
schemas and domain validators remain enforced. Internal prompt documentation
|
||||||
|
distinguishes shipped asset instructions from operator-effective messages;
|
||||||
|
no prompt-quality rewrite or new global prompt-policy requirement is added.
|
||||||
|
4. Describe layered runtime assembly, source accounting, snapshot use, and
|
||||||
|
startup classification in their internal owner. Update the pinned PromptKit
|
||||||
|
consumer link in the LLM integration contract. Expand operations' loaded
|
||||||
|
source description to include consumed operator message files; keep physical
|
||||||
|
output/collision rules with operations and diagnostics.
|
||||||
|
5. Repair example navigation for separate local/service roots, especially the
|
||||||
|
README's local-profile link. Update validation recipes to use the maintained
|
||||||
|
config files' explicit roots and keep the service testing overlay workflow.
|
||||||
|
Review other affected links without rewriting historical release notes.
|
||||||
|
6. Update the PromptKit and both catalog attribution entries to selected versions
|
||||||
|
and exact commit-pinned source/notice URLs. Resolve catalog tag commits from
|
||||||
|
the selected module origins or authoritative upstream refs, not annotated
|
||||||
|
tag object IDs. Check the releases' COPYRIGHT/LICENSE files and retain the
|
||||||
|
applicable unchanged local LGPL/GPL and MIT texts. Keep the existing House
|
||||||
|
Format; no new license selection, policy rewrite, or attribution inventory
|
||||||
|
is needed. Review `go.mod`/`go.sum` against those entries.
|
||||||
|
|
||||||
|
**Verification and completion:** Check affected local links/anchors, format
|
||||||
|
examples against the implemented loader, verify source revisions/notices, and
|
||||||
|
run `git diff --check`. Commit the documentation and attribution unit.
|
||||||
|
|
||||||
|
**Completion marker:** Stage 8 complete. Configuration, runtime, prompt,
|
||||||
|
integration, operations, contributor, and README guidance now reflects native
|
||||||
|
PromptKit layering, migration, presence, snapshot, and separate example-root
|
||||||
|
behavior. Attribution matches PromptKit v0.12.0 and both v3 catalog module
|
||||||
|
origins and notices. Local links/anchors and `git diff --check` passed.
|
||||||
|
|
||||||
|
## Stage 9 — Validate the Integrated End State and Close the Plan
|
||||||
|
|
||||||
|
**Primary files:** This plan and the feature roadmap; correct an implementation
|
||||||
|
file only if the final checks reveal an actual gap.
|
||||||
|
|
||||||
|
1. Verify the full roadmap acceptance criteria against the landed code and the
|
||||||
|
distinct tests from these stages. Review for unintended requirements:
|
||||||
|
profile-only filtering, mandatory shared text in operator messages,
|
||||||
|
filename-derived IDs, complete unselected profiles, catalog ID bans,
|
||||||
|
successful-path asset preflights, compatibility readers, and new output
|
||||||
|
guarantees are not part of the accepted target.
|
||||||
|
2. Run `go build -o ./bin/audita ./cmd/audita` and `go test ./...`. Keep the
|
||||||
|
generated binary out of commits. Reuse successful checks on unchanged
|
||||||
|
content; rerun checks affected by any corrective code changes.
|
||||||
|
3. Validate the maintained examples offline using the built binary:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./bin/audita config validate --config examples/minimal-config.yml
|
||||||
|
./bin/audita config validate --config examples/configkit-inheritance.yml
|
||||||
|
AUDITA_SERVICE_KEY=synthetic-example ./bin/audita config validate --config examples/production-config.yml
|
||||||
|
AUDITA_SERVICE_KEY=synthetic-example ./bin/audita config validate --config examples/production-config.yml --overlay testing
|
||||||
|
./bin/audita config print-effective --config examples/minimal-config.yml --view profiles
|
||||||
|
./bin/audita config print-effective --config examples/configkit-inheritance.yml --view explain
|
||||||
```
|
```
|
||||||
|
|
||||||
The [official release-plugin documentation](https://woodpecker-ci.org/plugins/release)
|
Confirm local/service roots and testing selections work without provider
|
||||||
supports a file path in `note`; the
|
dispatch. Check safe views exclude resolved secrets and preserve the
|
||||||
[environment reference](https://woodpecker-ci.org/docs/usage/environment)
|
established explanation fields. Do not call a model to validate examples.
|
||||||
makes `CI_COMMIT_TAG` available for configuration substitution. These interfaces
|
4. Check the selected dependency graph, attribution, affected links, and final
|
||||||
were checked during planning. Use the checked-out tagged note directly; no
|
diff. Run `git diff --check`. Confirm only the released modules are used and
|
||||||
generator, custom publisher, or additional dependency is needed. Preserve
|
no obsolete `/v2` catalog or removed construction API remains in active code.
|
||||||
`overwrite: false`, `file-exists: skip`, existing assets/checksum settings,
|
Historical release documents and historical roadmap links may retain old
|
||||||
secret reference, image, and step dependency.
|
versions as accurate records.
|
||||||
2. In `docs/release.md`, add the footer preparation instruction with this pattern
|
5. Mark this plan and the feature roadmap complete only after the full target
|
||||||
(substitute the selected release version for `<version>`):
|
passes. Record the tested implementation revision and checks accurately;
|
||||||
|
distinguish a subsequent documentation-only completion commit. Commit the
|
||||||
|
completion documentation and report remaining work, if any. Do not select
|
||||||
|
a release number, publish, tag, push, poll CI, or add permanent release gates.
|
||||||
|
|
||||||
```markdown
|
**Completion marker:** Stage 9 complete. The tested implementation revision is
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/docs/oss/attribution.md)
|
`5525fe8`; the preceding documentation migration commit is `9bcf183`, and this
|
||||||
```
|
roadmap closure is documentation-only. At `9bcf183`, the application built,
|
||||||
|
`go test ./...` passed, all six maintained example commands passed offline,
|
||||||
|
`go mod verify` passed, the selected module graph contained PromptKit v0.12.0
|
||||||
|
and both `/v3` catalogs at `v3.20260930.0` with no `/v2` catalogs, and
|
||||||
|
`git diff --check` passed. The active Go/module scan found no removed PromptKit
|
||||||
|
construction options.
|
||||||
|
|
||||||
Explain that the plugin uses the checked-in note for new hosted releases and
|
**End state:** Native PromptKit configuration and catalogs are adopted,
|
||||||
preserves existing release information on retries. Keep old notes and pages
|
operator overrides and captured resources work through the real runtime,
|
||||||
intact. Validate intended paths locally during preparation; do not add remote
|
application backend/domain ownership and resilience remain effective, and
|
||||||
footer checks, a footer parser, or a new release-script gate.
|
maintained examples and migration documentation match the implementation.
|
||||||
3. Align the runbook with the adopted policy: retain `origin`, `main`, atomic
|
No implementation-blocking decisions remain; exact major-release numbering
|
||||||
source/tag publication, one offline validation pass, required Linux builds,
|
and publication timing remain outside this implementation unit.
|
||||||
optional check mode, and asynchronous immutable delivery retries. Reference
|
|
||||||
the policy for requirements rather than introducing additional ones. Add
|
|
||||||
proportionate completed-roadmap retirement guidance for future release
|
|
||||||
preparation without making it a script-enforced condition. No release script
|
|
||||||
change is expected from this adoption.
|
|
||||||
4. Repair both current-plan links in `decision-set-reconciliation.md` to the
|
|
||||||
historical plan at the full Audita planning baseline listed above. Remove
|
|
||||||
the stale sentence in `library-adoption.md` saying the current implementation
|
|
||||||
plan covers span deletion. Preserve historical records and deferred work.
|
|
||||||
5. Review the final adoption diff against upstream and the feature roadmap.
|
|
||||||
Check all policy source metadata, template removal, local exception reasons,
|
|
||||||
copied-material scope, license text fidelity, local links/anchors, and pinned
|
|
||||||
upstream destinations. Ensure the feature roadmap records implemented status
|
|
||||||
only after its accepted outcomes are achieved.
|
|
||||||
6. Parse/review the affected YAML using an available YAML parser or Woodpecker
|
|
||||||
linter. Substitute a representative existing release tag locally and confirm
|
|
||||||
the note path resolves to the tracked note; old notes need not have the new
|
|
||||||
footer. Check a temporary example of the new footer against candidate paths.
|
|
||||||
Verify non-overwriting publication settings remain unchanged. Do not call the
|
|
||||||
forge to create a test release, run a plugin with production credentials, or
|
|
||||||
wait for CI. Static configuration and documented plugin-interface validation
|
|
||||||
are sufficient for this wiring change; report that hosted execution awaits
|
|
||||||
the next real release rather than claiming it was exercised.
|
|
||||||
|
|
||||||
**Validation and completion:** Run `git diff --check`, review staged content and
|
|
||||||
all affected links, and record the final verification evidence. No full Go suite
|
|
||||||
or build matrix is required when only documents and the note setting changed.
|
|
||||||
If an actual release-script behavior change becomes necessary, justify its
|
|
||||||
scope and run focused `go test ./scripts` coverage for that behavior rather than
|
|
||||||
inventing additional release gates. Commit the completed work; leave a clean
|
|
||||||
workspace without publishing.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
None requiring a policy or design decision before implementation. The upstream
|
|
||||||
revision, local exceptions, attribution arrangement, target documents, and
|
|
||||||
release-body mechanism are selected. Exact component notice details and source
|
|
||||||
revisions are factual verification work in Stage 3, not permission gates. If
|
|
||||||
that inspection reveals an uncovered obligation, follow the adopted exception
|
|
||||||
procedure and identify the concrete decision instead of silently expanding scope.
|
|
||||||
|
|||||||
@@ -1,128 +0,0 @@
|
|||||||
# Coordinated ConfigKit and PromptKit Adoption
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-28
|
|
||||||
|
|
||||||
**Status:** Production implementation complete. ConfigKit v0.4.2 and PromptKit
|
|
||||||
v0.11.2 are adopted; [historical implementation evidence](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) records the
|
|
||||||
completed checks. Release compliance and publication remain separate work.
|
|
||||||
|
|
||||||
## Goal and Ownership
|
|
||||||
|
|
||||||
Adopt ConfigKit and PromptKit to align Audita with the application suite's shared
|
|
||||||
configuration and LLM execution conventions. Consistency is the primary benefit;
|
|
||||||
removing overlapping application infrastructure is a secondary benefit. Preserve
|
|
||||||
Audita's transcript-correction capabilities and domain protections.
|
|
||||||
|
|
||||||
| Owner | Responsibilities |
|
|
||||||
| --- | --- |
|
|
||||||
| ConfigKit | Strict application configuration loading, presence/default semantics, explicit overrides, recorded resolution, secret references, and safe inspection |
|
|
||||||
| PromptKit | Prompt/profile formats and rendering, provider calls, structural output validation and repair, transient retries, engine-local backend admission, and execution observations |
|
|
||||||
| Audita | Setting meanings, file/CLI selection, role/profile association, normalization/chunking, ordered modules, domain validation, correction application, reports, diagnostic redaction, and artifact lifecycle |
|
|
||||||
|
|
||||||
The [configuration roadmap](configkit.md) owns configuration and inspection policy.
|
|
||||||
The [boundary findings](library-boundary-findings.md) own the selected defaults,
|
|
||||||
full migration inventory, failure mapping, and prototype evidence. The selected
|
|
||||||
production versions are ConfigKit v0.4.2 and PromptKit v0.11.2, with Go 1.25.5 as
|
|
||||||
the minimum. Use published modules, never sibling checkouts or local replacements.
|
|
||||||
The existing YAML dependency remains while glossary parsing needs it.
|
|
||||||
|
|
||||||
## Migration Posture
|
|
||||||
|
|
||||||
This is an intentional breaking transition targeting Audita v2.0.0. No backward
|
|
||||||
compatibility is required for existing installations, config files, CLI flags,
|
|
||||||
environment overlays, or inspection output. Provide one migration guide, without
|
|
||||||
legacy loaders, aliases, dual modes, or automatic conversion tooling.
|
|
||||||
|
|
||||||
Both libraries belong to one refactor. Intermediate commits need not build or run;
|
|
||||||
there is no supported intermediate product state. No temporary adapters, compatibility
|
|
||||||
shims, or processing gates are needed. The complete application must work and pass
|
|
||||||
its checks before the refactor is finished. Release compliance and publication
|
|
||||||
come last, with no intermediate public release.
|
|
||||||
|
|
||||||
## Configuration and Execution Target
|
|
||||||
|
|
||||||
ConfigKit selected overlays enable a testing configuration that chooses cheaper or
|
|
||||||
faster models. The example selects both proposal and validation profiles. ConfigKit
|
|
||||||
application overlays and PromptKit execution profiles remain separate concepts.
|
|
||||||
Complete operator-maintained profiles own model/generation controls; embedded
|
|
||||||
application prompts and schemas are not operator-selected files. No catalog wiring,
|
|
||||||
profile-inheritance feature, ConfigKit imports, or secret-directory bootstrap is
|
|
||||||
part of this migration.
|
|
||||||
|
|
||||||
Capacity controls serve two needs: respecting a local service's hard simultaneous-
|
|
||||||
query limit and bounding concurrent paid API activity. One finite limit per backend
|
|
||||||
is shared across both roles and every initial, retry, and repair attempt. Limits
|
|
||||||
are engine-local, not coordinated across processes. No additional global or
|
|
||||||
per-role limiter or fine-grained scheduling policy is required. Profiles must use
|
|
||||||
registered backends and ConfigKit-resolved credentials; endpoint-only bypass and
|
|
||||||
competing profile credential lookup are excluded.
|
|
||||||
|
|
||||||
Structural repair is enabled by default for both roles because additional attempts
|
|
||||||
to obtain valid output are desired. Transient retries and structural repairs have
|
|
||||||
separate budgets; explicit zero disables the respective behavior. Each generation,
|
|
||||||
including a repair, receives its own transient-retry budget. The Audita logical-call
|
|
||||||
deadline covers preparation, waiting, backoff, all generations, and repairs; profile
|
|
||||||
HTTP timeouts apply per attempt. Exact defaults and bounds are in the
|
|
||||||
[selected contract](library-boundary-findings.md#configuration-and-profile-contract).
|
|
||||||
No Audita outer retry loop restores the old broader retry policy.
|
|
||||||
|
|
||||||
Audita retains all four correction modules, five LLM-validator prompts,
|
|
||||||
deterministic validators, confidence/applicability checks, glossary protection,
|
|
||||||
validation batching/cardinality, deterministic proposal and ledger ordering,
|
|
||||||
transcript encoders, and artifact retention. PromptKit-valid JSON still passes
|
|
||||||
Audita's domain checks. Repair exhaustion, provider failure, and domain rejection
|
|
||||||
follow the [selected failure table](library-boundary-findings.md#execution-and-failure-mapping).
|
|
||||||
Retention and transcript timestamp changes remain separate work; this migration
|
|
||||||
resolves the [retry-count follow-up](runtime-behavior.md).
|
|
||||||
|
|
||||||
## Reporting and Diagnostic Target
|
|
||||||
|
|
||||||
Keep stable Audita-owned serialized records; do not expose raw library structs.
|
|
||||||
Use process report schema `v2` with the existing domain outcomes and a new
|
|
||||||
`llm_summary` separating logical calls, dispatched provider attempts, transient
|
|
||||||
retries, structural repairs, and usage. Transcript schema versions remain unchanged.
|
|
||||||
The implementation plan specifies the concrete new fields and artifact wiring.
|
|
||||||
|
|
||||||
Usage distinguishes known zero, unknown, and incomplete totals. Preserve observations
|
|
||||||
from failed calls even when no final result exists. Count terminal totals once;
|
|
||||||
attempt observations provide diagnosis without being added to those totals again.
|
|
||||||
This is operational evidence, not a complete billing ledger.
|
|
||||||
|
|
||||||
Replace global/role scheduler utilization with per-backend configured capacity and
|
|
||||||
observed queue/generation/retry/logical-call timing. Do not fabricate measurements
|
|
||||||
that the event API does not provide. Keep domain timing and deterministic correction
|
|
||||||
ledger ordering. Record prompt definition/rendered hashes and selected profile/model
|
|
||||||
without promising the old hash values. Callbacks are short, concurrency-safe, and
|
|
||||||
free of blocking I/O or waits on their own execution.
|
|
||||||
|
|
||||||
Effective configuration and selected-profile snapshots remain distinct safe views.
|
|
||||||
Secret values must not escape through inspection, logs, reports, runtime errors,
|
|
||||||
or retained diagnostic files. Observation records exclude payloads, URLs, raw
|
|
||||||
provider messages, prompts, and credentials. Existing artifact lifecycle and
|
|
||||||
partial-failure handling remain Audita responsibilities.
|
|
||||||
|
|
||||||
## Completion Criteria and Release Boundary
|
|
||||||
|
|
||||||
- Both libraries own their selected responsibilities in the production application.
|
|
||||||
Superseded loaders, transport, renderers, schedulers, aliases, and the isolated
|
|
||||||
prototype are gone once its useful evidence is represented in production tests.
|
|
||||||
- Offline tests prove configuration-to-execution routing for both roles, selected
|
|
||||||
overlays and explicit overrides, shared backend capacity, retry/repair outcomes,
|
|
||||||
cancellation/deadlines, domain decisions, redaction, and truthful partial reports.
|
|
||||||
Tests exercise real internal collaborators and controlled HTTP boundaries.
|
|
||||||
- Maintained examples and canonical docs describe the implemented final model.
|
|
||||||
One migration guide covers files, flags, environment inputs, profiles, credentials,
|
|
||||||
overlays, inspection, and changed runtime/report/diagnostic semantics.
|
|
||||||
- The assembled application passes its required checks; Linux amd64 and arm64 build.
|
|
||||||
macOS arm64 remains best effort. No live-provider or CI-success gate is added.
|
|
||||||
|
|
||||||
PromptKit v0.11.0's reviewed retry, observation, and capacity APIs remove the earlier
|
|
||||||
upstream feature prerequisites; no upstream edit is required for adoption. The
|
|
||||||
prototype proved representative behavior, not release compliance or model quality.
|
|
||||||
|
|
||||||
The selected PromptKit release uses LGPL-3.0-or-later. A later release unit must
|
|
||||||
review the actual production dependency graph and distribution requirements,
|
|
||||||
including notices and source/rebuild materials, before binary publication. This
|
|
||||||
implementation plan does not claim that review is complete. Release publication
|
|
||||||
uses the existing [procedure](../release.md) and completes on successful source/tag
|
|
||||||
push, without waiting for CI or artifact delivery.
|
|
||||||
@@ -1,392 +0,0 @@
|
|||||||
# Shared Library Boundary: Decisions and Evidence
|
|
||||||
|
|
||||||
**Status:** Boundary and offline prototype completed, 2026-09-23. Selected prototype contract and proposed
|
|
||||||
production migration target; not the current Audita configuration contract.
|
|
||||||
The [bounded roadmap](library-boundary.md) owns scope and completion criteria.
|
|
||||||
|
|
||||||
**Subsequent scope decision:** The [ConfigKit overlay plan](configkit.md#selected-overlays-for-testing)
|
|
||||||
adds optional selected overlays, with v0.4.0 as the migration baseline. It supersedes
|
|
||||||
the original no-overlay decision and adds an input layer between the main file and
|
|
||||||
explicit setting overrides. The v0.3.1 prototype results below remain historical
|
|
||||||
evidence; they do not establish overlay integration.
|
|
||||||
|
|
||||||
## Selected Boundary
|
|
||||||
|
|
||||||
The former prototype is retained in Git history at `590604b`; it was a separate
|
|
||||||
Go module using published ConfigKit v0.4.0, PromptKit v0.11.0, and
|
|
||||||
Audita v1.1.0. The Audita dependency lets this consumer reuse actual internal domain
|
|
||||||
helpers through its Audita module namespace without a local replacement. It does
|
|
||||||
not make those internal packages a new public API. Remove this dependency along
|
|
||||||
with the prototype after equivalent production integration tests landed.
|
|
||||||
|
|
||||||
The initial prototype results below were recorded with ConfigKit v0.3.1. On
|
|
||||||
2026-09-24, upgrading only its ConfigKit pin and sums to v0.4.0 required no source
|
|
||||||
changes. With Go 1.26.8 on linux/amd64, the nested offline `go test ./...` passed
|
|
||||||
and `go run .` still reported one accepted correction, four attempts, zero
|
|
||||||
transient retries, and two structural repairs. This verifies the dependency
|
|
||||||
upgrade, not selected-overlay behavior.
|
|
||||||
|
|
||||||
ConfigKit owns loading, presence, explicit overrides, secret resolution, and
|
|
||||||
redacted effective settings. PromptKit owns profile parsing, rendering, provider
|
|
||||||
execution, transient retries, structural repair, and backend admission. Audita
|
|
||||||
owns file selection, setting meanings, role selection, domain acceptance, and
|
|
||||||
mapping execution facts into reports. Production CLI and runtime remain unchanged.
|
|
||||||
|
|
||||||
## Configuration and Profile Contract
|
|
||||||
|
|
||||||
Selected for the prototype and recommended for production:
|
|
||||||
|
|
||||||
- Main YAML requires integer `version: 2`; absent, null, unknown fields, and older
|
|
||||||
versions fail. No legacy parser, aliases, environment overlays, or automatic
|
|
||||||
conversion. Defaults precede the selected file; explicitly supplied CLI settings
|
|
||||||
become ConfigKit overrides. Validation runs on the resolved result, so an override
|
|
||||||
may repair an invalid lower-precedence setting when decoding succeeds.
|
|
||||||
- Production file selection: explicit `--config`, then
|
|
||||||
`/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`. Select one file;
|
|
||||||
do not merge them. Missing system files are skipped; explicit missing or empty
|
|
||||||
paths and inaccessible selected files fail. No selected file fails with a request
|
|
||||||
to configure backend/profile selection: no hidden provider or model default.
|
|
||||||
Remove `AUDITA_CONFIG`. The prototype loads its explicit fixture only; discovery
|
|
||||||
and command parsing belong to the ConfigKit implementation stage.
|
|
||||||
- `profiles_dir` defaults to `profiles`, resolved relative to the selected main
|
|
||||||
configuration file, with absolute paths accepted. PromptKit reads that directory.
|
|
||||||
Prompts and schemas are application-embedded, not operator-selected prompt files.
|
|
||||||
The prototype uses complete application-owned profile files. Catalog profiles and
|
|
||||||
profile inheritance are not needed for this migration; do not add catalog wiring.
|
|
||||||
- `backends.<id>.endpoint` is required. Use one application backend identity per
|
|
||||||
service capacity pool. Do not split proposal and validation into different backend
|
|
||||||
IDs when they use the same service. Limits are engine-local, not shared across
|
|
||||||
multiple Audita processes or other applications.
|
|
||||||
- `backends.<id>.concurrency` defaults to **1**, must be positive, and is shared by
|
|
||||||
both roles and all initial/retry/repair attempts. Operators can select **4** for
|
|
||||||
the described local backend or a suitable finite paid API limit. There is no
|
|
||||||
additional global or per-role limiter. `queue_capacity` defaults to **1024**,
|
|
||||||
following PromptKit; non-negative, with zero meaning no waiting admission.
|
|
||||||
This is an operational escape hatch rather than a second scheduling hierarchy.
|
|
||||||
- `backends.<id>.credential` is an optional ConfigKit secret reference: a scalar
|
|
||||||
environment variable name. Omission or null means no credential. A supplied
|
|
||||||
reference must resolve to a nonempty value, including during validation and
|
|
||||||
inspection. There is no direct key field or CLI key flag. ConfigKit resolves it
|
|
||||||
once and Audita supplies `RunRequest.APIKey`. Profiles must not introduce another
|
|
||||||
`api_key_env` lookup. Backend credentials also support unauthenticated local use.
|
|
||||||
- Profiles own model, temperature, output token limit, top-p, per-generation timeout,
|
|
||||||
service tier, reasoning effort, and extra provider parameters using PromptKit's
|
|
||||||
format. They select the application `backend`; their effective endpoint must
|
|
||||||
equal that backend's endpoint. Endpoint-only execution is rejected because it
|
|
||||||
bypasses the requested finite capacity boundary. A named registered backend is
|
|
||||||
mandatory even if both roles use the same profile.
|
|
||||||
- `llm.proposal_profile` is required. Omitted `llm.validation_profile` inherits the
|
|
||||||
final proposal profile as a whole, with ConfigKit recording the derivation.
|
|
||||||
Explicit empty or null values fail; no individual model/endpoint/key inheritance.
|
|
||||||
- `llm.transient_retries` defaults to **2** additional attempts per generation;
|
|
||||||
explicit zero disables retries; negative is invalid. PromptKit owns which failures
|
|
||||||
are eligible. Do not add an Audita retry loop.
|
|
||||||
- `llm.repair_attempts` defaults to **2** additional generations for both roles;
|
|
||||||
explicit zero disables repair when deliberately requested. PromptKit supports
|
|
||||||
0–3. Positive defaults are exercised in the prototype. Audita owns this common
|
|
||||||
setting and supplies a complete request output contract with the role's embedded
|
|
||||||
schema, avoiding separate user-facing repair defaults in each prompt.
|
|
||||||
- `llm.operation_timeout` defaults to **15m**, must be positive, and uses a Go
|
|
||||||
duration string. It starts before each logical proposal/assessment call and covers
|
|
||||||
preparation, capacity waiting, transient backoff, generation, and repairs. It is
|
|
||||||
separate from whole-process cancellation. Profile `timeout_seconds` defaults to
|
|
||||||
PromptKit's **600** seconds and is applied afresh to each built-in HTTP attempt.
|
|
||||||
It does not bound the combined retry/repair sequence. A stricter caller/transport timeout can
|
|
||||||
also end execution. The two defaults are selected starting points, not throughput
|
|
||||||
measurements. With the default T=2 and R=2, each logical call dispatches at most
|
|
||||||
**9** attempts before deadline/cancellation; proposal plus assessment may use 18.
|
|
||||||
- Omission retains static defaults or invokes the recorded role/backend derivations.
|
|
||||||
Null is not a general synonym for omission: nonoptional scalars reject it. Optional
|
|
||||||
credentials and the future optional section target accept null. Explicit empty
|
|
||||||
module lists will mean an empty correction pipeline, not “restore defaults.”
|
|
||||||
Repeated modules retain their existing ordered meaning.
|
|
||||||
- The original prototype disabled ConfigKit imports, selected configuration layers
|
|
||||||
(then called configuration profiles), and secret-directory bootstrap. Selected
|
|
||||||
overlays are now verified under the linked overlay decision; the other two
|
|
||||||
features remain disabled. PromptKit execution profiles retain their meaning.
|
|
||||||
|
|
||||||
Configuration validation/inspection must assemble and inspect both selected profiles
|
|
||||||
without provider calls. ConfigKit's effective YAML describes only its loaded state;
|
|
||||||
a separate selected-profile view shows effective model controls/backend selection.
|
|
||||||
Do not imply either snapshot alone represents the full execution configuration.
|
|
||||||
The prototype exercises the two APIs but does not implement production subcommands.
|
|
||||||
Do not serialize raw profile/library objects as a permanent Audita report schema.
|
|
||||||
|
|
||||||
## Migration Inventory
|
|
||||||
|
|
||||||
This inventory covers the current configuration and process flags documented in
|
|
||||||
[configuration](../config.md) and [CLI](../cli.md), including environment-only fields.
|
|
||||||
“Retain” below means retain domain meaning in the new schema, not accept old files.
|
|
||||||
All legacy environment overlays listed afterward are removed.
|
|
||||||
|
|
||||||
| Current file field / setting | Current CLI flag(s) | Target and rationale |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| File selection | `--config` | Retain explicit invocation flag; discovery described above. |
|
|
||||||
| Transcript input | positional path | Invocation only; retain one transcript path. |
|
|
||||||
| Glossary input | `--glossary` | Invocation only; retain required glossary path. |
|
|
||||||
| Corrected output | `--output` | Invocation only; retain stdout fallback. |
|
|
||||||
| Report output | `--report-json` | Invocation only; retain separate report file. |
|
|
||||||
| `pipeline.modules` | `--modules` | Retain sequence/default `[glossary, homophones, glossary, spoken_word, grammar]`; explicit empty clears rather than inherits. |
|
|
||||||
| `output.schema` | `--output-schema` | Retain encoder selection/default `bare-segments`. |
|
|
||||||
| `llm.proposal.base_url`, `llm.validation.base_url` | `--base-url`, `--validation-base-url` | Replace with backend endpoint; remove per-role endpoint flags. |
|
|
||||||
| Both `llm.*.model` | `--model`, `--validation-model` | Replace with profile model; new `--proposal-profile` / `--validation-profile` select whole profiles. |
|
|
||||||
| Both `llm.*.api_key_env`, direct runtime keys | `--llm-api-key`, `--validation-llm-api-key` | Replace with backend ConfigKit secret reference; remove direct key flags. |
|
|
||||||
| Both `llm.*.timeout` | `--llm-timeout-seconds`, `--validation-llm-timeout-seconds` | Replace with profile `timeout_seconds`, plus `llm.operation_timeout`; no duplicate timeout flags. |
|
|
||||||
| Both `llm.*.max_retries` | `--max-retries`, `--validation-max-retries` | Replace with common `llm.transient_retries` and distinct `llm.repair_attempts`; remove misleading old flags. |
|
|
||||||
| `concurrency.total_llm` | `--total-llm-concurrency`, `--llm-concurrency` | Replace with per-backend concurrency; remove global/alias flags. |
|
|
||||||
| `concurrency.proposal_llm`, `concurrency.validation_llm` | `--proposal-llm-concurrency`, `--validation-llm-concurrency` | Remove role budgets; shared backend handles service capacity. |
|
|
||||||
| Validation prompt budget (no old YAML field) | `--validation-max-prompt-tokens` | Add `validation.max_prompt_tokens`, default 2048, positive; retain explicit flag override. Audita owns batching. |
|
|
||||||
| `chunking.target_sections` | `--target-sections` | Retain optional positive target; omission/null means automatic, zero invalid. |
|
|
||||||
| `chunking.max_section_tokens` | `--max-section-tokens` | Retain 8192, positive. |
|
|
||||||
| `chunking.min_section_tokens` | `--min-section-tokens` | Retain 2048, positive, no greater than maximum. |
|
|
||||||
| `normalization.max_segment_gap` | `--normalize-max-segment-gap` | Retain 4.0 numeric seconds, non-negative; drop alternate YAML duration syntax for these domain numeric fields. |
|
|
||||||
| `normalization.ellipsis_gap` | `--normalize-ellipsis-gap` | Retain 3.5 numeric seconds, non-negative. |
|
|
||||||
| `normalization.max_segment_duration` | `--normalize-max-segment-duration` | Retain 60.0 numeric seconds, positive. |
|
|
||||||
| `normalization.max_segment_tokens` | `--normalize-max-segment-tokens` | Retain 2048, positive. |
|
|
||||||
| `thresholds.glossary` | `--glossary-confidence-threshold` | Retain 0.8 in [0,1]. |
|
|
||||||
| `thresholds.homophones` | `--homophones-confidence-threshold` | Retain 0.8 in [0,1]. |
|
|
||||||
| `thresholds.spoken_word` | `--spoken-word-confidence-threshold` | Retain 0.8 in [0,1]. |
|
|
||||||
| `thresholds.grammar` | `--grammar-confidence-threshold` | Retain 0.8 in [0,1]. |
|
|
||||||
| `context.description` | `--transcript-description` | Retain trimming and 500-byte limit; empty clears context. |
|
|
||||||
| `diagnostics.work_dir` | `--work-dir` | Retain `/tmp/audita`, nonempty; artifact lifecycle remains Audita-owned. |
|
|
||||||
| `diagnostics.retention` | `--work-dir-retention` | Retain `auto` default and selection; existing `never` follow-up is separate. |
|
|
||||||
| Help/version/config commands | `--help`, `-h` | Retain command purpose; help should parse before loading configuration. Effective-config output intentionally becomes ConfigKit YAML, not legacy JSON. |
|
|
||||||
|
|
||||||
Retained setting flags become explicit ConfigKit overrides only when supplied;
|
|
||||||
CLI parser defaults must not overwrite file values. Normalization flag values stay
|
|
||||||
numeric seconds. Domain values convert into existing domain structs after validation;
|
|
||||||
do not spread ConfigKit or PromptKit types into correction algorithms.
|
|
||||||
|
|
||||||
Complete environment removal map (names remain permissible if explicitly named as
|
|
||||||
secret references; none have implicit configuration meaning):
|
|
||||||
|
|
||||||
| Old environment names | Replacement |
|
|
||||||
| --- | --- |
|
|
||||||
| `AUDITA_CONFIG` | File selection above. |
|
|
||||||
| `AUDITA_MODULES` | `pipeline.modules` / explicit flag. |
|
|
||||||
| `AUDITA_LLM_API_KEY`, `OPENROUTER_API_KEY`, `AUDITA_VALIDATION_LLM_API_KEY` | Explicit backend credential references; no implicit fallback. |
|
|
||||||
| `AUDITA_MODEL`, `AUDITA_VALIDATION_MODEL` | Profile files and role selection. |
|
|
||||||
| `AUDITA_BASE_URL`, `AUDITA_VALIDATION_BASE_URL` | Backend endpoints. |
|
|
||||||
| `AUDITA_LLM_TIMEOUT_SECONDS`, `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | Profile timeout and operation deadline. |
|
|
||||||
| `AUDITA_MAX_RETRIES`, `AUDITA_VALIDATION_MAX_RETRIES` | Common transient/repair settings. |
|
|
||||||
| `AUDITA_TOTAL_LLM_CONCURRENCY`, `AUDITA_LLM_CONCURRENCY`, `AUDITA_PROPOSAL_LLM_CONCURRENCY`, `AUDITA_VALIDATION_LLM_CONCURRENCY` | Backend concurrency; remove alias/role budgets. |
|
|
||||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `validation.max_prompt_tokens` / explicit flag. |
|
|
||||||
| `AUDITA_TARGET_SECTIONS`, `AUDITA_MAX_SECTION_TOKENS`, `AUDITA_MIN_SECTION_TOKENS` | Corresponding chunking fields / flags. |
|
|
||||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`, `AUDITA_NORMALIZE_ELLIPSIS_GAP`, `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`, `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | Corresponding normalization fields / flags. |
|
|
||||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`, `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`, `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`, `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | Corresponding threshold fields / flags. |
|
|
||||||
| `AUDITA_WORK_DIR`, `AUDITA_WORK_DIR_RETENTION` | Diagnostics fields / flags. |
|
|
||||||
|
|
||||||
## Execution and Failure Mapping
|
|
||||||
|
|
||||||
The executable uses grammar proposal plus grammar review, current response schemas,
|
|
||||||
current hardening text, transcript-section/description helpers, proposal preview, and
|
|
||||||
actual decision-cardinality enforcement from Audita v1.1.0. PromptKit renders the
|
|
||||||
adapted templates; the custom HTTP transport supplies OpenAI-compatible responses.
|
|
||||||
The prototype pre-expands Audita's `hardening` template function into the embedded
|
|
||||||
PromptKit definitions; it passes existing named string variables unchanged. No
|
|
||||||
provider, paid call, or injected high-level LLM client participates.
|
|
||||||
|
|
||||||
Generated JSON is structurally validated/repaired by PromptKit before Audita decodes
|
|
||||||
it. Audita then checks proposal applicability, confidence, and decision identity and
|
|
||||||
cardinality. The demonstration previews corrections without mutation. Production
|
|
||||||
retains every module's full validator chain, glossary protection, batching, and
|
|
||||||
apply-time checks; the representative prototype does not claim to replace them.
|
|
||||||
|
|
||||||
| Outcome | Section/batch effect | Overall run |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| Structural repair succeeds | Continue domain validation. | Normal execution. |
|
|
||||||
| Proposal repair exhausted | No proposals from this section; malformed-output warning. | Continue other work; successful completion may include warnings. |
|
|
||||||
| Assessment repair exhausted | Reject this batch; malformed-output warning. | Continue other work. |
|
|
||||||
| Schema-valid invalid/missing/duplicate decision indices | Reject batch and warn; no semantic retry/repair. | Continue other work. |
|
|
||||||
| Inapplicable/low-confidence proposal or negative assessment | Reject affected proposal; preserve domain reasons. | Continue other work. |
|
|
||||||
| Provider failure, retry exhaustion, transport or malformed HTTP response | Stop affected operation; retain available observations. | Runtime failure; cancel remaining work, retain existing Audita diagnostic handling. |
|
|
||||||
| Deadline, caller cancellation, capacity admission failure | Stop affected operation; no outer retry or fabricated output. | Runtime failure with distinct classified cause. |
|
|
||||||
| Invalid settings/profile/schema setup | Fail before processing where detectable. | Configuration error for user input, runtime error for broken embedded assets. |
|
|
||||||
|
|
||||||
Zero transient retries produces one ordinary provider failure, not
|
|
||||||
`ErrRetriesExhausted`; the latter signals an exhausted **positive** budget.
|
|
||||||
Provider failures retain a typed cause. Do not expose raw provider messages through
|
|
||||||
user diagnostics: continue Audita's separate secret-redaction boundary. A failed
|
|
||||||
logical call may return no result while its observations still record completed
|
|
||||||
attempts or an earlier successful generation. Already completed domain results can
|
|
||||||
remain in diagnostics; failure must not be presented as a fully corrected output.
|
|
||||||
|
|
||||||
## Observation and Report Handoff
|
|
||||||
|
|
||||||
The prototype maps library events into Audita-owned records under a short mutex.
|
|
||||||
Records contain logical run/role/backend/profile identity, generation and attempt
|
|
||||||
indices, dispatch state, queue/generation/retry/wall timing, classified outcomes,
|
|
||||||
HTTP status, exhaustion, repair/retry counts, and each usage field's value,
|
|
||||||
knownness, and completeness. It omits URLs, credentials, payloads, raw provider
|
|
||||||
errors, and prompt text. Identifiers themselves are metadata, not a secret channel.
|
|
||||||
|
|
||||||
Use terminal records once for logical totals; retain attempt records for diagnosis
|
|
||||||
without adding them to terminal totals. A known zero is distinct from unknown;
|
|
||||||
partial totals remain incomplete after unmetered failed attempts. These observations
|
|
||||||
are not a billing ledger. Callback code must not wait on its own run's progress.
|
|
||||||
|
|
||||||
Production report changes must distinguish logical calls from dispatched provider
|
|
||||||
attempts. Replace scheduler utilization fields based on global/proposal/validation
|
|
||||||
pools with per-backend admission/active timing. Preserve deterministic ledger and
|
|
||||||
validator ordering. Map prompt definition/rendered hashes and selected profile/model
|
|
||||||
into request metadata, but do not promise equality with legacy prompt hashes or
|
|
||||||
serialize PromptKit's raw event struct as the stable report contract. Update the
|
|
||||||
output integration reference, report mapper tests, and downstream migration guide
|
|
||||||
when the production representation is implemented.
|
|
||||||
|
|
||||||
## Verification and Remaining Handoff
|
|
||||||
|
|
||||||
The focused matrix is implemented in the nested module:
|
|
||||||
|
|
||||||
| Roadmap scenario | Evidence |
|
|
||||||
| --- | --- |
|
|
||||||
| Successful assembled path | `go run .`; `TestWorkflowBoundaries/repair_both_roles`; actual Audita domain helpers and both selected profiles. |
|
|
||||||
| Presence, overrides, inheritance | `TestConfigurationContract`: omitted role/backend values, explicit zero, null/empty, version, unknown keys, and explicit overrides. |
|
|
||||||
| Credentials | Missing/unset and empty references fail; optional null accepted; effective/profile inspection redacted; HTTP authorization verified; no secret/prompt text in observations. `TestProfilesRespectSharedBackendBoundary` rejects competing lookup and capacity bypass. |
|
|
||||||
| Both roles repaired | `repair_both_roles`: four attempts, two repairs, one accepted correction. |
|
|
||||||
| Repair exhausted | Proposal and assessment exhaustion subtests: bounded counts and selected warning/rejection outcomes. |
|
|
||||||
| Semantic boundary | Missing span, missing/duplicate decision indices, negative assessment; no semantic repair loop. |
|
|
||||||
| Retry/zero/exhaustion | 429 recovery, zero-budget 503, positive-budget exhaustion after completed proposal; typed error assertions. |
|
|
||||||
| Combined budgets | 503 → invalid JSON → repair 503 → valid proposal → assessment: five attempts, two transient retries, one repair. |
|
|
||||||
| Shared capacity | `TestSharedCapacityAndRecovery`: caps four and two, both roles, a retry and repair, queue rejection, measured peak equal to cap, later successful call. |
|
|
||||||
| Waiting/cancellation/deadline | `TestQueuedCancellationAndDeadline`; `TestDeadlineStopsRetryAfter`: no queued dispatch after cancellation/deadline and a 60-second provider retry delay interrupted by the operation deadline. |
|
|
||||||
| Failed-run observations | `TestPartialUsageOnFailedRun`: valid HTTP output with known-zero usage followed by repair failure; nil result, retained events, incomplete known aggregate, correct counts. |
|
|
||||||
| Representative assets | `TestRealPromptRenderingParity` compares both rendered message pairs to Audita v1.1.0; `TestSchemaFixturesMatchAudita` compares schema bytes. |
|
|
||||||
|
|
||||||
Commands executed from `examples/library-adoption` on Linux amd64 with Go 1.26.8:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
GOWORK=off go mod download all
|
|
||||||
GOWORK=off GOPROXY=off GOSUMDB=off go mod verify
|
|
||||||
GOWORK=off GOPROXY=off GOSUMDB=off go run .
|
|
||||||
GOWORK=off GOPROXY=off GOSUMDB=off go test -race ./...
|
|
||||||
GOWORK=off GOPROXY=off GOSUMDB=off go vet ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
All passed. The demonstration reports accepted=1, rejected=0, attempts=4,
|
|
||||||
transient_retries=0, structural_repairs=2. Module verification reports all modules
|
|
||||||
verified. Published tag availability was checked with `git ls-remote --tags` for
|
|
||||||
ConfigKit v0.3.1 and PromptKit v0.11.0; successful module acquisition verifies this
|
|
||||||
is not merely a local-checkout experiment. The module has no replacements.
|
|
||||||
|
|
||||||
The minimum Go 1.25.5 toolchain was also exercised with offline tests and the same
|
|
||||||
successful demonstration. The exact installed-toolchain invocation was:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off /home/eric/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.5.linux-amd64/bin/go test ./...
|
|
||||||
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off /home/eric/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.5.linux-amd64/bin/go run .
|
|
||||||
```
|
|
||||||
|
|
||||||
An initial attempt to select that cached toolchain through `GOTOOLCHAIN=go1.25.5`
|
|
||||||
with checksum verification disabled was rejected by Go's toolchain verifier. Using
|
|
||||||
the installed binary with `GOTOOLCHAIN=local` proved offline execution without
|
|
||||||
weakening dependency checks. The portable README commands assume an installed
|
|
||||||
sufficiently new Go executable on PATH and disable toolchain switching.
|
|
||||||
|
|
||||||
From the Audita root, `go test ./...` and
|
|
||||||
`go build -o ./bin/audita ./cmd/audita` passed. These check existing production code;
|
|
||||||
they do not cover the nested module. Affected relative documentation links,
|
|
||||||
inventory coverage, and `git diff --check` were checked separately.
|
|
||||||
|
|
||||||
### Findings and Limits
|
|
||||||
|
|
||||||
No adoption-blocking upstream API gap was found. Two integration details deserve
|
|
||||||
explicit attention: PromptKit's profile GoDoc calls `timeout_seconds` a generation
|
|
||||||
deadline, but its built-in HTTP path applies it separately to each attempt; the
|
|
||||||
consumer therefore needs the selected total-operation deadline. Also,
|
|
||||||
`APIKeyRequired` is an in-memory profile facility, not a profile-YAML field. The
|
|
||||||
file-profile design uses the presence of ConfigKit's optional credential reference
|
|
||||||
to require local secret resolution; it does not invent a new PromptKit YAML key.
|
|
||||||
|
|
||||||
The chosen defaults are proposed operational policy, not measured model-quality or
|
|
||||||
cost recommendations. Tests use controlled transport responses and synthetic data;
|
|
||||||
they do not establish real provider compatibility, model quality, network behavior,
|
|
||||||
throughput, cross-process coordination, complete billing, or release compliance.
|
|
||||||
Only the representative LLM-facing subset is implemented in the prototype; the
|
|
||||||
inventory specifies the remaining domain fields for the ConfigKit migration.
|
|
||||||
Production file discovery, CLI parsing, full module chains, all nine prompts,
|
|
||||||
filesystem diagnostics/redaction, and serialized report migration remain their
|
|
||||||
assigned production tasks. Those are deliberate scope boundaries, not missing
|
|
||||||
claims of completion for the prototype.
|
|
||||||
|
|
||||||
The new profile/credential/capacity and deadline choices are explicit above; no
|
|
||||||
unresolved upstream requirement prevents starting the ConfigKit implementation
|
|
||||||
plan. Production report field names and final runtime wiring are selected in the
|
|
||||||
[implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md), following the outcome and observation
|
|
||||||
contract here. The coordinated refactor targets both libraries directly; intermediate commits do not require a working legacy runtime.
|
|
||||||
|
|
||||||
### Production Implementation Reference
|
|
||||||
|
|
||||||
The [implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) owns ordered production work,
|
|
||||||
component changes, tests, and acceptance criteria. The [combined roadmap](library-adoption.md)
|
|
||||||
and [configuration roadmap](configkit.md) own the final target. This evidence does
|
|
||||||
not require an interim runtime bridge or separately runnable configuration migration.
|
|
||||||
|
|
||||||
Retained domain scope includes transcript/glossary schemas, normalization/chunking,
|
|
||||||
output encoding, proposals and apply-time checks, deterministic validators, ordered
|
|
||||||
modules, LLM-validator batching/cardinality, ledger ordering, and artifact lifecycle.
|
|
||||||
The production integration replaces loading, provider execution, prompt rendering,
|
|
||||||
and scheduler composition while preserving those responsibilities. Retention and
|
|
||||||
timestamp changes remain separate work.
|
|
||||||
|
|
||||||
### Distribution Follow-Up
|
|
||||||
|
|
||||||
The pinned graph includes ConfigKit, PromptKit, both PromptKit backend catalogs,
|
|
||||||
jsonschema/v6, yaml.v3, yaml.v4, x/text, and kr/text (including module/test dependency
|
|
||||||
edges). Before distribution, review the actual production dependency graph and
|
|
||||||
its license files, including PromptKit/catalog LGPL-3.0-or-later terms. Decide and
|
|
||||||
verify the notices, license copies, corresponding library source/modifications,
|
|
||||||
and rebuild/relink materials needed for Audita's Go binary/container distribution.
|
|
||||||
This unit does not publish binaries or claim that release compliance is complete.
|
|
||||||
No upstream relicensing request or library implementation change is required for
|
|
||||||
this prototype.
|
|
||||||
|
|
||||||
## Selected-Overlay Extension: 2026-09-24
|
|
||||||
|
|
||||||
The isolated prototype now pins published ConfigKit v0.4.0. ConfigKit loads
|
|
||||||
`overlays/testing.yml` relative to the selected main file; the overlay selects
|
|
||||||
complete PromptKit execution profiles from the separate `profiles/` directory.
|
|
||||||
The original v0.3.1 observations above are historical and do not prove this
|
|
||||||
extension. No production Audita behavior changed.
|
|
||||||
|
|
||||||
With Go 1.26.8 on linux/amd64, from `examples/library-adoption/`, these offline
|
|
||||||
checks passed with `GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off`:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go test -race ./...
|
|
||||||
go vet ./...
|
|
||||||
go run .
|
|
||||||
go run . --overlay testing
|
|
||||||
go run . --overlay testing --proposal-profile proposal
|
|
||||||
```
|
|
||||||
|
|
||||||
Each demonstration reported one accepted correction, four attempts, zero
|
|
||||||
transient retries, and two structural repairs. The base run selected
|
|
||||||
`synthetic-grammar`/`synthetic-review`; testing selected
|
|
||||||
`synthetic-grammar-testing`/`synthetic-review-testing`; the explicit proposal
|
|
||||||
override restored only `synthetic-grammar`. These are synthetic routing IDs,
|
|
||||||
not price, latency, quality, or provider-compatibility measurements.
|
|
||||||
|
|
||||||
| Requirement | Consumer evidence |
|
|
||||||
| --- | --- |
|
|
||||||
| Base, empty/default, testing selection, role overrides, final inheritance, and relative profile resolution | `TestSelectedOverlayConfiguration` loads temporary on-disk layouts and assembles both roles. |
|
|
||||||
| Actual base/testing/override HTTP models and repairs | `TestDemonstrationSelectsOutboundModels` captures every request body and matches model IDs to the reported selections and domain outcome. |
|
|
||||||
| Missing/invalid selection, version/directive rejection, and ordinary backend-setting validation | `TestOverlayFailuresAndApplicationValidation` checks ConfigKit diagnostic codes and effective backend limits. |
|
|
||||||
| Surviving credential resolution, absent/empty failures, null unauthenticated use, and profile-only switching | `TestOverlayCredentialResolutionAndInspection` controls synthetic environment values and observes HTTP Authorization. |
|
|
||||||
| Redacted effective output, explanation and metadata origin, profile inspection, and observations | `TestOverlayCredentialResolutionAndInspection` and `TestOverlaySelectionProvenance` inspect safe output and overlay/CLI origins. |
|
|
||||||
| Existing retry, repair, domain, capacity, and concurrent-observation behavior | `TestWorkflowBoundaries`, `TestSharedCapacityAndRecovery`, `TestQueuedCancellationAndDeadline`, `TestDeadlineStopsRetryAfter`, and `TestPartialUsageOnFailedRun` pass, including under `-race`. |
|
|
||||||
|
|
||||||
Changing a profile does not remove a backend credential reference from the
|
|
||||||
effective configuration. ConfigKit validates a surviving reference when only
|
|
||||||
profile selection changes. Replacing the reference in an
|
|
||||||
overlay resolves only the replacement; explicit null allows unauthenticated use.
|
|
||||||
This distinction is part of the selected configuration boundary, not an
|
|
||||||
application fallback. Imports and secret-directory bootstrap remain disabled.
|
|
||||||
|
|
||||||
The tests use a controlled built-in HTTP transport and synthetic credentials;
|
|
||||||
there were no live provider calls. The prototype still covers one representative
|
|
||||||
proposal/assessment pair, not all production modules, prompts, CLI paths, or
|
|
||||||
diagnostic artifacts. Root `go test ./...` does not run this nested module. No
|
|
||||||
upstream API gap appeared in this extension. The [production implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md)
|
|
||||||
keeps implementation and release pending.
|
|
||||||
@@ -1,269 +0,0 @@
|
|||||||
# Library Adoption: Shared Boundary and Prototype
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-24
|
|
||||||
|
|
||||||
**Status:** Original boundary-design and offline-prototype unit completed;
|
|
||||||
overlay extension and production-planning handoff completed, 2026-09-24.
|
|
||||||
The work packages below retain the original scope;
|
|
||||||
[decisions, evidence, limitations, and handoff](library-boundary-findings.md)
|
|
||||||
record the outcome. The executable prototype was removed after production
|
|
||||||
integration; Git history at `590604b` retains it. No publication or upstream
|
|
||||||
changes were made in that historical unit.
|
|
||||||
|
|
||||||
## Next Unit: Testing Overlays and Production Migration Plan
|
|
||||||
|
|
||||||
**Status:** Complete. The [production implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md)
|
|
||||||
owns subsequent work; the [overlay findings](library-boundary-findings.md#selected-overlay-extension-2026-09-24)
|
|
||||||
record this unit's checks. The original work packages below remain
|
|
||||||
historical scope for the completed first unit.
|
|
||||||
|
|
||||||
Extend the existing prototype to prove the [selected-overlay contract](configkit.md#selected-overlays-for-testing)
|
|
||||||
with ConfigKit v0.4.0, then turn the verified findings into a detailed production
|
|
||||||
ConfigKit migration plan. Do not repeat the original inventory or redesign the
|
|
||||||
established library boundary without new evidence.
|
|
||||||
|
|
||||||
Required work:
|
|
||||||
|
|
||||||
- Upgrade only the nested prototype module from ConfigKit v0.3.1 to v0.4.0.
|
|
||||||
- Pass an explicit overlay selection through ConfigKit and add `overlays/testing.yml`
|
|
||||||
plus two PromptKit testing profiles. Select both roles and retain one shared
|
|
||||||
finite backend. Synthetic model names demonstrate routing; they do not benchmark
|
|
||||||
real model cost, speed, or quality.
|
|
||||||
- Make the offline executable accept overlay and explicit role-profile selections,
|
|
||||||
and show the selected models alongside domain outcomes and retry/repair counts.
|
|
||||||
- Verify base/overlay/CLI precedence, role inheritance, invalid selections,
|
|
||||||
surviving credential references, safe inspection, and overlay provenance.
|
|
||||||
Prove the actual HTTP request models for both roles; loading changed strings
|
|
||||||
alone is insufficient evidence.
|
|
||||||
- Record new results separately from the historical v0.3.1 findings. Expand the
|
|
||||||
existing ConfigKit roadmap into a concrete production implementation handoff,
|
|
||||||
including final runtime integration and removal of legacy configuration.
|
|
||||||
|
|
||||||
Target end state: reviewers can run the prototype with and without the testing
|
|
||||||
selection and see both LLM roles switch models. The offline test suite establishes
|
|
||||||
that selection, inspection, and execution agree, and the production ConfigKit work
|
|
||||||
has bounded tasks with explicit acceptance criteria.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- The published v0.4.0 dependency is pinned without local replacements; the original
|
|
||||||
no-overlay path and existing integration tests continue to pass.
|
|
||||||
- `testing` switches both roles, explicit profile flags override its selections,
|
|
||||||
and omission/empty/`default` retains base selection according to ConfigKit.
|
|
||||||
- Invalid overlays fail clearly; inheritance and secret resolution use surviving
|
|
||||||
effective settings. Inspection records overlay origin without disclosing keys.
|
|
||||||
- Both selected testing models reach the controlled HTTP transport; structural
|
|
||||||
repair, shared capacity, observations, and Audita domain checks remain active.
|
|
||||||
- Updated examples, executable instructions, findings, and a detailed production
|
|
||||||
migration plan are reviewable. Exact verification commands and limitations are
|
|
||||||
recorded, including the fact that root tests do not cover the nested module.
|
|
||||||
- Production code, root dependencies, and upstream libraries remain unchanged.
|
|
||||||
No release or live provider call is part of this unit.
|
|
||||||
|
|
||||||
This unit ends with a production migration **plan**, not production implementation.
|
|
||||||
The ConfigKit → PromptKit → combined release sequence remains unchanged. Imports
|
|
||||||
and secret-directory bootstrap remain disabled; PromptKit execution profiles keep
|
|
||||||
their existing names and meaning.
|
|
||||||
|
|
||||||
## Objective
|
|
||||||
|
|
||||||
Define and prove the boundary between Audita, ConfigKit, and PromptKit before
|
|
||||||
replacing Audita's production configuration or LLM runtime. This unit covers the
|
|
||||||
shared design and representative prototype supporting the
|
|
||||||
[coordinated adoption roadmap](library-adoption.md). The
|
|
||||||
[ConfigKit roadmap](configkit.md) continues to own the broader configuration scope.
|
|
||||||
|
|
||||||
The deliverable is a concrete target contract supported by an offline executable
|
|
||||||
prototype, sufficient to implement the ConfigKit migration without subsequently
|
|
||||||
redesigning its LLM-facing settings. A successful prototype is not completion of
|
|
||||||
either production migration.
|
|
||||||
|
|
||||||
## Established Direction
|
|
||||||
|
|
||||||
- This is an intentional breaking migration with a migration guide, not legacy
|
|
||||||
configuration support, aliases, or a permanent compatibility mode.
|
|
||||||
- ConfigKit owns shared configuration mechanics; PromptKit owns shared prompt and
|
|
||||||
generation mechanics; Audita retains domain meaning and transcript correction.
|
|
||||||
- Structural repair is enabled from the outset for proposal and LLM-validator calls.
|
|
||||||
Positive default repair budgets must be selected, not silently left at zero.
|
|
||||||
- Transient retries and structural repair are separate. Explicit zero disables
|
|
||||||
transient retries; Audita must not add a second automatic retry loop.
|
|
||||||
- One finite concurrency limit per backend protects local capacity and bounds paid
|
|
||||||
API activity. Calls from both roles share that limit. Fine-grained role budgets
|
|
||||||
and an additional global concurrency control are unnecessary.
|
|
||||||
- Keep the sequence: boundary/prototype, ConfigKit migration, PromptKit migration,
|
|
||||||
then combined major release. No intermediate public release is planned.
|
|
||||||
|
|
||||||
## Scope and Deliverables
|
|
||||||
|
|
||||||
| Deliverable | Required result |
|
|
||||||
| --- | --- |
|
|
||||||
| Configuration inventory and migration map | Every existing setting and invocation option has a proposed destination, replacement, or removal, with rationale. |
|
|
||||||
| Boundary specification | One authoritative source for each setting, explicit precedence/defaults, credential flow, runtime conversion, and failure handling. |
|
|
||||||
| Representative executable prototype | New-style configuration drives one proposal and one assessment through released libraries and a controlled provider. |
|
|
||||||
| Focused behavioral evidence | Retry, repair, capacity, credential, observation, and domain-boundary scenarios below are demonstrated. |
|
|
||||||
| Decision and integration handoff | Concrete choices, measured limitations, affected components, removal plan, and next implementation tasks are recorded. |
|
|
||||||
|
|
||||||
Do not switch the production CLI to the prototype, migrate every module/prompt,
|
|
||||||
redesign transcript/glossary formats, change correction algorithms, modify retention
|
|
||||||
or timestamp behavior, or publish a release in this unit. Do not reproduce the
|
|
||||||
libraries' full test suites. Upstream changes require a separately scoped task if
|
|
||||||
an actual gap is found.
|
|
||||||
|
|
||||||
## Work Package 1: Inventory and Decide Ownership
|
|
||||||
|
|
||||||
Read the current [configuration](../config.md), [CLI](../cli.md),
|
|
||||||
[LLM runtime](../internal/llm-runtime.md), [prompts](../internal/prompts.md),
|
|
||||||
[validators](../internal/validators.md), and [output contracts](../integrations/output-files.md).
|
|
||||||
Use graph-first discovery to verify the relevant code and tests, especially
|
|
||||||
configuration application, prompt construction, structured responses, scheduling,
|
|
||||||
and report assembly.
|
|
||||||
|
|
||||||
Build a migration table grouped by application/domain settings, invocation state,
|
|
||||||
backend settings, execution profiles, output contracts, and inspection. Include
|
|
||||||
current environment settings and aliases without treating their existence as a
|
|
||||||
requirement to preserve them. Identify report fields whose meanings change when
|
|
||||||
one logical generation contains multiple provider attempts.
|
|
||||||
|
|
||||||
Resolve these design decisions explicitly:
|
|
||||||
|
|
||||||
| Decision | Proposed direction to prove |
|
|
||||||
| --- | --- |
|
|
||||||
| File selection and version | Audita selects the main configuration and validates a new schema version; specify discovery and the treatment of old versions. |
|
|
||||||
| Application versus invocation state | ConfigKit loads persistent settings; Audita parses invocation-only input/output paths and translates explicitly supplied setting flags into supported overrides. |
|
|
||||||
| Backend identity | One registration represents the service capacity shared by proposal and validation calls. Avoid separate identities for the same service merely because roles differ. |
|
|
||||||
| Endpoint and capacity ownership | Define them once per backend, with finite limits and an explicit policy for waiting capacity/exhaustion. Do not expose endpoint-only unlimited execution accidentally. |
|
|
||||||
| Execution profiles | Prefer complete application-owned profiles using PromptKit conventions. Decide their source and selection without duplicating model/settings fields in application configuration. Catalog selection remains a deliberate option. |
|
|
||||||
| Role selection and inheritance | Specify how proposal and validation choose profiles, including omission and any inheritance. Do not inherit old behavior solely for compatibility. |
|
|
||||||
| Credentials | Prefer one ConfigKit-owned secret-resolution path passed through PromptKit's supported request credential field. Define backend/profile credential association and avoid competing environment lookups. Validate this against profile requirements. |
|
|
||||||
| Retry, repair, and deadlines | Assign one owner and explicit defaults for each control. Use positive structural-repair defaults and document combined limits and total-operation scope. |
|
|
||||||
| Optional library features | The original prototype omitted imports, selected configuration layers, and secret-directory bootstrap. Selected overlays are now planned for testing; see the [overlay plan](configkit.md#selected-overlays-for-testing). Imports and secret-directory bootstrap remain out. |
|
|
||||||
| Inspection | Define effective configuration and selected-profile inspection without implying that one library's snapshot contains the other's complete state. |
|
|
||||||
|
|
||||||
Choose concrete field names, example values, defaults, units, and omission/null/zero
|
|
||||||
semantics for the prototype. In particular, decide what validation and inspection
|
|
||||||
do when configured credentials are unavailable. Classify choices as proposed,
|
|
||||||
selected, or unresolved; do not present candidates as implemented contracts.
|
|
||||||
|
|
||||||
## Work Package 2: Specify Execution and Reporting Semantics
|
|
||||||
|
|
||||||
Document the flow from selected configuration through backend/profile assembly,
|
|
||||||
prompt inputs, generation, structural validation/repair, typed decoding, and Audita
|
|
||||||
domain validation. Identify which values cross the boundary and which component
|
|
||||||
owns each conversion. Keep library types out of unrelated domain code where a
|
|
||||||
small existing domain value is sufficient.
|
|
||||||
|
|
||||||
Use the current correction-set and validator-decision schemas as representative
|
|
||||||
inputs. Reuse domain checks where practical. Select one real proposal/validator
|
|
||||||
pair; account for prompt helpers, template variables, shared fragments, and schema
|
|
||||||
references rather than testing only an unrelated hello-world prompt.
|
|
||||||
|
|
||||||
Record the following distinctions:
|
|
||||||
|
|
||||||
- Repair seeks output conforming to its structural contract. Success does not
|
|
||||||
establish semantic correctness or authorize applying a correction.
|
|
||||||
- A schema-valid decision set still requires Audita's cardinality and proposal
|
|
||||||
identity checks. Semantic rejection is not automatically another repair request.
|
|
||||||
- Exhausted structural repair yields failed content validation; provider failures,
|
|
||||||
retry exhaustion, cancellation, and capacity rejection have separate outcomes.
|
|
||||||
- Specify each outcome's intended effect on an Audita section/batch and overall run,
|
|
||||||
including warnings, rejected work, partial domain results, and terminal failures.
|
|
||||||
Preserve domain intent without preserving obsolete transport behavior by accident.
|
|
||||||
- For transient retry budget T and structural repair budget R, one logical call can
|
|
||||||
dispatch up to `(T + 1) * (R + 1)` provider attempts. For example, T=2 and R=2
|
|
||||||
permits nine attempts; these illustrative values do not select production defaults.
|
|
||||||
- Choose the total-operation deadline scope for a logical proposal or assessment;
|
|
||||||
distinguish it from per-attempt timeouts and whole-process cancellation. Provider
|
|
||||||
retry timing may exceed local backoff, so deadline behavior must be explicit.
|
|
||||||
|
|
||||||
Design the Audita observation record and aggregation rules. Include logical-call
|
|
||||||
and attempt identity, role/backend correlation, queue/generation/retry timing,
|
|
||||||
repair and retry counts, outcomes, and available usage. Preserve unknown/incomplete
|
|
||||||
usage rather than inventing zeros or complete costs. Do not add terminal totals to
|
|
||||||
attempt totals. Collect events safely across concurrent calls, without blocking
|
|
||||||
callbacks on progress by the same run or persisting raw library events as a stable
|
|
||||||
Audita schema. Transcript-bearing artifacts remain separately governed by Audita.
|
|
||||||
|
|
||||||
## Work Package 3: Build the Bounded Offline Prototype
|
|
||||||
|
|
||||||
Use ConfigKit v0.3.1 and PromptKit v0.11.0 as reviewed starting points; verify their
|
|
||||||
published availability and APIs before pinning the prototype. Recheck the combined
|
|
||||||
Go minimum (the reviewed PromptKit release requires Go 1.25.5). Do not depend on
|
|
||||||
sibling checkouts, an active Go workspace, or committed local module replacements.
|
|
||||||
|
|
||||||
A proposed home is `examples/library-adoption/` as an explicitly experimental,
|
|
||||||
standalone Go module, isolating prototype dependencies from the production binary.
|
|
||||||
Confirm this placement during execution. Its README must give exact offline run
|
|
||||||
and test commands after dependency acquisition; root `go test ./...` must not be
|
|
||||||
claimed to cover a nested module. Keep complete proposed configuration and prompt
|
|
||||||
fixtures with it and clearly label them as prototype-only.
|
|
||||||
|
|
||||||
The prototype should:
|
|
||||||
|
|
||||||
1. Load representative application settings with ConfigKit, using synthetic secret
|
|
||||||
values supplied by the test environment and an application-owned version rule.
|
|
||||||
2. Assemble shared backend registrations and proposal/validation profile selections.
|
|
||||||
3. Supply representative transcript/proposal inputs to PromptKit definitions and
|
|
||||||
actual response schemas using supported sources.
|
|
||||||
4. Use the built-in HTTP generation path with a controlled local endpoint or
|
|
||||||
transport. An injected LLM client alone cannot prove built-in HTTP retry behavior.
|
|
||||||
5. Run proposal generation, decode and check candidates, run assessment, and perform
|
|
||||||
the selected Audita decision checks without modifying real transcript files.
|
|
||||||
6. Collect selected safe observations and show a compact result understandable to a
|
|
||||||
reviewer, including repaired output and final domain acceptance/rejection.
|
|
||||||
|
|
||||||
No paid calls, real credentials, host environment assumptions, persistent caches,
|
|
||||||
or general-purpose test framework are needed. Make concurrency and cancellation
|
|
||||||
checks deterministic through controlled synchronization. Keep delays bounded and
|
|
||||||
use library-supported controls; do not patch library internals to make tests pass.
|
|
||||||
|
|
||||||
## Focused Verification Matrix
|
|
||||||
|
|
||||||
| Scenario | Evidence required |
|
|
||||||
| --- | --- |
|
|
||||||
| Successful assembled path | ConfigKit-loaded selections drive both real prompt roles through PromptKit and Audita checks. |
|
|
||||||
| Presence, overrides, and inheritance | Representative omission, explicit values, and role selection follow the proposed contract; unknown fields and unsupported versions fail clearly. |
|
|
||||||
| Credentials | Required missing/empty references fail as intended; controlled calls receive the chosen synthetic credential; inspection and observations do not expose it. |
|
|
||||||
| Structural repair succeeds | Invalid proposal output and invalid assessment output are repaired in their respective workflows; positive budgets are actually exercised. |
|
|
||||||
| Repair exhausted | Repeated invalid output produces the selected Audita outcome and accurate accounting, without an unbounded outer retry. |
|
|
||||||
| Domain rejection | Structurally valid but unacceptable correction or decision is rejected by Audita, demonstrating the semantic boundary. |
|
|
||||||
| Transient retry and explicit zero | An eligible temporary status recovers; zero makes one attempt. Exhaustion retains a useful classified cause. |
|
|
||||||
| Combined repair and retry | A repaired generation encountering a transient failure respects both budgets and reports distinct counts. |
|
|
||||||
| Shared capacity | Concurrent calls from both roles, including retry/repair activity, stay within one selected backend cap; test a representative local cap of four and a finite API cap. |
|
|
||||||
| Waiting, cancellation, and deadline | Cancellation/deadline stops pending work and retry waits; capacity exhaustion has the intended application outcome and later calls can still proceed. |
|
|
||||||
| Failed-run observations | Completed work remains observable after later failure; unknown usage, known zero, and aggregate counts are mapped correctly. |
|
|
||||||
|
|
||||||
Tests should protect Audita's integration decisions and important assembled behavior.
|
|
||||||
Rely on PromptKit's tests for exhaustive HTTP-status, backoff, and scheduler details;
|
|
||||||
rely on ConfigKit's tests for its parser and presence machinery. Use race testing for
|
|
||||||
new concurrent observation collection. Keep consumer checks offline and credential-free.
|
|
||||||
|
|
||||||
## Completion Criteria and Handoff
|
|
||||||
|
|
||||||
This unit is complete only when:
|
|
||||||
|
|
||||||
- The settings/ownership table and migration inventory are concrete enough to start
|
|
||||||
the ConfigKit migration, including file/profile sources, CLI scope, credentials,
|
|
||||||
defaults, finite backend limits, positive repair budgets, and deadline policy.
|
|
||||||
- The offline prototype and focused matrix demonstrate the selected boundary with
|
|
||||||
both libraries together, including both LLM roles and Audita domain checks.
|
|
||||||
- Exact verification commands, results, limitations, and unresolved upstream gaps
|
|
||||||
are recorded. Missing evidence is not reported as successful integration.
|
|
||||||
- No adoption-critical decision remains hidden in prototype code. If a gap prevents
|
|
||||||
settling the boundary, record it and revise the plan before declaring readiness.
|
|
||||||
- The existing Audita production behavior remains intact in this unit. Temporary
|
|
||||||
code is clearly isolated and has a removal/integration destination during the
|
|
||||||
production migrations; no second supported runtime is introduced.
|
|
||||||
- A component-level follow-up identifies code to replace, domain code to retain,
|
|
||||||
tests to update, documentation/migration-guide work, and direct integration of
|
|
||||||
the final configuration and PromptKit runtime.
|
|
||||||
- Licensing/distribution follow-up for the selected dependency graph is identified
|
|
||||||
for the later release stage; this prototype does not claim release compliance.
|
|
||||||
- Roadmap links and examples resolve, appropriate checks pass, and commits contain
|
|
||||||
only this unit's work. Significant settled architectural choices may be recorded
|
|
||||||
in an ADR under the [documentation policy](../policy/documentation.md), while
|
|
||||||
implementation status stays in the roadmaps.
|
|
||||||
|
|
||||||
Retain the overall adoption order unless evidence or the other planned large changes
|
|
||||||
justifies revising it. Completing this unit enables detailed production migration;
|
|
||||||
it does not authorize an intermediate release or require unrelated runtime fixes.
|
|
||||||
@@ -1,252 +0,0 @@
|
|||||||
# Standard Policy Alignment
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-28
|
|
||||||
|
|
||||||
**Status:** Implemented. This roadmap replaces its earlier licensing-only scope
|
|
||||||
and records the accepted end state. [implementation.md](implementation.md)
|
|
||||||
records the ordered work and verification; the adopted policies are in
|
|
||||||
`docs/policy/`.
|
|
||||||
|
|
||||||
## Purpose and Review Revision
|
|
||||||
|
|
||||||
Align Audita with [Standard Repository Policies](https://gitea.maximumdirect.net/eric/policy),
|
|
||||||
using identical policy language except where a substantive, locally significant
|
|
||||||
requirement or decision justifies a variation. Adopt upstream wording verbatim
|
|
||||||
for non-substantive differences. Perform the required template customizations
|
|
||||||
without inventing application contracts, restrictions, or guarantees.
|
|
||||||
|
|
||||||
The accepted review compared Audita's current documents directly against upstream
|
|
||||||
commit **`d84a38ebab26f048cafbe59a2246700c35d29843`**, beginning with the
|
|
||||||
[adoption and update guide at that revision](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/adoption.md).
|
|
||||||
Use that revision for this adoption. Historical adoption baselines need not be
|
|
||||||
reconstructed or assessed. A later upstream revision requires deliberate review,
|
|
||||||
not an automatic substitution during implementation.
|
|
||||||
|
|
||||||
The public upstream repository, discovery links, and copied-policy attribution
|
|
||||||
are part of the accepted direction. This supersedes the earlier instruction to
|
|
||||||
omit references to the policy repository.
|
|
||||||
|
|
||||||
## Current Baseline
|
|
||||||
|
|
||||||
- Audita has five adopted policies: agents 1.0, architecture 1.0, documentation
|
|
||||||
1.1, testing 1.0, and release 1.0. None has per-document `Upstream Source`
|
|
||||||
metadata. `meta.md` and `oss.md` are absent.
|
|
||||||
- [AGENTS.md](../../AGENTS.md) routes readers to the contributor guide and agent
|
|
||||||
policy. [docs/development.md](../development.md) has component-specific reading
|
|
||||||
guidance, but lacks the upstream selective Policy Reading Guide and currently
|
|
||||||
directs all tasks to the architecture policy.
|
|
||||||
- [COPYRIGHT](../../COPYRIGHT) and [LICENSE](../../LICENSE) already establish
|
|
||||||
Audita's GPL-3.0-or-later application grant and full GPLv3 text. The grant does
|
|
||||||
not yet identify the scope of separately licensed copied policy documents.
|
|
||||||
- There is no canonical `docs/oss/attribution.md` or local dependency license-text
|
|
||||||
collection. Relevant component notices still require an evidence-based inventory.
|
|
||||||
- The [README](../../README.md) lacks the prescribed final licensing statements.
|
|
||||||
The [release procedure](../release.md) lacks licensing-footer instructions, and
|
|
||||||
the [release workflow](../../.woodpecker/release.yml) does not explicitly reuse
|
|
||||||
checked-in release notes as hosted release bodies.
|
|
||||||
- Audita already releases through one publication command with an atomic source
|
|
||||||
and tag push. Linux amd64 and arm64 are required; macOS arm64 is best-effort
|
|
||||||
development support. Successful publication completes the release without
|
|
||||||
waiting for CI or artifact delivery.
|
|
||||||
|
|
||||||
## Targeted Policy Set and Justified Customizations
|
|
||||||
|
|
||||||
All seven policy files belong under `docs/policy/`. The versions below are those
|
|
||||||
at the selected upstream revision. No existing policy is wholly unchanged;
|
|
||||||
existing architectural decisions and platform commitments need no substantive
|
|
||||||
revision merely to align the text.
|
|
||||||
|
|
||||||
| Policy | Target version | Required alignment and justified local treatment |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| `agents.md` | 1.3 | Adopt upstream substantive language verbatim. Retain Audita's graph-first Codebase Discovery section because it is an explicitly selected repository workflow. |
|
|
||||||
| `architecture.md` | 1.3 | Adopt general ownership, boundary, and constraint guidance, then perform the template's required application-specific customization. Preserve Audita's ordered pipeline, ConfigKit/PromptKit responsibilities, deterministic processing, recovery, concurrency, and secret-handling decisions. These describe real project requirements rather than stylistic preferences. |
|
|
||||||
| `documentation.md` | 1.11 | Adopt upstream prose and applicable ownership rows. Retain concrete output-contract ownership, contributor change recipes, and the decision to place operating/recovery guidance in operations instead of restoring troubleshooting documentation. Omit HTTP API and public Go API ownership rows because Audita has neither responsibility. |
|
|
||||||
| `testing.md` | 1.3 | Adopt substantive text verbatim. Its distinction between established contracts and invented test requirements needs no local exception. |
|
|
||||||
| `release.md` | 1.4 | Adopt upstream substantive language. Preserve Linux amd64/arm64 requirements, best-effort macOS arm64, unsupported Windows/macOS amd64, and Audita's explicit decision against human release gates. These are selected support and workflow commitments. |
|
|
||||||
| `meta.md` | 1.0 | Adopt provenance, local authority, and public discovery guidance with accurate metadata and local attribution navigation. No additional substantive exception is needed. |
|
|
||||||
| `oss.md` | 1.8 | Adopt the House Format and maintenance rules, with required project-specific facts and reference links. Preserve Audita's GPL-3.0-or-later application grant, Gitea host, and existing distribution channels. |
|
|
||||||
|
|
||||||
Remove each `Base Policy Template` section after carrying out its instructions.
|
|
||||||
Keep local customizations identifiable and explain their project-specific reason
|
|
||||||
in the appropriate policy; do not maintain alternate paraphrases of shared rules.
|
|
||||||
Architecture customization does not authorize a package redesign or additional
|
|
||||||
threat-model guarantees. Supporting documents are warranted by existing
|
|
||||||
responsibilities and substantive content, not by unused template examples.
|
|
||||||
|
|
||||||
Successful-tag-push completion, asynchronous artifacts, minimal automated gates,
|
|
||||||
and proportionate validation are now shared upstream rules. Replace Audita's
|
|
||||||
parallel explanations with the upstream language rather than describing these
|
|
||||||
as continuing exceptions. Keep concrete commands, remotes, build targets, and
|
|
||||||
publication mechanics in the release procedure with appropriate policy links.
|
|
||||||
|
|
||||||
## Provenance and Selective Reading
|
|
||||||
|
|
||||||
For every adopted policy, retain the selected upstream `Base Policy Version`,
|
|
||||||
set `Last Updated` to its actual local adoption date, and introduce `Upstream
|
|
||||||
Source` linking to that exact file at the selected full commit. The URL pattern is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/<filename>.md
|
|
||||||
```
|
|
||||||
|
|
||||||
Substitute each of `agents`, `architecture`, `documentation`, `testing`, `release`,
|
|
||||||
`meta`, and `oss`; a repository-root or shared generic source link is insufficient.
|
|
||||||
These links describe the source actually incorporated, not a reconstructed claim
|
|
||||||
about older local documents. Git and per-document metadata are sufficient; no
|
|
||||||
separate synchronization registry or recurring check is required.
|
|
||||||
|
|
||||||
In `meta.md`, retain these public discovery destinations:
|
|
||||||
|
|
||||||
- [Standard Repository Policies](https://gitea.maximumdirect.net/eric/policy).
|
|
||||||
- [Current adoption and update guide](https://gitea.maximumdirect.net/eric/policy/src/branch/main/docs/adoption.md).
|
|
||||||
|
|
||||||
Discovery links follow the current upstream project and guide. Adopted-source,
|
|
||||||
notice, and example links remain commit-pinned. Preserve valid pinned example
|
|
||||||
references supplied by the selected policy, or pin them to this review revision
|
|
||||||
after checking the referenced assets; do not turn them into moving-branch links.
|
|
||||||
|
|
||||||
Adapt upstream `agents/AGENTS.md` into Audita's root orientation, retaining its
|
|
||||||
existing applicable instructions. Add the adoption guide's Policy Reading Guide
|
|
||||||
to `docs/development.md` and retain the component-specific task guide. Route
|
|
||||||
policy adoption and updates to `meta.md`, repository recommendations and changes
|
|
||||||
to agents, application-code or structure work to architecture, validation choices
|
|
||||||
to testing, documentation work to documentation, dependency/licensing work to
|
|
||||||
OSS, and release/versioning work to release policy. Read applicable policies
|
|
||||||
before their decisions, reuse unchanged context, and expand reading with scope.
|
|
||||||
Do not require reading the entire suite for every task.
|
|
||||||
|
|
||||||
## Copied Policy Licensing and Local Material
|
|
||||||
|
|
||||||
Follow the selected revision's
|
|
||||||
[canonical policy attribution entry](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/policy-attribution.md)
|
|
||||||
and [project grant](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT).
|
|
||||||
Insert the entry beneath the title in `docs/oss/attribution.md`, substitute the
|
|
||||||
full review commit in all three URLs, and accurately identify copied/adapted
|
|
||||||
material, including orientation or reading-guide text where copied. Preserve
|
|
||||||
supplied creator, copyright, license, disclaimer notices, and modification
|
|
||||||
indications. Keep the material statement concise.
|
|
||||||
|
|
||||||
Copy `CC-BY-SA-4.0.txt` into `docs/oss/licenses/`. Add the adoption guide's scoped
|
|
||||||
notice to `COPYRIGHT`, identifying the adapted policies as CC-BY-SA-4.0 and
|
|
||||||
routing readers to attribution for the affected material. Account accurately for
|
|
||||||
other copied guidance in that scope. Audita's application code remains
|
|
||||||
GPL-3.0-or-later; unrelated documentation is not automatically relicensed.
|
|
||||||
The root GPLv3 `LICENSE` needs no change.
|
|
||||||
|
|
||||||
| Material | Destination or treatment |
|
|
||||||
| --- | --- |
|
|
||||||
| Applicable policies, including `meta.md` and `oss.md` | Copy and customize under `docs/policy/`, retaining provenance. |
|
|
||||||
| Upstream `agents/AGENTS.md` | Adapt into root `AGENTS.md`. |
|
|
||||||
| Selective reading guidance | Integrate into Audita's own `docs/development.md`. |
|
|
||||||
| Upstream README, root AGENTS.md, developer guide, and adoption guide | Keep upstream and consult through links; do not copy as local guides. |
|
|
||||||
| Licensing example collection | Keep upstream with commit-pinned references; use it to create actual notices. Original example formatting is CC0 and alone needs no policy attribution entry. |
|
|
||||||
| Applicable standard and companion license texts | Copy complete, unmodified texts locally; no blanket collection. |
|
|
||||||
|
|
||||||
## Component Attribution
|
|
||||||
|
|
||||||
Complete the previously agreed inventory of relevant incorporated or distributed
|
|
||||||
components: direct and transitive dependencies, copied code/documentation/assets,
|
|
||||||
and applicable Go runtime notices. Determine inclusion from actual material and
|
|
||||||
use, not simply presence in `go.mod`; tooling or consultation alone does not
|
|
||||||
require an entry. Inspect selected sources directly rather than copying a
|
|
||||||
dependency's attribution document wholesale.
|
|
||||||
|
|
||||||
Use the exact House Format in OSS policy 1.8: the prescribed title followed by
|
|
||||||
component entries, factual identification, applicable supplied copyrights and
|
|
||||||
SPDX expressions, standard license/notice labels, and `[Source](COMMIT_URL).`
|
|
||||||
links. Omit unavailable optional fields instead of adding explanatory filler.
|
|
||||||
Preserve required upstream material, license alternatives, and exceptions.
|
|
||||||
Keep research narratives, compliance conclusions, and exception reasoning out
|
|
||||||
of attribution; place reasoning in the OSS policy or an appropriate decision
|
|
||||||
record. This replaces the older roadmap's permission to put exception reasoning
|
|
||||||
in a component entry.
|
|
||||||
|
|
||||||
Upstream component and notice links identify exact imported commits. Local
|
|
||||||
standard texts include directly reachable companions, such as GPLv3 for LGPLv3.
|
|
||||||
Apply the adopted house interpretation of notice links without blanket extra
|
|
||||||
local-copy requirements or repeated approval. Concrete obligations beyond notice
|
|
||||||
reproduction receive their own applicable treatment; the format alone does not
|
|
||||||
certify compatibility or resolve every distribution obligation.
|
|
||||||
|
|
||||||
The reviewed dependency revisions remain useful starting evidence, subject to
|
|
||||||
checking the dependencies selected when implementation occurs:
|
|
||||||
|
|
||||||
| Component | Reviewed version | Source commit |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| ConfigKit | v0.4.1 | `0adb94f59e44827908eff401d4381a398fe17107` |
|
|
||||||
| PromptKit | v0.11.1 | `15e40679546f2e21b6f51cc84b6f0a087a8486db` |
|
|
||||||
| OpenRouter catalog | v2.20260926.0 | `4f731d217e0bc39151c010f50fb5912706e7b6b3` |
|
|
||||||
| Rakestrawhome catalog | v2.20260926.0 | `48918a3ad39fd99843cda94a86133b1b3881d596` |
|
|
||||||
|
|
||||||
These library patches changed licensing/documentation rather than library code
|
|
||||||
or catalog profiles. ConfigKit and PromptKit identify LGPL-3.0-or-later; the
|
|
||||||
selected catalogs identify MIT. Other relevant dependencies still need inventory.
|
|
||||||
|
|
||||||
## Discoverability and Release Integration
|
|
||||||
|
|
||||||
End README with the project-license and attribution statements prescribed by the
|
|
||||||
OSS policy, using Audita's actual grant. `COPYRIGHT` owns the separately licensed
|
|
||||||
policy scope; README links to that grant and the attribution document.
|
|
||||||
|
|
||||||
Update `docs/release.md` to specify the final `COPYRIGHT | LICENSE | ATTRIBUTION`
|
|
||||||
links in every new release note and hosted download page. Use absolute Gitea URLs
|
|
||||||
at the release's immutable tag or source commit. Reuse the checked-in note as the
|
|
||||||
hosted body where supported; verify the publication plugin's supported mechanism
|
|
||||||
and make the necessary focused `.woodpecker/release.yml` change. Avoid separately
|
|
||||||
maintained release prose or notice lists.
|
|
||||||
|
|
||||||
Preserve the current atomic source/tag publication approach, supported targets,
|
|
||||||
and completion point. Align procedure wording with upstream's preparation,
|
|
||||||
ambiguous-push recovery, and asynchronous retry rules. Handle completed-roadmap
|
|
||||||
retirement during preparation, preserving planned/deferred work and repairing
|
|
||||||
links; this roadmap update does not authorize deleting other roadmaps now.
|
|
||||||
|
|
||||||
Check intended future-tag paths against the candidate source without requiring
|
|
||||||
the tag to resolve before publication. Apply the formatting convention
|
|
||||||
prospectively; leave published tags, notes, artifacts, and checksums intact.
|
|
||||||
Concrete historical omissions, if discovered, receive deliberate treatment under
|
|
||||||
their applicable terms and release correction rules.
|
|
||||||
|
|
||||||
## Work Required and Acceptance Criteria
|
|
||||||
|
|
||||||
Implementation covers the seven policy documents, `AGENTS.md`,
|
|
||||||
`docs/development.md`, `COPYRIGHT`, `docs/oss/attribution.md`, applicable files in
|
|
||||||
`docs/oss/licenses/`, README, the release procedure, and focused hosted-body
|
|
||||||
publication wiring. Update this roadmap's implementation status when appropriate.
|
|
||||||
The [implementation plan](implementation.md) owns the editing sequence and
|
|
||||||
verification steps.
|
|
||||||
|
|
||||||
The work is complete when:
|
|
||||||
|
|
||||||
- Shared policy language matches the selected upstream revision after required
|
|
||||||
template removal and customization; retained variations have substantive
|
|
||||||
Audita-specific reasons as identified above.
|
|
||||||
- All seven policies have accurate versions, adoption dates, and file-specific
|
|
||||||
commit-pinned source links. Public discovery links and selective orientation
|
|
||||||
are in place, with no leftover template-only instructions or broken links.
|
|
||||||
- Copied policy/guidance scope, modification status, credit, CC-BY-SA text, and
|
|
||||||
root grant notice follow the documented upstream arrangement while preserving
|
|
||||||
Audita's application license.
|
|
||||||
- Relevant components have verified House Format entries, exact source/notice
|
|
||||||
commits, and complete applicable license texts, without editorial commentary.
|
|
||||||
- README and future release documentation provide the required navigation, and
|
|
||||||
hosted release content comes from the checked-in note through a verified
|
|
||||||
publication mechanism.
|
|
||||||
- Audita's application contracts, architectural decisions, platform commitments,
|
|
||||||
release completion semantics, and published history are preserved.
|
|
||||||
- Affected text, links, license copies, metadata, and publication configuration
|
|
||||||
pass proportionate review and validation. No release is needed solely to test
|
|
||||||
adoption; behavioral workflow changes receive focused validation if required.
|
|
||||||
|
|
||||||
## Scope Boundaries and Open Questions
|
|
||||||
|
|
||||||
No policy decisions remain open for the agreed scope. Component-specific notices
|
|
||||||
and the publication plugin's supported release-body mechanism require factual
|
|
||||||
verification during implementation. Raise a decision only for a concrete
|
|
||||||
unresolved obligation or material tradeoff uncovered by that work.
|
|
||||||
|
|
||||||
This feature does not require application refactoring, dependency upgrades,
|
|
||||||
mandatory scanners, a full SBOM, recurring legal approvals, synchronization
|
|
||||||
checks, additional human release gates, publication-time network checks, or CI
|
|
||||||
waiting. The documentation convention alone does not require downloadable
|
|
||||||
license assets, source bundles, or a different binary package format.
|
|
||||||
276
docs/roadmap/promptkit.md
Normal file
276
docs/roadmap/promptkit.md
Normal file
@@ -0,0 +1,276 @@
|
|||||||
|
# PromptKit v0.12.0 Adoption Roadmap
|
||||||
|
|
||||||
|
**Last updated:** 2026-10-04
|
||||||
|
|
||||||
|
**Status:** Complete. The implementation plan records the ordered work and
|
||||||
|
verification details.
|
||||||
|
|
||||||
|
## Purpose and Baseline
|
||||||
|
|
||||||
|
Adopt PromptKit v0.12.0 now that the independent
|
||||||
|
[ConfigKit v0.5.0 upgrade](configkit.md) is complete. Align Audita with the
|
||||||
|
libraries' native configuration model while retaining Audita's domain rules,
|
||||||
|
application-owned backend settings, and resilient processing workflow.
|
||||||
|
|
||||||
|
At reviewed Audita revision `8cba751835ba9faad340e933a3cfbb59c03924fd`,
|
||||||
|
[go.mod](../../go.mod) selects ConfigKit v0.5.0 and PromptKit v0.11.2, with
|
||||||
|
OpenRouter and Rakestrawhome `/v2` catalogs at `v2.20260926.0`. Go 1.25.5
|
||||||
|
remains the application minimum. The implemented contracts belong to
|
||||||
|
[configuration](../config.md), the
|
||||||
|
[LLM integration](../integrations/openai-compatible-llm.md), and the
|
||||||
|
[LLM runtime](../internal/llm-runtime.md).
|
||||||
|
|
||||||
|
`internal/framework/llm.NewRuntime` currently assembles one engine using the
|
||||||
|
old prompt/profile filesystem options and programmatic backend registration.
|
||||||
|
Audita embeds seven prompt definitions and separate response schemas. Operators
|
||||||
|
select proposal and validation profiles from `profiles_dir`; the effective
|
||||||
|
backend must be registered in application configuration, its endpoint must
|
||||||
|
match, and its credential comes from Audita's ConfigKit-resolved settings.
|
||||||
|
Successful profile reads are recorded to protect those sources from output
|
||||||
|
collisions. Both roles share the same engine's backend admission controls.
|
||||||
|
|
||||||
|
The reviewed upstream release is
|
||||||
|
[PromptKit v0.12.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/releases/v0.12.0.md),
|
||||||
|
commit `4b8169da286917fde7875425bb32b799b12c6af3`. It depends on ConfigKit
|
||||||
|
v0.5.0 and both maintained `/v3` catalog modules at `v3.20260930.0`. This is an
|
||||||
|
intentional incompatible pre-1.0 minor release: it replaces construction and
|
||||||
|
configuration APIs, rather than simply updating provider data. PromptKit's
|
||||||
|
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
|
||||||
|
and [consumer migration guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
|
||||||
|
own the upstream details.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
- Audita imports released PromptKit v0.12.0 and its `/v3` catalogs, retaining
|
||||||
|
ConfigKit v0.5.0 and using no local dependency replacement.
|
||||||
|
- The runtime uses `Config.Sources` and native ConfigKit source groups.
|
||||||
|
PromptKit owns composition, inheritance, presence, final-definition
|
||||||
|
validation, prompt capture, and execution. Audita does not translate old
|
||||||
|
profiles or implement a second YAML loader or inheritance resolver.
|
||||||
|
- Embedded prompts and maintained operator profiles use keyed `prompts` and
|
||||||
|
`profiles` mappings. Existing prompt IDs, versions, role selections,
|
||||||
|
instructions, and response-schema associations remain the adoption baseline.
|
||||||
|
The application configuration schema, overlay selection, CLI interface, and
|
||||||
|
main-file-relative `profiles_dir` resolution remain in place.
|
||||||
|
- Native profile `extends`, field-presence semantics, and source-relative
|
||||||
|
resource rules apply. Documents can contain multiple definitions, and IDs
|
||||||
|
come from mapping keys rather than filenames. The existing `profiles_dir`
|
||||||
|
accepts native unified PromptKit documents containing profiles, backends,
|
||||||
|
and prompts, including intentional operator prompt overrides.
|
||||||
|
- Application backend endpoints, configured concurrency and queue capacity,
|
||||||
|
and credential ownership remain authoritative for selected roles. Preserve
|
||||||
|
the existing profile-selection and prepared-target checks; do not introduce
|
||||||
|
a second admission controller or implicit provider credential lookup.
|
||||||
|
- A runtime uses one captured configuration snapshot for both roles. Each CLI
|
||||||
|
invocation constructs a new engine; hot reload is not part of this feature.
|
||||||
|
Existing inspection remains offline and safe, and consumed host sources
|
||||||
|
remain protected from output collisions.
|
||||||
|
- PromptKit retains responsibility for transport retries, schema repair,
|
||||||
|
observation, and prepared execution. Preserve Audita's configured retry and
|
||||||
|
repair budgets, operation deadlines, cancellation, partial-success behavior,
|
||||||
|
error/recovery reporting, and debug selection.
|
||||||
|
|
||||||
|
## Configuration Assembly and Ownership
|
||||||
|
|
||||||
|
Replace removed `WithPromptFS`, `WithProfileFS`, and `WithBackend` usage with
|
||||||
|
the released source API. Discover embedded YAML through `FilesystemInputs` and
|
||||||
|
operator YAML through `DirectoryInputs`, with explicit caller tiers. Keep the
|
||||||
|
retained schema filesystem integration and HTTP client configuration.
|
||||||
|
Use the helpers' default-disabled imports; this adoption does not require new
|
||||||
|
application source-selection flags or operator underlay tiers.
|
||||||
|
|
||||||
|
PromptKit supplies catalog definitions first and its own defaults next. Caller
|
||||||
|
tiers then overlay in their supplied order, followed by an optional highest
|
||||||
|
priority main input. Files within one directory group compose strictly;
|
||||||
|
alphabetical filenames do not establish override precedence. Use separate
|
||||||
|
tiers for genuine priority differences rather than manufacturing file-order
|
||||||
|
semantics.
|
||||||
|
|
||||||
|
Represent application backend settings as a final programmatic source, using
|
||||||
|
`BackendSource` or ConfigKit mappings where explicit null is needed. Preserve
|
||||||
|
configured zero queue capacity as an explicit value. Suppress backend-derived
|
||||||
|
credential environment lookup explicitly where catalog layering would otherwise
|
||||||
|
introduce it; omission alone can retain a lower-tier value. Continue detecting
|
||||||
|
competing effective profile credential lookups rather than silently clearing
|
||||||
|
operator profile fields. A missing backend credential remains valid for an
|
||||||
|
unauthenticated local service unless the selected profile requires a key.
|
||||||
|
|
||||||
|
Catalog backend IDs are no longer reserved upstream. Audita should not invent a
|
||||||
|
new reservation rule. Catalog selection alone must not bypass application
|
||||||
|
registration, endpoint, capacity, or credential requirements. Catalog profile
|
||||||
|
contents may evolve; this upgrade does not promise fixed model choices or live
|
||||||
|
provider availability.
|
||||||
|
|
||||||
|
The accepted caller order is embedded Audita prompts, operator definitions,
|
||||||
|
then application backend settings. The existing `profiles_dir` accepts native
|
||||||
|
unified PromptKit documents containing profiles, backends, and prompts. Keep
|
||||||
|
the name and path setting for continuity, and document its broader accepted
|
||||||
|
content. This follows the upstream source API without a parallel parser or
|
||||||
|
top-level filtering. Profiles, provider defaults, and intentional prompt
|
||||||
|
overrides use the same native composition model.
|
||||||
|
|
||||||
|
Audita continues to ship its domain prompts, enforce its selected-backend
|
||||||
|
requirements, and supply its per-call schemas. Operator prompt overrides are
|
||||||
|
an accepted capability of this adoption; documentation, source accounting,
|
||||||
|
and focused behavioral coverage should reflect them.
|
||||||
|
|
||||||
|
## Work Required
|
||||||
|
|
||||||
|
### Runtime Construction and Source Accounting
|
||||||
|
|
||||||
|
Migrate `internal/framework/llm/runtime.go` to layered construction and retain
|
||||||
|
the role-bound adapter boundary. Review startup inspection and failure
|
||||||
|
classification because upstream now validates all final definitions, including
|
||||||
|
unused ones, during engine construction. Distinguish invalid operator inputs
|
||||||
|
from broken embedded assets under the existing
|
||||||
|
[CLI exit contract](../cli.md); do not treat every construction failure as an
|
||||||
|
operator error merely because validation moved earlier.
|
||||||
|
|
||||||
|
Upstream permits partial profiles and defers endpoint/model completeness until
|
||||||
|
request overrides are applied. Audita supplies no model override, so explicitly
|
||||||
|
validate that both selected role profiles have a nonblank model and a complete
|
||||||
|
effective endpoint during offline configuration validation and runtime setup.
|
||||||
|
Retain the application-backend, endpoint-match, and credential checks. Do not
|
||||||
|
rely on engine construction or profile inspection alone to establish selected
|
||||||
|
profile completeness. Unselected partial profiles remain permitted if they
|
||||||
|
satisfy upstream definition validation; they need not be runnable.
|
||||||
|
|
||||||
|
Adapt source accounting to the new discovery/capture boundary. The existing
|
||||||
|
recording filesystem wrapper may be replaceable with discovered inputs or
|
||||||
|
upstream source metadata. Protect the host sources actually consumed by the
|
||||||
|
runtime, including inherited and differently named definitions, without
|
||||||
|
guessing filenames from selected profile IDs. Account for consumed operator
|
||||||
|
message files as well. Keep this proportionate to the existing collision
|
||||||
|
contract; do not add filesystem concurrency or power-loss guarantees.
|
||||||
|
|
||||||
|
Keep the existing safe effective-profile summaries and application explanations.
|
||||||
|
Use upstream metadata where useful, but do not expose the full
|
||||||
|
`InspectConfiguration` snapshot as ordinary CLI or report output: prompt text
|
||||||
|
and arbitrary provider fields are outside those summaries' current scope.
|
||||||
|
|
||||||
|
### Prompt Assets, Profiles, and Presence Semantics
|
||||||
|
|
||||||
|
Move each of the seven files in `internal/prompts/assets/prompts` to the keyed
|
||||||
|
prompt/version format. Keep the shared instruction fragments and the complete
|
||||||
|
per-call output contracts owned by Audita. This migration does not authorize
|
||||||
|
another prompt-quality rewrite or changes to domain validators.
|
||||||
|
|
||||||
|
The adapter supplies `HardeningPolicy`, `ProposalContract`, and
|
||||||
|
`ValidationContract` as template variables. Embedded prompts retain their
|
||||||
|
references to the applicable variables. Operator message overrides control
|
||||||
|
whether those variables are rendered; supplying a variable does not inject its
|
||||||
|
text into a template that omits it. Preserve this native override behavior and
|
||||||
|
document the distinction. Per-call schemas and domain validation remain enforced
|
||||||
|
regardless of those references. Unconditional shared-instruction injection is
|
||||||
|
outside this adoption.
|
||||||
|
|
||||||
|
Convert the six maintained profiles in `examples/profiles` and test-generated
|
||||||
|
profile/prompt fixtures to the new format. Convert any old `base_profile`
|
||||||
|
usage to `extends`, and remove embedded identities. Migrate tests that construct
|
||||||
|
engines using removed options or old `Profile`/`Backend` values.
|
||||||
|
|
||||||
|
Reorganize the maintained profiles into separately selected local and service
|
||||||
|
directory roots. The current shared directory includes profiles referring to
|
||||||
|
backends absent from each example's application configuration; eager validation
|
||||||
|
would reject those unselected definitions after a format-only conversion.
|
||||||
|
Update `profiles_dir` in `examples/minimal-config.yml`,
|
||||||
|
`examples/production-config.yml`, and `examples/configkit-inheritance.yml` to
|
||||||
|
select the appropriate root. Keep the service testing profiles in the service
|
||||||
|
root so the testing overlay remains usable. Discovery is recursive: placing
|
||||||
|
both groups beneath a common selected root does not isolate them. Update example
|
||||||
|
links and validation recipes to match the final layout.
|
||||||
|
|
||||||
|
Review presence deliberately rather than mechanically preserving old serialized
|
||||||
|
values. In particular, explicit zero temperature, top-p, or max-tokens now means
|
||||||
|
an outbound zero; zero timeout disables PromptKit's generation deadline while
|
||||||
|
Audita's outer operation deadline still applies. Omission, blank values, and
|
||||||
|
null have field-specific meanings. Empty provider maps do not erase inherited
|
||||||
|
entries, and changing model/backend does not erase unrelated explicit settings.
|
||||||
|
|
||||||
|
Use the native meanings without a compatibility shim. Retaining the examples'
|
||||||
|
literal `temperature: 0` as an explicit provider setting is recommended; explain
|
||||||
|
that omitting the field requests an unspecified value, and null suppresses an
|
||||||
|
inherited control. Do not automatically turn every existing zero into omission
|
||||||
|
or every omission into a zero. Demonstrate representative profile inheritance
|
||||||
|
without requiring every maintained example to use it.
|
||||||
|
|
||||||
|
### Documentation, Migration, and Attribution
|
||||||
|
|
||||||
|
Update the profile section of `docs/config.md` and maintained examples when the
|
||||||
|
implementation lands. Explain the broader accepted content of `profiles_dir`,
|
||||||
|
operator prompt overrides, their precedence and optional use of shared instruction
|
||||||
|
variables, keyed identities, composition versus precedence, profile inheritance,
|
||||||
|
presence, eager final-definition validation, selected-root errors, and captured
|
||||||
|
configuration. Link to the pinned
|
||||||
|
upstream reference for generic field semantics rather than reproducing its
|
||||||
|
entire contract.
|
||||||
|
|
||||||
|
Include operator migration guidance for old profile documents and behaviorally
|
||||||
|
significant zeros/nulls. This is a deliberate format migration, not an automatic
|
||||||
|
upgrade of existing installation files. Preserve supported application overlays,
|
||||||
|
including the cheaper/faster testing-profile workflow.
|
||||||
|
|
||||||
|
Update the pinned consumer link in `docs/integrations/openai-compatible-llm.md`,
|
||||||
|
construction/source-accounting descriptions in `docs/internal/llm-runtime.md`,
|
||||||
|
and asset-format guidance in `docs/internal/prompts.md`. Update other affected
|
||||||
|
references only where implemented behavior changes.
|
||||||
|
|
||||||
|
Update `go.mod`, `go.sum`, and `docs/oss/attribution.md` for PromptKit and the two
|
||||||
|
catalog modules. Verify exact selected source revisions and upstream notices
|
||||||
|
under the [OSS policy](../policy/oss.md); retain the applicable LGPL/GPL and MIT
|
||||||
|
texts rather than duplicating unchanged license documents. Dependency adoption
|
||||||
|
does not change Audita's application license.
|
||||||
|
|
||||||
|
Plan this adoption for an Audita major release under the
|
||||||
|
[release policy](../policy/release.md#versioning-and-compatibility). The public
|
||||||
|
profile format changes incompatibly and no compatibility loader is planned;
|
||||||
|
PromptKit's own pre-1.0 minor version does not change Audita's classification.
|
||||||
|
The exact version and publication timing remain deferred to release preparation.
|
||||||
|
|
||||||
|
## Verification and Completion
|
||||||
|
|
||||||
|
Use existing offline tests as the starting point. Update fixtures intentionally,
|
||||||
|
and add focused behavioral coverage where the source migration creates a
|
||||||
|
distinct risk. Completion means:
|
||||||
|
|
||||||
|
- The released dependency set builds and the complete Audita test suite passes.
|
||||||
|
- All registered embedded prompts load and render from keyed assets, retaining
|
||||||
|
their applicable shared-instruction references and established per-call schemas.
|
||||||
|
- Offline configuration validation rejects incomplete selected profiles, including
|
||||||
|
a missing or blank model, while accepting valid unselected partial profiles.
|
||||||
|
- Selected roles use final application backend settings and direct resolved
|
||||||
|
credentials; both roles still share configured capacity, including explicit
|
||||||
|
zero queue capacity. Catalog layering does not introduce a competing lookup.
|
||||||
|
- Representative keyed profiles, native inheritance, and explicit presence work
|
||||||
|
through the real runtime. Directory conflicts and invalid final definitions
|
||||||
|
produce useful startup errors without filename-derived identity assumptions.
|
||||||
|
- Operator prompt and backend contributions follow the accepted source order;
|
||||||
|
prompt overrides render, including when their messages omit shared-instruction
|
||||||
|
variables, while Audita retains its per-call schemas, domain validation, and
|
||||||
|
authoritative application backend settings.
|
||||||
|
- Source collision protection, safe inspection, and operator-versus-asset error
|
||||||
|
classification remain effective at the changed construction boundary.
|
||||||
|
- Representative prepared execution, schema repair, transient retry, and
|
||||||
|
partial-success tests retain their existing behavior. Reuse adequate coverage
|
||||||
|
rather than duplicating PromptKit's complete upstream test matrix.
|
||||||
|
- Local, service, testing-overlay, and ConfigKit inheritance examples validate
|
||||||
|
offline using their separately selected profile roots. Canonical documentation
|
||||||
|
describes the migrated format, shared-instruction behavior, and major-release
|
||||||
|
migration; attribution identifies the selected library sources accurately.
|
||||||
|
|
||||||
|
No live provider calls, new release gates, or CI-success dependency are needed
|
||||||
|
to establish adoption. Guided domain recovery and the other work recorded in
|
||||||
|
[future.md](future.md) remain separate features. Do not expand this upgrade into
|
||||||
|
new report formats, pipeline scheduling, or general configuration-inspection
|
||||||
|
commands. The [implementation plan](implementation.md) owns execution steps
|
||||||
|
and their verification.
|
||||||
|
|
||||||
|
## Completion Record
|
||||||
|
|
||||||
|
Implementation code is recorded through revision `5525fe8`; documentation was
|
||||||
|
updated in `9bcf183`, and the final roadmap closure is documentation-only. At
|
||||||
|
`9bcf183`, `go build -o ./bin/audita ./cmd/audita`, `go test ./...`, all six
|
||||||
|
offline maintained-example commands in the implementation plan, `go mod
|
||||||
|
verify`, and `git diff --check` passed. The selected dependency graph and
|
||||||
|
commit-pinned attribution match PromptKit v0.12.0 and both `/v3` catalogs at
|
||||||
|
`v3.20260930.0`.
|
||||||
@@ -1,161 +0,0 @@
|
|||||||
# Maintainability Audit: Findings and Recommended Changes
|
|
||||||
|
|
||||||
**Reviewed:** 2026-09-25, source commit `011b021`.
|
|
||||||
**Scope:** Recent partial-decision reconciliation, validation context, prompt
|
|
||||||
alignment, and request-local reference changes, including related existing code.
|
|
||||||
**Status:** R27–R30 addressed; see the resolution record below. Previous findings and resolution records remain in Git
|
|
||||||
history through `011b021`.
|
|
||||||
|
|
||||||
## Architecture Assessment
|
|
||||||
|
|
||||||
The recent changes fit Audita's existing boundaries. Proposal generation,
|
|
||||||
validation, prompt assets, and outcome reporting already have suitable owners;
|
|
||||||
this audit does not recommend a new common package or a broad refactor.
|
|
||||||
|
|
||||||
The strongest opportunities are shared prompt instructions and request-scoped
|
|
||||||
validation context preparation. Two smaller improvements concern recovery-reason
|
|
||||||
classification and partial-result assembly; both predate the latest commits.
|
|
||||||
These are bounded maintenance recommendations, not new release gates or changes
|
|
||||||
to correction policy, response acceptance, or reporting semantics.
|
|
||||||
|
|
||||||
## R27 — P2: Consolidate Shared Prompt Instructions
|
|
||||||
|
|
||||||
**Category:** Repeated behavioral-contract text; risk of future drift.
|
|
||||||
|
|
||||||
**Evidence:** The four [proposal prompts](../../internal/prompts/assets/prompts/modules.grammar.proposal.yaml)
|
|
||||||
repeat literal source-span matching, request-local identity, replacement-text,
|
|
||||||
source-based proposal, and confidence instructions. The three active
|
|
||||||
[validator prompts](../../internal/prompts/assets/prompts/validators.editorial_review.yaml)
|
|
||||||
repeat candidate-reference, supplemental-context, and verdict-confidence guidance.
|
|
||||||
Recent contract changes required coordinated edits across these assets. The
|
|
||||||
[prompt registry](../../internal/prompts/registry.go) already provides a shared
|
|
||||||
hardening fragment supplied through the role-bound adapter.
|
|
||||||
|
|
||||||
**Recommended change:** Extract genuinely common passages into embedded fragments
|
|
||||||
under `internal/prompts/assets/shared`, following the existing fragment-supply
|
|
||||||
approach. Keep the fragments owned by `internal/prompts` and use the existing
|
|
||||||
PromptKit rendering path. Keep module-specific examples, replacement-policy
|
|
||||||
instructions, protected-name exceptions, and semantic criteria in their individual
|
|
||||||
prompt definitions. Do not merge similar-looking rules with different meanings.
|
|
||||||
|
|
||||||
**Acceptance:** Common instructions have one maintained source and appear in all
|
|
||||||
intended rendered prompts. The existing module and validator distinctions remain
|
|
||||||
intact, including glossary replace-all behavior and the spoken-word protected-name
|
|
||||||
exception. Focused rendering checks cover fragment inclusion and representative
|
|
||||||
module-specific guidance without requiring exact-prose snapshots or introducing
|
|
||||||
a separate templating framework.
|
|
||||||
|
|
||||||
## R28 — P2: Prepare Validation Context Once Per Request
|
|
||||||
|
|
||||||
**Category:** Repeated computation and overlapping segment lookup.
|
|
||||||
|
|
||||||
**Evidence:** [BuildLLMValidationRequest](../../internal/framework/validators/llm_validators.go)
|
|
||||||
already indexes transcript segments, but
|
|
||||||
[validationContext](../../internal/framework/validators/llm_context.go) scans the
|
|
||||||
transcript again for each candidate. It also calls
|
|
||||||
[NewProtectedVocabulary](../../internal/framework/validators/protected_terms.go)
|
|
||||||
for each candidate, rebuilding the same vocabulary and compiling its matching
|
|
||||||
expression repeatedly. This is duplicated preparation rather than copied code.
|
|
||||||
|
|
||||||
**Recommended change:** Prepare a segment-position lookup and protected vocabulary
|
|
||||||
once per validation request, then pass that prepared context into the existing
|
|
||||||
context-selection helper. Keep the implementation within `framework/validators`.
|
|
||||||
Continue selecting glossary evidence for each candidate individually because its
|
|
||||||
proposed replacement can mention different terms. A process-wide cache or new
|
|
||||||
context-management package is unnecessary for this cleanup.
|
|
||||||
|
|
||||||
**Acceptance:** Candidates in one validation request reuse the prepared lookup and
|
|
||||||
vocabulary. Context still consists of the available immediate neighbors within the
|
|
||||||
current chunk and relevant glossary entries in glossary order. Candidate-specific
|
|
||||||
replacement text still participates in glossary selection. Existing optional-context
|
|
||||||
trimming, token estimation, and oversized-input handling retain their behavior.
|
|
||||||
Verify these outcomes with focused behavioral tests rather than assertions about
|
|
||||||
private helper call counts or unmeasured performance guarantees.
|
|
||||||
|
|
||||||
## R29 — P3: Share Execution-Recovery Classification
|
|
||||||
|
|
||||||
**Category:** Duplicate reporting rule; predates the latest commits.
|
|
||||||
|
|
||||||
**Evidence:** [Proposal generation](../../internal/framework/proposal_generation/generate.go)
|
|
||||||
and [LLM validation](../../internal/framework/validators/llm_validators.go)
|
|
||||||
independently translate structural-repair and transient-retry counts into
|
|
||||||
`output_repaired` and `service_recovered` reasons. The conversion is the same in
|
|
||||||
both locations; the surrounding success and domain-outcome decisions differ.
|
|
||||||
|
|
||||||
**Recommended change:** Place a small conversion helper in the existing
|
|
||||||
`framework/outcomes` package and use it from both callers. Keep the decision to
|
|
||||||
record recoveries at the call sites, along with domain-specific reasons such as
|
|
||||||
filtered validator decisions. Do not combine proposal and validation orchestration
|
|
||||||
or move domain-success classification into this helper.
|
|
||||||
|
|
||||||
**Acceptance:** Both callers use the same execution-count conversion. Existing
|
|
||||||
reason ordering, successful-operation reporting, and domain-specific additions
|
|
||||||
remain intact. A partially unavailable validation batch continues to record its
|
|
||||||
failure and execution counts without acquiring recovery records merely because a
|
|
||||||
repair or retry occurred. Existing outcome tests should continue to protect these
|
|
||||||
distinctions.
|
|
||||||
|
|
||||||
## R30 — P3: Centralize Partial Validator Result Assembly
|
|
||||||
|
|
||||||
**Category:** Repeated early-return construction; predates the latest commits.
|
|
||||||
|
|
||||||
**Evidence:** Several early returns in
|
|
||||||
[LLMBackedValidator.Validate](../../internal/framework/validators/llm_validators.go)
|
|
||||||
repeat nested slice appends to combine immediate decisions, oversized-input
|
|
||||||
decisions, and decisions from completed model batches, together with accumulated
|
|
||||||
operation outcomes. This makes it easier for a future failure path to omit facts
|
|
||||||
already collected by the validator.
|
|
||||||
|
|
||||||
**Recommended change:** Use a small local result-building helper to assemble the
|
|
||||||
partial result consistently. Keep error handling, batching, success-path ordering,
|
|
||||||
and cardinality enforcement explicit. A general-purpose result-aggregation package
|
|
||||||
is not needed.
|
|
||||||
|
|
||||||
**Acceptance:** Cancellation and failure paths retain the same previously collected
|
|
||||||
decisions and operations. Result ordering and cardinality behavior remain unchanged;
|
|
||||||
the helper does not enforce complete cardinality on interrupted work. Retain the
|
|
||||||
existing partial-progress and fatal-failure coverage, adding tests only for a
|
|
||||||
material gap exposed by the refactor.
|
|
||||||
|
|
||||||
## Boundaries to Preserve
|
|
||||||
|
|
||||||
- Keep proposal and validator reference mapping separate. Proposals can target the
|
|
||||||
same segment repeatedly; duplicate validator decisions make that candidate
|
|
||||||
unavailable. Similar indexing arithmetic alone does not justify a generic
|
|
||||||
reference-management abstraction.
|
|
||||||
- Keep distinct protected-term policy checks separate. They already share the
|
|
||||||
vocabulary implementation and intentionally apply different policies.
|
|
||||||
- Preserve exact source-span matching, request-local references, batching, and
|
|
||||||
partial-success behavior. Consolidation is not a reason to strengthen contracts
|
|
||||||
or broaden the accepted correction scope.
|
|
||||||
|
|
||||||
## Review Verification and Limits
|
|
||||||
|
|
||||||
- This is a source-level maintainability audit, not a finding of demonstrated
|
|
||||||
production failures or a measured performance assessment.
|
|
||||||
- No application tests were rerun for the original read-only audit; its roadmap
|
|
||||||
update used documentation link and whitespace checks. Implementation validation
|
|
||||||
is recorded below.
|
|
||||||
- Future remediation should run focused behavioral tests and the broader suite
|
|
||||||
appropriate to the actual change, following the testing policy. No live-provider
|
|
||||||
evaluation or new evaluation framework is required by these recommendations.
|
|
||||||
- The findings above describe the audited baseline. Implementation and validation
|
|
||||||
are recorded below; no release publication is included.
|
|
||||||
|
|
||||||
## Resolution Record
|
|
||||||
|
|
||||||
Implemented as focused changes within the existing packages. Proposal and validator
|
|
||||||
reference mapping and distinct protected-term policies remain separate.
|
|
||||||
|
|
||||||
| Finding | Resolution and evidence |
|
|
||||||
| --- | --- |
|
|
||||||
| R27 | Common proposal and validation instructions live in two embedded shared fragments, supplied through the existing role-bound PromptKit adapter. Individual YAML definitions retain policy differences and examples. Rendering tests check fragment inclusion once per applicable prompt and representative module-specific guidance. |
|
|
||||||
| R28 | Validation request construction builds one segment-position lookup and protected vocabulary, reused by context selection for every candidate. A new test verifies that two replacements of the same source span receive their own relevant glossary entries. Existing neighbor, glossary-order, chunk-boundary, and budget tests pass. |
|
|
||||||
| R29 | Both proposal generation and validation use `outcomes.ExecutionRecoveries`. Call sites still control success and domain-specific recovery reasons. Existing tests cover combined recoveries and failed validation with repair counts but no recovery record. |
|
|
||||||
| R30 | A local result builder combines collected decisions and operation outcomes for partial returns and completed validation. Sorting and full cardinality checks remain outside it. Existing fatal-batch, partial-progress, runner, and reporting tests pass. |
|
|
||||||
|
|
||||||
Validation: `go test ./...` passed, including PromptKit preparation, partial-decision
|
|
||||||
reconciliation, subprocess reports, and fatal-progress coverage. Documentation
|
|
||||||
links and whitespace checks passed. No live model evaluation or performance
|
|
||||||
measurement was performed; the changes make no new model-quality or latency
|
|
||||||
promise.
|
|
||||||
@@ -1,290 +0,0 @@
|
|||||||
# Resilient Processing and Focused Run Reports
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-24
|
|
||||||
|
|
||||||
**Status:** Implemented. This document records the selected behavior and user
|
|
||||||
intent. Concrete design decisions and completion evidence are in
|
|
||||||
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/a83a518/docs/roadmap/implementation.md).
|
|
||||||
|
|
||||||
## Purpose and User Intent
|
|
||||||
|
|
||||||
Audita should reliably produce a useful transcript even when its LLM services
|
|
||||||
cannot improve it. Exhausted retries or unusable model responses must not turn
|
|
||||||
an otherwise processable transcript into a failed production run.
|
|
||||||
|
|
||||||
> Failure of every LLM operation across every stage and chunk still produces a
|
|
||||||
> successful output containing the deterministically normalized transcript, with
|
|
||||||
> no LLM corrections applied. Partial success preserves as many fully validated,
|
|
||||||
> applicable corrections as possible.
|
|
||||||
|
|
||||||
Deterministic normalization remains intentional and may change segment layout,
|
|
||||||
identifiers, and text before LLM work begins. Output uses the selected transcript
|
|
||||||
encoder; byte-for-byte preservation of the original input and mapping corrections
|
|
||||||
back to pre-normalization segmentation are not requirements.
|
|
||||||
|
|
||||||
The overriding goal is to minimize how much useful processing is lost because of
|
|
||||||
one problematic transcript segment. A stage/chunk work unit is an attribution
|
|
||||||
boundary, not an all-or-nothing transaction.
|
|
||||||
|
|
||||||
Existing reports contain useful information, but there are no compatibility
|
|
||||||
obligations for reporting or debug output. Reuse existing machinery only when it
|
|
||||||
makes the implementation simpler and clearer; preservation is not itself a goal.
|
|
||||||
Internal structures and measurements must serve Audita's processing workflow.
|
|
||||||
The reporting package adapts those outcomes to structured and human-facing
|
|
||||||
presentations, rather than reshaping processing around a reporting convenience.
|
|
||||||
|
|
||||||
## Historical Baseline and Implemented Change
|
|
||||||
|
|
||||||
The [pipeline](../internal/pipeline.md), [LLM integration](../integrations/openai-compatible-llm.md),
|
|
||||||
[report assembly](../internal/diagnostics-reporting.md), and
|
|
||||||
[artifact lifecycle](../operations.md) describe current behavior.
|
|
||||||
|
|
||||||
Before this work, exhausted structural repairs could become warnings and skipped
|
|
||||||
proposals or rejected validation batches. Other model runtime failures canceled
|
|
||||||
peer work, stopped subsequent modules, and prevented final transcript output.
|
|
||||||
Successful repairs appeared in execution observations rather than a dedicated
|
|
||||||
recovery classification. Detailed artifacts were written during ordinary runs
|
|
||||||
and could later be removed by retention. The working transcript was re-chunked
|
|
||||||
between modules.
|
|
||||||
|
|
||||||
This feature changed those behaviors. It builds on the completed
|
|
||||||
[library adoption](library-adoption.md) and [review remediation](remediation.md).
|
|
||||||
Fatal cancellation remains necessary; unsuccessful model work must be separated
|
|
||||||
from failures that actually require cancellation of the run.
|
|
||||||
|
|
||||||
## Fixed Chunks and Ordered Stages
|
|
||||||
|
|
||||||
Normalize first, then partition the transcript once. Preserve chunk boundaries,
|
|
||||||
segment membership, segment identity, and ordering throughout LLM processing.
|
|
||||||
Text evolves as accepted corrections are applied. Recombine chunks in original
|
|
||||||
order after the LLM stages finish.
|
|
||||||
|
|
||||||
A stage is a configured module instance. Repeated instances of the same module
|
|
||||||
have distinct identities. Proposal generation and each validator are operations
|
|
||||||
within a stage; validation may use multiple batches within a chunk. A work unit
|
|
||||||
is one stage instance applied to one chunk.
|
|
||||||
|
|
||||||
Preserve the overall stage sequence: finish a stage across its chunks before
|
|
||||||
advancing to the next stage. Concurrent chunk work shares PromptKit's backend
|
|
||||||
capacity. Failure in one unit does not prevent other chunks or later stages
|
|
||||||
from executing.
|
|
||||||
|
|
||||||
Retain operator control over chunk size and the current defaults. Production
|
|
||||||
experience indicates that chunk size affects model latency nonlinearly and that
|
|
||||||
useful settings are well below backend context limits. Tuning is an empirical
|
|
||||||
operator decision. Do not add automatic headroom calculations, cross-profile
|
|
||||||
sizing, adaptive repartitioning, or automatic tuning in this feature. A later
|
|
||||||
operation that cannot process its input follows the same safe error handling;
|
|
||||||
it does not silently alter chunk boundaries. Validation batching within a chunk
|
|
||||||
remains supported.
|
|
||||||
|
|
||||||
Independent whole-pipeline traversal by chunks and domain-aware response repair
|
|
||||||
are explicitly deferred in [future work](future.md). Read-only neighboring
|
|
||||||
context, cross-chunk edits, and transcript-wide restructuring are outside this
|
|
||||||
unit, without becoming permanent prohibitions on future capabilities.
|
|
||||||
|
|
||||||
## Partial Success and Failure Isolation
|
|
||||||
|
|
||||||
- Keep bounded transport retries, structural repairs, operation deadlines, and
|
|
||||||
shared backend capacity. Completion comes from safe fallback, not unlimited
|
|
||||||
attempts or an outer retry loop around PromptKit.
|
|
||||||
- Apply a correction only after it passes every required validator and the
|
|
||||||
application checks. Failed validation must never be treated as approval.
|
|
||||||
- If a validation batch fails, exclude its affected candidates while allowing
|
|
||||||
other candidates to continue through the remaining validators. Preserve fully
|
|
||||||
validated, applicable corrections from successful batches in the same unit.
|
|
||||||
- Track candidates by stable proposal identity. Batch membership can change
|
|
||||||
between validators; equal batch numbers do not establish candidate identity.
|
|
||||||
- A failed proposal-generation operation leaves its chunk unchanged for that
|
|
||||||
stage because there are no reliable proposals to salvage. Failures later in
|
|
||||||
validation need not discard unrelated successful candidates.
|
|
||||||
- Preserve earlier stages' corrections and other chunks' successful work.
|
|
||||||
Record partial success explicitly; a unit can apply corrections and have errors.
|
|
||||||
- Continue despite model-operation failures, including exhausted repairs/retries,
|
|
||||||
unusable model decisions, provider/transport failures, local operation deadlines,
|
|
||||||
and admission failures. An operation need not exhaust its attempt budget to fail.
|
|
||||||
- A valid empty proposal list or valid assessments rejecting all corrections is
|
|
||||||
successful processing. Correction count alone does not determine failure.
|
|
||||||
- Keep safe filtering of defective proposals. Do not retry ordinary editorial
|
|
||||||
rejections. For now, an unusable validator decision set excludes the affected
|
|
||||||
candidates and records an error after existing permitted recovery. New bounded
|
|
||||||
feedback/regeneration for domain-invalid responses belongs to future work.
|
|
||||||
- Retain fatal treatment for invalid input/configuration, internal defects,
|
|
||||||
explicit run cancellation, and inability to produce/write the transcript.
|
|
||||||
Caller cancellation is distinct from a local model-operation timeout.
|
|
||||||
- Completed runs with model errors return success. Required report-write failures
|
|
||||||
are an additional fatal output condition when reporting is enabled, as below.
|
|
||||||
|
|
||||||
PromptKit owns transport, transient retries, structural repair, and admission.
|
|
||||||
Audita owns domain acceptance, partial outcomes, fallback, and classification.
|
|
||||||
ConfigKit owns configuration resolution. Debug and reporting choices must not
|
|
||||||
change correction decisions, retry budgets, or continuation.
|
|
||||||
|
|
||||||
## Errors, Recoveries, and Debug
|
|
||||||
|
|
||||||
Classify by eventual outcome and impact. These are three reporting categories;
|
|
||||||
debug is evidence, not a failure severity. Whether execution continues is a
|
|
||||||
separate property.
|
|
||||||
|
|
||||||
| Category | Meaning | Representative cases |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| Errors | Intended work could not complete reliably, or a fatal run-level operation failed. Affected candidates are excluded; other corrections can succeed. | Exhausted repair; unavailable provider; unusable validation decisions; operation deadline. |
|
|
||||||
| Recoveries | A problem was handled successfully through retries, repair, or safe filtering. No operator intervention is implied. | Invalid output repaired; transient service failure recovered; defective proposals safely discarded. |
|
|
||||||
| Debug | Detailed evidence of processing and ordinary decisions. | Proposals, applied corrections, validator decisions, prompts/responses, individual attempts, execution traces. |
|
|
||||||
|
|
||||||
Normal editorial disagreement belongs in debug. A validator correctly rejecting
|
|
||||||
a meaning-changing edit is expected behavior. A proposal referring to nonexistent
|
|
||||||
original text is defective; safely filtering it is a recovery. An unresolved
|
|
||||||
missing/duplicate decision-index response is an error, not a recovery.
|
|
||||||
|
|
||||||
Record an operation's eventual outcome once. Several failed attempts followed by
|
|
||||||
success produce a consolidated recovery. Attempts ending in failure contribute
|
|
||||||
to error details, not additional recoveries. Earlier operations that recovered
|
|
||||||
remain recorded if a later operation introduces an error in the same unit.
|
|
||||||
|
|
||||||
Separate output-quality recovery, defective-proposal filtering, and service
|
|
||||||
recovery with structured reasons. Network instability must not look like a prompt
|
|
||||||
defect. Do not introduce an arbitrary warning threshold or a fourth category.
|
|
||||||
|
|
||||||
### Counting and Processing Volume
|
|
||||||
|
|
||||||
- Count error-bearing work units once per stage/chunk, consolidating underlying
|
|
||||||
operation failures as details. A partially successful unit still counts once.
|
|
||||||
- Count recoveries once per recovered operation. Aggregate defective-proposal
|
|
||||||
filtering per proposal response, retaining affected-proposal counts as detail.
|
|
||||||
Attempts and individual discarded proposals are not independent incidents.
|
|
||||||
- Keep work-unit coverage separate from operation/recovery counts. Show processing
|
|
||||||
volume with explicitly labeled denominators; do not divide recovered-operation
|
|
||||||
counts by work-unit counts and call that a recovery rate.
|
|
||||||
- Report fatal run-level errors separately from the stage/chunk grid when they
|
|
||||||
cannot be attributed to a work unit. Do not invent a chunk to accommodate them.
|
|
||||||
- Derive YAML records, Markdown summaries, and CLI counts from actual processing
|
|
||||||
outcomes with deterministic ordering, rather than independently calculating them.
|
|
||||||
|
|
||||||
## Production Outputs, Placement, and Lifecycle
|
|
||||||
|
|
||||||
Reporting is enabled by default. The ordinary bundle is:
|
|
||||||
|
|
||||||
- `errors.yml`: structured error-bearing units, their underlying failures, and
|
|
||||||
applicable run-level failures.
|
|
||||||
- `recoveries.yml`: structured recovered operations and pertinent counts.
|
|
||||||
- `report.md`: a concise human-facing summary of outcome, corrections, processing
|
|
||||||
volume, errors, recoveries, and their locations in the pipeline.
|
|
||||||
|
|
||||||
Write all three documents, including empty event lists for uneventful completed
|
|
||||||
runs. Use Audita-owned schemas with safe explanations and stable reason codes;
|
|
||||||
retain run, stage-instance, chunk, operation, and validator/batch identities where
|
|
||||||
applicable. The implementation plan specifies the versioned schemas; neither old
|
|
||||||
report formats nor raw library structures define the new output contract.
|
|
||||||
|
|
||||||
Place reports beside the output transcript. Retain transcript-to-stdout support
|
|
||||||
with an explicit report-directory selection, defaulting to the invocation working
|
|
||||||
directory. Preserve the existing overwrite convention: a new run can replace files
|
|
||||||
at its selected destinations. Do not add a mandatory fresh-directory restriction
|
|
||||||
or compatibility exports.
|
|
||||||
|
|
||||||
Before model work, create initial versions of all three reports carrying a common
|
|
||||||
run identifier and explicit `in_progress` status (rendered as prose in Markdown).
|
|
||||||
Fail immediately if initialization fails. Also check the transcript destination;
|
|
||||||
creating neighboring reports does not prove that an existing transcript file is
|
|
||||||
writable. Preflight must not prematurely destroy an existing transcript.
|
|
||||||
|
|
||||||
Finalize all required outputs and check every final write. Initial write success
|
|
||||||
cannot guarantee later success if space or permissions change. Failure to write
|
|
||||||
the transcript or finalize a required report returns nonzero and must not erase
|
|
||||||
an already produced transcript. Where possible, finalize available reports with
|
|
||||||
an explicit failed status on fatal termination. Abrupt termination may leave
|
|
||||||
`in_progress`; elaborate crash-recovery machinery is not required.
|
|
||||||
|
|
||||||
Provide an option to disable reporting entirely. It suppresses all three files
|
|
||||||
(`errors.yml`, `recoveries.yml`, and `report.md`), including initialization and
|
|
||||||
final writes, while retaining the brief CLI summary and independently selectable
|
|
||||||
debug output. Disabled reporting must not create or require a writable report
|
|
||||||
destination, and must not remove reports from earlier runs.
|
|
||||||
Transcript output failures remain fatal regardless of reporting selection.
|
|
||||||
|
|
||||||
Ordinary reports survive independently of debug cleanup. Reporting configuration
|
|
||||||
and finalization must not depend on retaining a debug run directory.
|
|
||||||
|
|
||||||
## Debug and Internal Boundaries
|
|
||||||
|
|
||||||
Production issue records are compact. Full responses, transcript excerpts,
|
|
||||||
correction lists, and complete validation decisions are opt-in debug evidence.
|
|
||||||
Use a single ConfigKit-backed debug boolean with a `--debug` CLI override. Debug
|
|
||||||
artifacts may use JSON where useful. Preserve configured-secret redaction in every
|
|
||||||
presentation.
|
|
||||||
|
|
||||||
Debug-off prevents detailed artifact persistence; writing and later deleting
|
|
||||||
those files is insufficient. In-memory domain state needed to process candidates
|
|
||||||
correctly is not debug output. Debug and ordinary reporting are independent
|
|
||||||
selections and do not alter processing behavior.
|
|
||||||
|
|
||||||
There is no obligation to retain `--report-json`, old artifact layouts, parallel
|
|
||||||
exports, or old retention settings. Reuse code only where useful to the new
|
|
||||||
workflow. Enabled debug output is retained on success or failure; disabled debug
|
|
||||||
creates no debug directory. This supersedes the retention question in the
|
|
||||||
[runtime follow-up](runtime-behavior.md). The legacy `never` behavior is not the
|
|
||||||
meaning of the new reporting-disable option. Optional debug-write failures produce
|
|
||||||
a safe stderr diagnostic without changing correction decisions or the run exit
|
|
||||||
status; ordinary report and transcript-write failures retain their fatal semantics.
|
|
||||||
|
|
||||||
## Deterministic Human-Facing Presentation
|
|
||||||
|
|
||||||
Use the CLI style:
|
|
||||||
|
|
||||||
> Completed: 18 corrections applied; 3 work units with errors; 7 recoveries.
|
|
||||||
|
|
||||||
Do not say that all error-bearing units were skipped: some apply valid corrections.
|
|
||||||
Include report locations when reporting is enabled and the files exist. Send the
|
|
||||||
summary to stderr so stdout remains exclusively available for transcript output.
|
|
||||||
Disabled reporting does not suppress the summary or fatal-error messages.
|
|
||||||
|
|
||||||
Keep `report.md` concise and high-level. Use separate error and recovery matrices
|
|
||||||
with stages as columns and chunks as rows. Include every chunk in fixed order;
|
|
||||||
retain configured stage order and distinct labels for repeated module instances.
|
|
||||||
No automatic width detection, thresholds, adaptive omission, or table splitting.
|
|
||||||
A wide or long table is acceptable.
|
|
||||||
|
|
||||||
The error matrix counts error-bearing work units: each attempted cell is 0 or 1.
|
|
||||||
Recovery cells count recovered operations and can exceed 1. An error cell may
|
|
||||||
coexist with recoveries and successfully applied corrections in the same unit.
|
|
||||||
Distinguish completed work with zero events from unattempted work; do not portray
|
|
||||||
canceled/unavailable processing as clean completion. Include totals, processing
|
|
||||||
volume, and a short reason breakdown. Detailed inspection belongs in YAML or debug.
|
|
||||||
|
|
||||||
## Target Acceptance Criteria
|
|
||||||
|
|
||||||
- Total LLM failure still returns success and the encoded normalized transcript,
|
|
||||||
provided required output writes succeed.
|
|
||||||
- Mixed-success runs preserve independently validated, applicable corrections
|
|
||||||
within the same stage/chunk as a failed batch. No candidate bypasses a required
|
|
||||||
validator; later validators handle survivors by stable proposal identity.
|
|
||||||
- Subsequent stages and unaffected chunks execute after recoverable failures.
|
|
||||||
Fatal errors and explicit cancellation stop and join active work appropriately.
|
|
||||||
- Chunk boundaries and identity remain fixed, with operator-selected sizing and
|
|
||||||
unchanged defaults; final recombination preserves ordering.
|
|
||||||
- Valid no-change/editorial-rejection outcomes are successful. Repaired/filtered
|
|
||||||
output and unresolved failures receive distinct classifications and counts.
|
|
||||||
- YAML, Markdown, and CLI agree on partial outcomes and totals; error matrices
|
|
||||||
count affected work units, recovery matrices count recovered operations.
|
|
||||||
- With reporting enabled, all three reports are initialized before model work,
|
|
||||||
finalized and retained, including empty lists. Initial or final required-output
|
|
||||||
write failures produce nonzero exits without destroying completed transcript output.
|
|
||||||
- Reporting can be disabled without requiring its destination. Debug can be enabled
|
|
||||||
independently; neither selection changes processing outcomes. Debug-off writes
|
|
||||||
no detailed debug artifacts, and stdout transcript output remains supported.
|
|
||||||
- Reports use the fixed deterministic layout without adaptive presentation logic.
|
|
||||||
- Meaningful controlled-provider tests cover exhausted recovery, failed validation
|
|
||||||
batches with successful survivors, total failure, fatal cancellation, output
|
|
||||||
failures, and reporting/debug selections. Preserve domain safeguards, capacity
|
|
||||||
limits, and redaction.
|
|
||||||
- Canonical CLI, configuration, operations, integration, internal, and architecture
|
|
||||||
documentation describes the implemented final behavior.
|
|
||||||
|
|
||||||
## Implementation Reference
|
|
||||||
|
|
||||||
The product decisions are settled. The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/a83a518/docs/roadmap/implementation.md) defines
|
|
||||||
the selected settings and precedence, YAML schemas, counts and ordering, interrupted
|
|
||||||
work presentation, destination checks, and output finalization. It also owns the
|
|
||||||
ordered implementation stages and validation work. This document remains the
|
|
||||||
reference for the desired behavior and its acceptance criteria.
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
# Runtime Behavior Decisions
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-24
|
|
||||||
|
|
||||||
The documentation migration identified existing behaviors for separate product
|
|
||||||
consideration. The resilience implementation resolved the diagnostics-retention
|
|
||||||
question; the input-timestamp question remains open.
|
|
||||||
|
|
||||||
- Resolved: legacy `auto`/`always`/`never` retention settings were removed.
|
|
||||||
Debug is explicitly enabled with `diagnostics.debug` or `--debug`, creates no
|
|
||||||
directory when disabled, and retains its directory on every outcome when enabled.
|
|
||||||
- Resolved: [configuration](../config.md) now treats explicit zero as disabling
|
|
||||||
transient retries or structural repair independently; PromptKit performs both.
|
|
||||||
- Decide whether missing or null input timestamps should be rejected; current
|
|
||||||
decoding is documented in the [input contract](../integrations/transcript-glossary-files.md).
|
|
||||||
|
|
||||||
These decisions are independent of the completed documentation migration and
|
|
||||||
release workflow.
|
|
||||||
@@ -1,203 +0,0 @@
|
|||||||
# Consistent Span Deletion
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-25
|
|
||||||
|
|
||||||
**Status:** Implemented. This document owns the desired behavior and user intent.
|
|
||||||
The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md)
|
|
||||||
records the concrete steps, selected implementation details, and completion
|
|
||||||
evidence.
|
|
||||||
|
|
||||||
## Purpose and User Intent
|
|
||||||
|
|
||||||
Align proposal prompts, the LLM response schema, internal validation, and
|
|
||||||
application behavior so legitimate text-span deletions can be considered and
|
|
||||||
applied through the ordinary correction workflow. An empty replacement should
|
|
||||||
not trigger structural repair merely because it is empty, nor invalidate other
|
|
||||||
usable corrections returned alongside it.
|
|
||||||
|
|
||||||
This is a consistency fix within Audita's current editing scope. It does not
|
|
||||||
authorize whole-segment removal, broaden module responsibilities, or introduce
|
|
||||||
new recovery machinery. The accepted [resilience behavior](resilience.md) remains
|
|
||||||
in effect, including partial success, fixed chunks, and validation before mutation.
|
|
||||||
|
|
||||||
## Historical Baseline (`0b6cd98`)
|
|
||||||
|
|
||||||
The source reviewed at `0b6cd98` contains a mismatch:
|
|
||||||
|
|
||||||
- The [spoken-word prompt](../../internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml)
|
|
||||||
permits removing fillers and asks for the exact affected span and replacement,
|
|
||||||
without explaining how to represent deletion. Grammar permits punctuation and
|
|
||||||
spacing changes, which can also naturally involve deletion.
|
|
||||||
- The shared [correction response schema](../../internal/framework/responseschema/assets/correction_set.v1.json)
|
|
||||||
requires `corrected_text` to have at least one character. Every built-in proposal
|
|
||||||
module uses that schema. An empty replacement invalidates the response before
|
|
||||||
Audita evaluates individual proposals.
|
|
||||||
- PromptKit performs bounded structural repair. If repair does not produce a
|
|
||||||
valid response, the proposal operation fails and yields no candidates, including
|
|
||||||
otherwise valid proposals in the same response. The current resilience path
|
|
||||||
records the error and continues other work.
|
|
||||||
- [CorrectionProposal](../../internal/framework/proposals/proposal.go) already
|
|
||||||
allows empty replacement text. The
|
|
||||||
[preview/application logic](../../internal/framework/proposals/preview.go)
|
|
||||||
supports literal span deletion when the resulting segment remains nonblank.
|
|
||||||
- Domain shape checks currently reject whitespace-only original spans, although
|
|
||||||
the response schema permits them and grammar prompts permit spacing cleanup.
|
|
||||||
This is another restriction to align with the target contract below.
|
|
||||||
- The validator named `non_empty_corrected_text` actually checks the resulting
|
|
||||||
segment, not the replacement string. Both this check and application prevent
|
|
||||||
a segment from becoming empty or whitespace-only. Application changes segment
|
|
||||||
text; it does not remove segment records.
|
|
||||||
|
|
||||||
A temporary controlled-provider review reproduced whole-response rejection for
|
|
||||||
an empty replacement alongside a valid correction, and immediate schema acceptance
|
|
||||||
for a space-only replacement. Existing proposal tests confirm internal support for
|
|
||||||
partial deletion and rejection of blank resulting segments. These observations
|
|
||||||
explain a plausible production failure mechanism; they do not establish the cause
|
|
||||||
of any particular historical incident without its payload and running version.
|
|
||||||
|
|
||||||
## Target Correction Contract
|
|
||||||
|
|
||||||
- `original_text` identifies an existing, nonempty literal span in the current
|
|
||||||
target segment. Whitespace-only spans are eligible; an empty string identifies
|
|
||||||
no source span and remains invalid. Matching and ambiguity checks still apply
|
|
||||||
under the module's replacement policy: `require_unique` for grammar, spoken-word,
|
|
||||||
and homophones; `replace_all` for glossary.
|
|
||||||
- `corrected_text` remains a required string. An empty string means delete the
|
|
||||||
identified span; it does not mean remove the segment record or skip validation.
|
|
||||||
Missing fields and non-string values remain structurally invalid.
|
|
||||||
- After applying a correction, the segment must contain non-whitespace text.
|
|
||||||
Enforce this using the source segment and replacement result in Audita, not a
|
|
||||||
minimum replacement length in the structural schema.
|
|
||||||
- Every accepted deletion passes the same required validators as other edits in
|
|
||||||
its module, including confidence, protected terms, editorial review, and meaning
|
|
||||||
preservation where applicable. Empty replacement is neither automatic approval
|
|
||||||
nor an automatic rejection.
|
|
||||||
- Application preserves segment identity, speaker, timing, membership, ordering,
|
|
||||||
and fixed chunk boundaries. Recheck applicability against text produced by
|
|
||||||
earlier accepted corrections, retaining existing conflict handling.
|
|
||||||
- Preserve literal replacement behavior. Prompts should select spans and
|
|
||||||
replacements that handle adjacent whitespace and punctuation appropriately;
|
|
||||||
this feature does not add automatic trimming or a second normalization pass.
|
|
||||||
Neither side of a correction is rejected solely for containing only whitespace.
|
|
||||||
For example, reducing a uniquely identified run of two spaces to one space may
|
|
||||||
be appropriate; validity still depends on the resulting text and domain checks.
|
|
||||||
|
|
||||||
Preserve the intended safeguards and outcomes, not every detail of the current
|
|
||||||
implementation. Validator ordering, internal structure, and payload construction
|
|
||||||
may change when needed to achieve this consistent contract. Existing behavior or
|
|
||||||
a passing test does not justify retaining a conflicting restriction.
|
|
||||||
|
|
||||||
Illustrative cases:
|
|
||||||
|
|
||||||
| Segment | Original span | Replacement | Intended handling |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| `uh hello` | `uh ` | empty string | Structurally valid; may produce `hello` after required validation. |
|
|
||||||
| `uh hello` | `uh hello` | `hello` | Remains a valid representation of the same cleanup, subject to validation. |
|
|
||||||
| `hello world` | two spaces | one space | Eligible spacing correction, subject to unique matching and required validation. |
|
|
||||||
| `uh` | `uh` | empty string | Structurally valid proposal, then filtered because the resulting segment would be blank. |
|
|
||||||
| `hello world` | `hello world` | spaces only | Filtered because the resulting segment would be blank. |
|
|
||||||
|
|
||||||
These examples describe eligibility and mechanical results, not unconditional
|
|
||||||
editorial approval. An unchanged segment or valid empty correction list remains
|
|
||||||
a successful no-change outcome.
|
|
||||||
|
|
||||||
## Prompt and Module Alignment
|
|
||||||
|
|
||||||
Review relevant prompt definitions for consistency and edit only those needing
|
|
||||||
clarification. Where needed, explain empty-string deletion, whitespace-only spans,
|
|
||||||
and the resulting-segment limit with a concise, module-appropriate example. The
|
|
||||||
model should omit a correction that would leave a blank segment rather than invent
|
|
||||||
padding or replacement content. Consistent guidance is the goal; adding deletion
|
|
||||||
language to every prompt is not a requirement.
|
|
||||||
|
|
||||||
Retain each module's existing scope:
|
|
||||||
|
|
||||||
- Spoken-word cleanup can propose appropriate filler or dysfluency deletion while
|
|
||||||
preserving substantive content and intentional emphasis.
|
|
||||||
- Grammar can propose permitted punctuation or spacing deletion; allowing an
|
|
||||||
empty replacement does not authorize word removal or spoken-word cleanup.
|
|
||||||
- Glossary and homophone modules retain their existing correction purposes.
|
|
||||||
Shared deletion syntax does not grant them new editorial responsibilities.
|
|
||||||
|
|
||||||
Review the active LLM-validator prompts and their input construction as well as
|
|
||||||
proposal prompts. Validators must understand the before/after effect of deletion
|
|
||||||
and assess it under their existing criteria. A correct negative editorial decision
|
|
||||||
remains ordinary processing, not a defect that should be regenerated until approved.
|
|
||||||
|
|
||||||
The resulting-segment validator's name and documented identity should describe
|
|
||||||
that responsibility accurately, rather than imply that the replacement string
|
|
||||||
itself must be nonempty. Preserve the existing blank-result failure reason and
|
|
||||||
safeguard while making that deliberate identity change; current chain position
|
|
||||||
and preview implementation are not independent product requirements.
|
|
||||||
|
|
||||||
## Outcomes and Reporting
|
|
||||||
|
|
||||||
A legitimate deletion is an ordinary correction. Empty replacement alone must
|
|
||||||
produce neither structural repair nor a recovery classification. If another
|
|
||||||
problem required successful repair or filtering, retain that actual recovery.
|
|
||||||
|
|
||||||
A schema-valid proposal that would blank its stage-input segment is a defective
|
|
||||||
proposal handled through existing safe filtering. Preserve the
|
|
||||||
`empty_resulting_segment` reason
|
|
||||||
and consolidated `proposals_filtered` recovery behavior; other usable proposals
|
|
||||||
in the response continue through required validation. Ordinary editorial rejection
|
|
||||||
remains debug evidence rather than a model error or filtering recovery.
|
|
||||||
|
|
||||||
A stage can propose several edits to one segment in the same response. Each edit
|
|
||||||
is evaluated against the stage input, then approved edits are applied sequentially.
|
|
||||||
If a proposal was applicable to that input but an earlier accepted edit makes it
|
|
||||||
unsafe, skip it as an apply-time conflict. Do not count that conflict as a defective
|
|
||||||
proposal, model error, or filtering recovery. For example, in `"uh um"`, deleting
|
|
||||||
`"uh "` and deleting `"um"` each leaves text individually; after the first deletion,
|
|
||||||
the second must be skipped to prevent a blank result. Retain the skipped decision
|
|
||||||
in debug evidence. The existing runner already distinguishes this situation.
|
|
||||||
|
|
||||||
Debug evidence must accurately retain the empty replacement and the corresponding
|
|
||||||
validation/application outcome. Ordinary reports retain their existing counts and
|
|
||||||
purpose; no new report schema, incident category, or deletion-specific counter is
|
|
||||||
needed. Unrelated structural defects may still exhaust PromptKit repair and produce
|
|
||||||
an error for the operation, as today.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- A controlled valid response containing an empty replacement reaches Audita's
|
|
||||||
domain validation without structural repair solely due to that empty string.
|
|
||||||
- A permitted span deletion can pass the required validators and reach the final
|
|
||||||
transcript while preserving segment metadata and fixed chunk membership.
|
|
||||||
- A mixed response can apply an approved deletion and another approved correction;
|
|
||||||
an empty replacement does not invalidate the whole response.
|
|
||||||
- A proposal that would blank its stage-input segment is filtered independently,
|
|
||||||
retains the existing reason, and does not prevent unrelated valid candidates
|
|
||||||
from succeeding. A proposal made unsafe only by an earlier accepted edit is
|
|
||||||
skipped without adding an error or filtering recovery; a blank result is still
|
|
||||||
prevented and the conflict remains visible in debug evidence.
|
|
||||||
- A uniquely matching whitespace-only original span can reach validation and
|
|
||||||
application. Empty originals remain invalid, ambiguous matches remain subject
|
|
||||||
to module policy, and whitespace-only resulting segments remain disallowed.
|
|
||||||
- Protected-term and meaning-changing deletions remain subject to their normal
|
|
||||||
rejection rules. No candidate bypasses required validation.
|
|
||||||
- A deletion needing no recovery counts as an ordinary applied correction; actual
|
|
||||||
repair/filtering and editorial rejection retain their existing classifications.
|
|
||||||
Debug evidence represents the empty replacement faithfully.
|
|
||||||
- Schema and domain tests distinguish an empty string from missing/non-string
|
|
||||||
replacement values and distinguish empty replacement from blank resulting text.
|
|
||||||
- Prompt guidance and validator naming express the same contract as the code.
|
|
||||||
Tests protect outcomes rather than exact prompt prose or incidental formatting.
|
|
||||||
|
|
||||||
Use deterministic, offline controlled responses for behavioral integration tests;
|
|
||||||
no live-provider or paid evaluation is a prerequisite. Representative production
|
|
||||||
examples can inform prompt quality without expanding this into an evaluation
|
|
||||||
framework or guaranteeing that models always follow the instructions.
|
|
||||||
|
|
||||||
## Deferred Work
|
|
||||||
|
|
||||||
[Whole-segment removal](future.md#whole-segment-removal) is only potential future
|
|
||||||
work and first requires deciding whether its benefit warrants implementation.
|
|
||||||
Domain-aware regeneration and independent whole-pipeline traversal also remain
|
|
||||||
in [future.md](future.md). This feature changes neither retry budgets nor the
|
|
||||||
Audita/PromptKit ownership boundary and requires no upstream library feature.
|
|
||||||
|
|
||||||
Implementation sequencing and concrete names belong to the
|
|
||||||
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md).
|
|
||||||
The target remains consistent deletion of spans within surviving segments, with
|
|
||||||
no new operator controls or report types.
|
|
||||||
15
examples/configkit-inheritance.yml
Normal file
15
examples/configkit-inheritance.yml
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
version: 2
|
||||||
|
profiles_dir: profiles/local
|
||||||
|
backends:
|
||||||
|
local-base:
|
||||||
|
endpoint: http://127.0.0.1:11434/v1
|
||||||
|
concurrency: 4
|
||||||
|
queue_capacity: 0
|
||||||
|
local:
|
||||||
|
extends: local-base
|
||||||
|
concurrency: 2
|
||||||
|
llm:
|
||||||
|
proposal_profile: local-proposal
|
||||||
|
validation_profile: local-validation
|
||||||
|
pipeline:
|
||||||
|
modules: [grammar]
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
version: 2
|
version: 2
|
||||||
|
profiles_dir: profiles/local
|
||||||
backends:
|
backends:
|
||||||
local:
|
local:
|
||||||
endpoint: http://127.0.0.1:11434/v1
|
endpoint: http://127.0.0.1:11434/v1
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
version: 2
|
version: 2
|
||||||
|
profiles_dir: profiles/service
|
||||||
backends:
|
backends:
|
||||||
service:
|
service:
|
||||||
endpoint: https://openrouter.ai/api/v1
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
id: local-proposal
|
|
||||||
backend: local
|
|
||||||
model: replace-with-local-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 2048
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: local-validation
|
|
||||||
backend: local
|
|
||||||
model: replace-with-local-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 1024
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
8
examples/profiles/local/local-proposal.yaml
Normal file
8
examples/profiles/local/local-proposal.yaml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
profiles:
|
||||||
|
local-proposal:
|
||||||
|
backend: local
|
||||||
|
model: replace-with-local-model
|
||||||
|
temperature: 0
|
||||||
|
max_tokens: 2048
|
||||||
|
top_p: 1
|
||||||
|
timeout_seconds: 120
|
||||||
8
examples/profiles/local/local-validation.yaml
Normal file
8
examples/profiles/local/local-validation.yaml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
profiles:
|
||||||
|
local-validation:
|
||||||
|
backend: local
|
||||||
|
model: replace-with-local-model
|
||||||
|
temperature: 0
|
||||||
|
max_tokens: 1024
|
||||||
|
top_p: 1
|
||||||
|
timeout_seconds: 120
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: service-proposal-testing
|
|
||||||
backend: service
|
|
||||||
model: replace-with-testing-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 2048
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: service-proposal
|
|
||||||
backend: service
|
|
||||||
model: replace-with-provider-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 2048
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: service-validation-testing
|
|
||||||
backend: service
|
|
||||||
model: replace-with-testing-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 1024
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: service-validation
|
|
||||||
backend: service
|
|
||||||
model: replace-with-provider-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 1024
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
8
examples/profiles/service/service-proposal-testing.yaml
Normal file
8
examples/profiles/service/service-proposal-testing.yaml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
profiles:
|
||||||
|
service-proposal-testing:
|
||||||
|
backend: service
|
||||||
|
model: replace-with-testing-model
|
||||||
|
temperature: 0
|
||||||
|
max_tokens: 2048
|
||||||
|
top_p: 1
|
||||||
|
timeout_seconds: 120
|
||||||
8
examples/profiles/service/service-proposal.yaml
Normal file
8
examples/profiles/service/service-proposal.yaml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
profiles:
|
||||||
|
service-proposal:
|
||||||
|
backend: service
|
||||||
|
model: replace-with-provider-model
|
||||||
|
temperature: 0
|
||||||
|
max_tokens: 2048
|
||||||
|
top_p: 1
|
||||||
|
timeout_seconds: 120
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
profiles:
|
||||||
|
service-validation-testing:
|
||||||
|
backend: service
|
||||||
|
model: replace-with-testing-model
|
||||||
|
temperature: 0
|
||||||
|
max_tokens: 1024
|
||||||
|
top_p: 1
|
||||||
|
timeout_seconds: 120
|
||||||
8
examples/profiles/service/service-validation.yaml
Normal file
8
examples/profiles/service/service-validation.yaml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
profiles:
|
||||||
|
service-validation:
|
||||||
|
backend: service
|
||||||
|
model: replace-with-provider-model
|
||||||
|
temperature: 0
|
||||||
|
max_tokens: 1024
|
||||||
|
top_p: 1
|
||||||
|
timeout_seconds: 120
|
||||||
8
go.mod
8
go.mod
@@ -3,14 +3,14 @@ module gitea.maximumdirect.net/eric/audita
|
|||||||
go 1.25.5
|
go 1.25.5
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2
|
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.11.2
|
gitea.maximumdirect.net/eric/promptkit v0.12.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 // indirect
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 // indirect
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 // indirect
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 // indirect
|
||||||
github.com/kr/pretty v0.3.1 // indirect
|
github.com/kr/pretty v0.3.1 // indirect
|
||||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||||
|
|||||||
16
go.sum
16
go.sum
@@ -1,11 +1,11 @@
|
|||||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2 h1:ZMxmv8pOnLtAx2hFTf8Z+pHKUbR89xErkBhwjr4yK24=
|
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.11.2 h1:2RqI+KtflvSh7vLQaTAQCmMaFQscEA7EPwPazyvXRoY=
|
gitea.maximumdirect.net/eric/promptkit v0.12.0 h1:oIFWEg6GPxtmX3Plqv0euP+FnK1LCk/t4xDy6oKCpP4=
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.11.2/go.mod h1:onm/FBPUKQ8/NSPwTEk26Ut+dVX2iXcq8vEseB+g/0E=
|
gitea.maximumdirect.net/eric/promptkit v0.12.0/go.mod h1:pFOOy5v1zH17stEi9rmHYmfbuLjnfVMcUhfmk+4VtJY=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 h1:wdab+R+lgMZxzYwhbAuCrW7zdSQ1cW9ovBR6UzLmZR4=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 h1:baCCIB6Ee5t0S4Zc++NTBIPcJEv5UPS6DlSrpZfykwg=
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ type configInput struct {
|
|||||||
TranscriptPath string
|
TranscriptPath string
|
||||||
GlossaryPath string
|
GlossaryPath string
|
||||||
OutputPath string
|
OutputPath string
|
||||||
|
ReportDirSet bool
|
||||||
Overrides []ck.Override
|
Overrides []ck.Override
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,6 +125,9 @@ func parseConfigInput(command string, args []string, output io.Writer) (configIn
|
|||||||
if f.Name == "config" {
|
if f.Name == "config" {
|
||||||
input.ConfigPathSet = true
|
input.ConfigPathSet = true
|
||||||
}
|
}
|
||||||
|
if f.Name == "report-dir" {
|
||||||
|
input.ReportDirSet = true
|
||||||
|
}
|
||||||
path, ok := settingPaths[f.Name]
|
path, ok := settingPaths[f.Name]
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
@@ -208,15 +212,22 @@ func configInputError(err error) string {
|
|||||||
return strings.Join(messages, "; ")
|
return strings.Join(messages, "; ")
|
||||||
}
|
}
|
||||||
|
|
||||||
func selectedReportDir(cfg config.Config, outputPath string) (string, error) {
|
func selectedReportDir(cfg config.Config, outputPath string, reportDirSet bool) (string, error) {
|
||||||
if !cfg.Reporting.Enabled {
|
if !cfg.Reporting.Enabled {
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
var reportDir string
|
||||||
if outputPath != "" {
|
if outputPath != "" {
|
||||||
return filepath.Dir(outputPath), nil
|
if reportDirSet {
|
||||||
|
reportDir = cfg.Reporting.StdoutDir
|
||||||
|
} else {
|
||||||
|
reportDir = filepath.Dir(outputPath)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.Reporting.StdoutDir) == "" {
|
} else {
|
||||||
return "", fmt.Errorf("report directory is required when reporting is enabled and transcript output is stdout")
|
reportDir = cfg.Reporting.StdoutDir
|
||||||
}
|
}
|
||||||
return cfg.Reporting.StdoutDir, nil
|
if strings.TrimSpace(reportDir) == "" {
|
||||||
|
return "", fmt.Errorf("report directory is required when reporting is enabled for the selected destination")
|
||||||
|
}
|
||||||
|
return reportDir, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -215,22 +215,42 @@ func TestReportingAndDebugOverridesRemainIndependent(t *testing.T) {
|
|||||||
t.Fatalf("%s did not honor explicit false: %+v", command, selected.Result.Config)
|
t.Fatalf("%s did not honor explicit false: %+v", command, selected.Result.Config)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
processInput, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--config", main, "--overlay", "testing"}, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
processConfig, err := loadConfigInput(processInput, "process")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, err := selectedReportDir(processConfig.Result.Config, filepath.Join("out", "corrected.json"), processInput.ReportDirSet); err != nil || got != "out" {
|
||||||
|
t.Fatalf("overlay stdout directory redirected file output: %q %v", got, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReportDirectorySelectionOnlyUsesActiveDestination(t *testing.T) {
|
func TestReportDirectorySelectionOnlyUsesActiveDestination(t *testing.T) {
|
||||||
cfg := config.Config{Reporting: config.Reporting{Enabled: true, StdoutDir: "reports"}}
|
cfg := config.Config{Reporting: config.Reporting{Enabled: true, StdoutDir: "reports"}}
|
||||||
if got, err := selectedReportDir(cfg, ""); err != nil || got != "reports" {
|
if got, err := selectedReportDir(cfg, "", false); err != nil || got != "reports" {
|
||||||
t.Fatalf("stdout destination: %q %v", got, err)
|
t.Fatalf("stdout destination: %q %v", got, err)
|
||||||
}
|
}
|
||||||
cfg.Reporting.StdoutDir = ""
|
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), false); err != nil || got != "out" {
|
||||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json")); err != nil || got != "out" {
|
|
||||||
t.Fatalf("file destination ignored stdout setting: %q %v", got, err)
|
t.Fatalf("file destination ignored stdout setting: %q %v", got, err)
|
||||||
}
|
}
|
||||||
if _, err := selectedReportDir(cfg, ""); err == nil {
|
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err != nil || got != "reports" {
|
||||||
|
t.Fatalf("explicit file report directory: %q %v", got, err)
|
||||||
|
}
|
||||||
|
cfg.Reporting.StdoutDir = ""
|
||||||
|
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), false); err != nil || got != "out" {
|
||||||
|
t.Fatalf("inactive blank configured report directory rejected for file output: %q %v", got, err)
|
||||||
|
}
|
||||||
|
if _, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err == nil {
|
||||||
|
t.Fatal("active explicit blank report directory accepted for file output")
|
||||||
|
}
|
||||||
|
if _, err := selectedReportDir(cfg, "", false); err == nil {
|
||||||
t.Fatal("enabled stdout reporting accepted blank destination")
|
t.Fatal("enabled stdout reporting accepted blank destination")
|
||||||
}
|
}
|
||||||
cfg.Reporting.Enabled = false
|
cfg.Reporting.Enabled = false
|
||||||
if got, err := selectedReportDir(cfg, ""); err != nil || got != "" {
|
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err != nil || got != "" {
|
||||||
t.Fatalf("disabled reporting probed destination: %q %v", got, err)
|
t.Fatalf("disabled reporting probed destination: %q %v", got, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -247,4 +267,14 @@ func TestBareBooleanFlagsAndReportDirectoryOverride(t *testing.T) {
|
|||||||
if values[ck.Keys("reporting", "enabled").String()] != true || values[ck.Keys("diagnostics", "debug").String()] != true || values[ck.Keys("reporting", "stdout_dir").String()] != "reports" {
|
if values[ck.Keys("reporting", "enabled").String()] != true || values[ck.Keys("diagnostics", "debug").String()] != true || values[ck.Keys("reporting", "stdout_dir").String()] != "reports" {
|
||||||
t.Fatalf("incorrect visited overrides: %+v", values)
|
t.Fatalf("incorrect visited overrides: %+v", values)
|
||||||
}
|
}
|
||||||
|
if !input.ReportDirSet {
|
||||||
|
t.Fatal("explicit --report-dir presence was not recorded")
|
||||||
|
}
|
||||||
|
withoutReportDir, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml"}, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if withoutReportDir.ReportDirSet {
|
||||||
|
t.Fatal("absent --report-dir was recorded as explicit")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
|||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
cfg := selected.Result.Config
|
cfg := selected.Result.Config
|
||||||
reportDir, err := selectedReportDir(cfg, input.OutputPath)
|
reportDir, err := selectedReportDir(cfg, input.OutputPath, input.ReportDirSet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
||||||
return 2
|
return 2
|
||||||
@@ -87,7 +87,7 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
|||||||
protected = append(protected, source.Path)
|
protected = append(protected, source.Path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
protected = append(protected, runtime.ProfilePaths...)
|
protected = append(protected, runtime.SourcePaths...)
|
||||||
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{InputTranscriptPath: input.TranscriptPath, TranscriptPath: input.OutputPath,
|
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{InputTranscriptPath: input.TranscriptPath, TranscriptPath: input.OutputPath,
|
||||||
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
|
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ func runConfigValidate(ctx context.Context, args []string, stdout, stderr io.Wri
|
|||||||
}
|
}
|
||||||
if _, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil); err != nil {
|
if _, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil); err != nil {
|
||||||
fmt.Fprintf(stderr, "audita config validate: %s\n", llm.RedactError(err, selected.Result.Config))
|
fmt.Fprintf(stderr, "audita config validate: %s\n", llm.RedactError(err, selected.Result.Config))
|
||||||
if llm.IsAssetError(err) {
|
if ctx.Err() != nil || llm.IsAssetError(err) {
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
return 2
|
return 2
|
||||||
@@ -124,7 +124,7 @@ func runConfigPrintEffective(ctx context.Context, args []string, stdout, stderr
|
|||||||
llmRuntime, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil)
|
llmRuntime, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "audita config print-effective: %s\n", llm.RedactError(err, selected.Result.Config))
|
fmt.Fprintf(stderr, "audita config print-effective: %s\n", llm.RedactError(err, selected.Result.Config))
|
||||||
if llm.IsAssetError(err) {
|
if ctx.Err() != nil || llm.IsAssetError(err) {
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
return 2
|
return 2
|
||||||
@@ -225,7 +225,7 @@ func writeProcessUsage(w io.Writer) {
|
|||||||
fmt.Fprintf(w, " --%s\n", name)
|
fmt.Fprintf(w, " --%s\n", name)
|
||||||
}
|
}
|
||||||
fmt.Fprintln(w, " --reporting[=true|false] Write the three-file report bundle (default true)")
|
fmt.Fprintln(w, " --reporting[=true|false] Write the three-file report bundle (default true)")
|
||||||
fmt.Fprintln(w, " --report-dir DIR Report directory for stdout output only; ignored with --output or --reporting=false")
|
fmt.Fprintln(w, " --report-dir DIR Select the report directory (stdout setting for stdout; output parent by default for files)")
|
||||||
fmt.Fprintln(w, " --debug[=true|false] Persist detailed diagnostics (default false)")
|
fmt.Fprintln(w, " --debug[=true|false] Persist detailed diagnostics (default false)")
|
||||||
fmt.Fprintln(w)
|
fmt.Fprintln(w)
|
||||||
fmt.Fprintln(w, "Example:")
|
fmt.Fprintln(w, "Example:")
|
||||||
|
|||||||
76
internal/cli/runtime_errors_test.go
Normal file
76
internal/cli/runtime_errors_test.go
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
profiles := filepath.Join(dir, "profiles")
|
||||||
|
if err := os.Mkdir(profiles, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main := filepath.Join(dir, "config.yml")
|
||||||
|
configText := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n"
|
||||||
|
if err := os.WriteFile(main, []byte(configText), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(profiles, "operator.yml"), []byte(operatorDocument), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return main
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
|
||||||
|
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
|
||||||
|
definitions := map[string]string{
|
||||||
|
"malformed operator YAML": "profiles:\n proposal: [\n",
|
||||||
|
"invalid operator template": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content: '{{'\n",
|
||||||
|
"missing operator message resource": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content_file: missing-instruction.md\n",
|
||||||
|
}
|
||||||
|
commands := []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
}{
|
||||||
|
{name: "validate", args: []string{"config", "validate"}},
|
||||||
|
{name: "print effective", args: []string{"config", "print-effective", "--view", "profiles"}},
|
||||||
|
}
|
||||||
|
for definition, document := range definitions {
|
||||||
|
for _, command := range commands {
|
||||||
|
t.Run(definition+"/"+command.name, func(t *testing.T) {
|
||||||
|
main := writeRuntimeErrorConfig(t, document)
|
||||||
|
args := append(append([]string(nil), command.args...), "--config", main)
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := Run(context.Background(), args, &stdout, &stderr); code != 2 {
|
||||||
|
t.Fatalf("exit=%d, stdout=%q, stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
|
||||||
|
t.Fatalf("configured secret leaked: %q", stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigCommandsKeepCallerCancellationAsRuntimeExit(t *testing.T) {
|
||||||
|
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
|
||||||
|
main := writeRuntimeErrorConfig(t, "profiles:\n proposal:\n backend: offline\n model: local-model\n")
|
||||||
|
commands := [][]string{
|
||||||
|
{"config", "validate", "--config", main},
|
||||||
|
{"config", "print-effective", "--config", main, "--view", "profiles"},
|
||||||
|
}
|
||||||
|
for _, args := range commands {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := Run(ctx, args, &stdout, &stderr); code != 1 {
|
||||||
|
t.Fatalf("args=%v exit=%d, stdout=%q, stderr=%q", args, code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -137,9 +137,167 @@ func TestSurvivingCredentials(t *testing.T) {
|
|||||||
if err != nil || loaded.Config.Backends["other"].Credential != nil {
|
if err != nil || loaded.Config.Backends["other"].Credential != nil {
|
||||||
t.Fatalf("null credential: %+v %v", loaded, err)
|
t.Fatalf("null credential: %+v %v", loaded, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inherited := `version: 2
|
||||||
|
backends:
|
||||||
|
parent:
|
||||||
|
endpoint: https://parent.invalid/v1
|
||||||
|
credential: BASE_SECRET
|
||||||
|
local:
|
||||||
|
extends: parent
|
||||||
|
credential: null
|
||||||
|
llm:
|
||||||
|
proposal_profile: proposal
|
||||||
|
`
|
||||||
|
if _, err := loadText(inherited, ""); err == nil {
|
||||||
|
t.Fatal("null child credential removed the unselected parent's surviving reference")
|
||||||
|
}
|
||||||
|
t.Setenv("BASE_SECRET", "synthetic-parent-secret")
|
||||||
|
loaded, err = loadText(strings.Replace(inherited, " credential: null\n", "", 1), "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := loaded.Config.Backends["local"].Credential.Value(); got != "synthetic-parent-secret" {
|
||||||
|
t.Fatalf("inherited secret: %q", got)
|
||||||
|
}
|
||||||
|
if got := loaded.Metadata().IsExplicit(ck.Keys("backends", "local", "credential")); !got {
|
||||||
|
t.Fatal("inherited credential is not explicit")
|
||||||
|
}
|
||||||
|
var inheritedEffective bytes.Buffer
|
||||||
|
if err := loaded.WriteEffective(&inheritedEffective); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(inheritedEffective.String(), "synthetic-parent-secret") {
|
||||||
|
t.Fatal("effective output leaked an inherited secret")
|
||||||
|
}
|
||||||
|
loaded, err = loadText(inherited, "")
|
||||||
|
if err != nil || loaded.Config.Backends["local"].Credential != nil {
|
||||||
|
t.Fatalf("child null credential: %+v %v", loaded, err)
|
||||||
|
}
|
||||||
|
if got := loaded.Config.Backends["parent"].Credential.Value(); got != "synthetic-parent-secret" {
|
||||||
|
t.Fatalf("parent credential after child null: %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
t.Setenv("AUDITA_MODEL", "legacy-model")
|
t.Setenv("AUDITA_MODEL", "legacy-model")
|
||||||
loaded, err = loadText(base, "")
|
loaded, err = loadText(base, "")
|
||||||
if err != nil || loaded.Config.LLM.ProposalProfile != "proposal" {
|
if err != nil || loaded.Config.LLM.ProposalProfile != "proposal" {
|
||||||
t.Fatalf("legacy environment affected loading: %+v %v", loaded, err)
|
t.Fatalf("legacy environment affected loading: %+v %v", loaded, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSiblingInheritancePreservesBackendSettings(t *testing.T) {
|
||||||
|
text := `version: 2
|
||||||
|
backends:
|
||||||
|
base:
|
||||||
|
endpoint: https://base.invalid/v1
|
||||||
|
concurrency: 3
|
||||||
|
queue_capacity: 0
|
||||||
|
local:
|
||||||
|
extends: base
|
||||||
|
llm:
|
||||||
|
proposal_profile: proposal
|
||||||
|
`
|
||||||
|
loaded, err := loadText(text, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backend := loaded.Config.Backends["local"]
|
||||||
|
if backend.Endpoint != "https://base.invalid/v1" || backend.Concurrency != 3 || backend.QueueCapacity != 0 {
|
||||||
|
t.Fatalf("inherited backend settings: %+v", backend)
|
||||||
|
}
|
||||||
|
for _, path := range []ck.Path{
|
||||||
|
ck.Keys("backends", "local", "endpoint"),
|
||||||
|
ck.Keys("backends", "local", "concurrency"),
|
||||||
|
ck.Keys("backends", "local", "queue_capacity"),
|
||||||
|
} {
|
||||||
|
if !loaded.Metadata().IsExplicit(path) {
|
||||||
|
t.Errorf("inherited field %s is not explicit", path.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
overridden, err := loadText(strings.Replace(text, " extends: base\n", " extends: base\n concurrency: 2\n", 1), "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := overridden.Config.Backends["local"].Concurrency; got != 2 {
|
||||||
|
t.Fatalf("child concurrency override: %d", got)
|
||||||
|
}
|
||||||
|
if got := overridden.Config.Backends["local"].QueueCapacity; got != 0 {
|
||||||
|
t.Fatalf("inherited zero queue capacity: %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSiblingInheritanceUsesSelectedOverlay(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main := filepath.Join(dir, "config.yml")
|
||||||
|
text := `version: 2
|
||||||
|
backends:
|
||||||
|
base:
|
||||||
|
endpoint: https://base.invalid/v1
|
||||||
|
concurrency: 3
|
||||||
|
queue_capacity: 0
|
||||||
|
local:
|
||||||
|
extends: base
|
||||||
|
concurrency: 2
|
||||||
|
llm:
|
||||||
|
proposal_profile: proposal
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("backends:\n base:\n endpoint: https://overlay.invalid/v1\n concurrency: 4\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, err := Load(ck.File(main), "testing")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
child := loaded.Config.Backends["local"]
|
||||||
|
if child.Endpoint != "https://overlay.invalid/v1" || child.Concurrency != 2 || child.QueueCapacity != 0 {
|
||||||
|
t.Fatalf("overlay inheritance and child override: %+v", child)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSiblingInheritanceDoesNotBypassBackendValidation(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
text string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "invalid inherited parent setting",
|
||||||
|
text: `version: 2
|
||||||
|
backends:
|
||||||
|
base:
|
||||||
|
endpoint: https://base.invalid/v1
|
||||||
|
concurrency: 0
|
||||||
|
local:
|
||||||
|
extends: base
|
||||||
|
llm:
|
||||||
|
proposal_profile: proposal
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid child override",
|
||||||
|
text: `version: 2
|
||||||
|
backends:
|
||||||
|
base:
|
||||||
|
endpoint: https://base.invalid/v1
|
||||||
|
concurrency: 3
|
||||||
|
local:
|
||||||
|
extends: base
|
||||||
|
concurrency: 0
|
||||||
|
llm:
|
||||||
|
proposal_profile: proposal
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if _, err := loadText(tc.text, ""); err == nil {
|
||||||
|
t.Fatal("invalid backend concurrency accepted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,9 +23,14 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
|
func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
|
||||||
|
t.Setenv("INHERITED_TEST_KEY", "synthetic-inherited-runtime-secret")
|
||||||
|
t.Setenv("OPERATOR_BACKEND_KEY", "synthetic-operator-backend-secret")
|
||||||
entered := make(chan struct{}, 3)
|
entered := make(chan struct{}, 3)
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if got := r.Header.Get("Authorization"); got != "Bearer synthetic-inherited-runtime-secret" {
|
||||||
|
t.Errorf("inherited credential header: %q", got)
|
||||||
|
}
|
||||||
entered <- struct{}{}
|
entered <- struct{}{}
|
||||||
<-release
|
<-release
|
||||||
content, _ := json.Marshal(`{"corrections":[]}`)
|
content, _ := json.Marshal(`{"corrections":[]}`)
|
||||||
@@ -37,16 +42,26 @@ func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
main := filepath.Join(dir, "config.yml")
|
main := filepath.Join(dir, "config.yml")
|
||||||
body := fmt.Sprintf("version: 2\nbackends:\n shared:\n endpoint: %s/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL)
|
body := fmt.Sprintf("version: 2\nbackends:\n shared-base:\n endpoint: %s/v1\n credential: INHERITED_TEST_KEY\n concurrency: 1\n queue_capacity: 0\n shared:\n extends: shared-base\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL)
|
||||||
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
|
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"proposal", "validation"} {
|
for _, name := range []string{"proposal", "validation"} {
|
||||||
profile := "id: " + name + "\nbackend: shared\nmodel: local\n"
|
profile := "profiles:\n " + name + ":\n backend: shared\n model: local\n"
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(profile), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(profile), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
operatorBackend := `backends:
|
||||||
|
shared:
|
||||||
|
endpoint: https://operator.invalid/v1
|
||||||
|
concurrency_limit: 77
|
||||||
|
queue_capacity: 88
|
||||||
|
api_key_env: OPERATOR_BACKEND_KEY
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "profiles", "backend-tuning.yaml"), []byte(operatorBackend), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
loaded, err := config.Load(ck.File(main), "")
|
loaded, err := config.Load(ck.File(main), "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -171,7 +186,7 @@ func TestDistinctBackendsHaveIndependentCapacity(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
for name, backend := range map[string]string{"proposal": "first", "validation": "second"} {
|
for name, backend := range map[string]string{"proposal": "first", "validation": "second"} {
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte("id: "+name+"\nbackend: "+backend+"\nmodel: local\n"), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte("profiles:\n "+name+":\n backend: "+backend+"\n model: local\n"), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,6 +30,22 @@ type reply struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) {
|
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) {
|
||||||
|
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{
|
||||||
|
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) {
|
||||||
|
documents := map[string]string{
|
||||||
|
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
|
||||||
|
}
|
||||||
|
if operatorPrompt != "" {
|
||||||
|
documents["operator-prompt.yaml"] = operatorPrompt
|
||||||
|
}
|
||||||
|
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents)
|
||||||
|
}
|
||||||
|
|
||||||
|
func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
var count atomic.Int32
|
var count atomic.Int32
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -65,13 +81,21 @@ func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook fu
|
|||||||
}))
|
}))
|
||||||
t.Cleanup(server.Close)
|
t.Cleanup(server.Close)
|
||||||
t.Setenv("ADAPTER_KEY", "adapter-secret")
|
t.Setenv("ADAPTER_KEY", "adapter-secret")
|
||||||
base := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\n credential: ADAPTER_KEY\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", server.URL, retries, repairs)
|
dir := t.TempDir()
|
||||||
dir := profileLayout(t, base, "")
|
profileDirectory := filepath.Join(dir, "profiles")
|
||||||
main := filepath.Join(dir, "config.yml")
|
if err := os.Mkdir(profileDirectory, 0700); err != nil {
|
||||||
profile := "id: base\nbackend: offline\nmodel: full-model\n"
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs)
|
||||||
|
main := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for name, content := range documents {
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
loaded, err := config.Load(ck.File(main), "")
|
loaded, err := config.Load(ck.File(main), "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -83,6 +107,10 @@ func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook fu
|
|||||||
return runtime, &count, dir
|
return runtime, &count, dir
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func operatorPromptDocument(system, user string) string {
|
||||||
|
return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user)
|
||||||
|
}
|
||||||
|
|
||||||
func adapterRequest(schema responseschema.Key) structuredoutput.Request {
|
func adapterRequest(schema responseschema.Key) structuredoutput.Request {
|
||||||
id := prompts.PromptIDModuleGlossaryProposal
|
id := prompts.PromptIDModuleGlossaryProposal
|
||||||
vars := map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}
|
vars := map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}
|
||||||
@@ -286,10 +314,238 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
|
|||||||
if !found {
|
if !found {
|
||||||
t.Fatal("hardening policy missing")
|
t.Fatal("hardening policy missing")
|
||||||
}
|
}
|
||||||
|
backend := runtime.Config.Backends["offline"]
|
||||||
|
backend.Endpoint = "http://127.0.0.1:1/v1"
|
||||||
|
runtime.Config.Backends["offline"] = backend
|
||||||
|
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 {
|
||||||
|
t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
system string
|
||||||
|
user string
|
||||||
|
replies []reply
|
||||||
|
repairs int
|
||||||
|
wantSystem string
|
||||||
|
wantUser string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "operator text without shared references",
|
||||||
|
system: "Operator system instructions.",
|
||||||
|
user: "Operator request without variables.",
|
||||||
|
replies: []reply{{200, `{"wrong":true}`}, {200, `{"corrections":[]}`}},
|
||||||
|
repairs: 1,
|
||||||
|
wantSystem: "Operator system instructions.",
|
||||||
|
wantUser: "Operator request without variables.",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "operator text with shared references",
|
||||||
|
system: "Policy: {{.HardeningPolicy}}",
|
||||||
|
user: "Section: {{.SectionJSON}}",
|
||||||
|
replies: []reply{{200, `{"corrections":[]}`}},
|
||||||
|
wantSystem: "Policy: " + prompts.HardeningText(),
|
||||||
|
wantUser: "Section: []",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var posted []struct {
|
||||||
|
Messages []structuredoutput.Message `json:"messages"`
|
||||||
|
}
|
||||||
|
runtime, count, _ := adapterFixtureWithPrompt(t, tc.replies, 0, tc.repairs, func(r *http.Request) {
|
||||||
|
var envelope struct {
|
||||||
|
Messages []structuredoutput.Message `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
posted = append(posted, envelope)
|
||||||
|
}, operatorPromptDocument(tc.system, tc.user))
|
||||||
|
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req := adapterRequest(responseschema.CorrectionSetKey)
|
||||||
|
if tc.name == "operator text with shared references" {
|
||||||
|
req.Variables["HardeningPolicy"] = "attacker-controlled"
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
response, err := client.CompleteStructured(context.Background(), req, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if int(count.Load()) != len(tc.replies) || response.Execution.StructuralRepairs != tc.repairs {
|
||||||
|
t.Fatalf("schema/repair execution = %d calls, %+v; want %d calls and %d repair", count.Load(), response.Execution, len(tc.replies), tc.repairs)
|
||||||
|
}
|
||||||
|
if response.Prepared == nil || len(posted) == 0 || len(posted[0].Messages) != len(response.Prepared.Messages) {
|
||||||
|
t.Fatal("prepared operator prompt was not dispatched")
|
||||||
|
}
|
||||||
|
for i := range response.Prepared.Messages {
|
||||||
|
if response.Prepared.Messages[i] != posted[0].Messages[i] {
|
||||||
|
t.Fatal("prepared operator messages differ from the first dispatched request")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(response.Prepared.Messages) != 2 || response.Prepared.Messages[0].Content != tc.wantSystem || response.Prepared.Messages[1].Content != tc.wantUser {
|
||||||
|
t.Fatalf("prepared messages = %+v", response.Prepared.Messages)
|
||||||
|
}
|
||||||
|
if tc.name == "operator text with shared references" {
|
||||||
|
if strings.Contains(response.Prepared.Messages[0].Content, "attacker-controlled") || !strings.Contains(response.Prepared.Messages[0].Content, prompts.HardeningText()) {
|
||||||
|
t.Fatal("caller variables replaced the reserved shared value")
|
||||||
|
}
|
||||||
|
} else if strings.Contains(response.Prepared.Messages[0].Content+response.Prepared.Messages[1].Content, prompts.HardeningText()) {
|
||||||
|
t.Fatal("shared text was injected into operator messages that omit its reference")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) {
|
||||||
|
t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used")
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
parentTemperature string
|
||||||
|
childTemperature string
|
||||||
|
wantTemperature bool
|
||||||
|
}{
|
||||||
|
{name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true},
|
||||||
|
{name: "omitted", wantTemperature: false},
|
||||||
|
{name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature +
|
||||||
|
" base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature +
|
||||||
|
" extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n"
|
||||||
|
operatorBackend := `backends:
|
||||||
|
openrouter:
|
||||||
|
endpoint: https://operator.invalid/v1
|
||||||
|
concurrency_limit: 91
|
||||||
|
queue_capacity: 92
|
||||||
|
api_key_env: CATALOG_TRAP_KEY
|
||||||
|
extra_params:
|
||||||
|
routing:
|
||||||
|
inherited: backend
|
||||||
|
overridden: backend
|
||||||
|
cleared: remove-me
|
||||||
|
`
|
||||||
|
documents := map[string]string{
|
||||||
|
"unrelated-profile-file.yml": profileDocument,
|
||||||
|
"provider-tuning.yaml": operatorBackend,
|
||||||
|
}
|
||||||
|
var wire map[string]json.RawMessage
|
||||||
|
runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
|
||||||
|
if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" {
|
||||||
|
t.Errorf("application credential was not authoritative: %q", got)
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&wire); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}, "openrouter", documents)
|
||||||
|
profile := runtime.Profiles["proposal"]
|
||||||
|
if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 {
|
||||||
|
t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if count.Load() != 1 {
|
||||||
|
t.Fatalf("provider calls = %d, want 1", count.Load())
|
||||||
|
}
|
||||||
|
temperature, hasTemperature := wire["temperature"]
|
||||||
|
if hasTemperature != tc.wantTemperature {
|
||||||
|
t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire)
|
||||||
|
}
|
||||||
|
if tc.wantTemperature {
|
||||||
|
var value float64
|
||||||
|
if err := json.Unmarshal(temperature, &value); err != nil || value != 0 {
|
||||||
|
t.Fatalf("explicit zero temperature = %s (%v)", temperature, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var routing map[string]any
|
||||||
|
if err := json.Unmarshal(wire["routing"], &routing); err != nil {
|
||||||
|
t.Fatalf("provider routing fields missing: %s (%v)", wire, err)
|
||||||
|
}
|
||||||
|
if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" {
|
||||||
|
t.Fatalf("recursively overlaid provider fields = %#v", routing)
|
||||||
|
}
|
||||||
|
if value, exists := routing["cleared"]; !exists || value != nil {
|
||||||
|
t.Fatalf("null did not replace inherited provider field: %#v", routing)
|
||||||
|
}
|
||||||
|
if response.Model != "full-model" || response.Prepared == nil {
|
||||||
|
t.Fatalf("execution did not use selected profile: %+v", response)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) {
|
||||||
|
var posted []struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []structuredoutput.Message `json:"messages"`
|
||||||
|
}
|
||||||
|
runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
|
||||||
|
var envelope struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []structuredoutput.Message `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
posted = append(posted, envelope)
|
||||||
|
}, operatorPromptDocument("Initial operator instructions.", "Initial payload."))
|
||||||
|
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profile := "profiles:\n base:\n backend: offline\n model: edited-model\n"
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.")
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out map[string]any
|
||||||
|
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" {
|
||||||
|
t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
main := filepath.Join(dir, "config.yml")
|
||||||
|
newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" {
|
||||||
|
t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
|
func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
|
||||||
runtime, count, dir := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
|
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
|
||||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -300,15 +556,6 @@ func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
|
|||||||
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || count.Load() != 0 {
|
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || count.Load() != 0 {
|
||||||
t.Fatalf("preparation failure dispatched: %v, %d", err, count.Load())
|
t.Fatalf("preparation failure dispatched: %v, %d", err, count.Load())
|
||||||
}
|
}
|
||||||
// PrepareExecution reloads the profile, so a changed target must be blocked.
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nendpoint: http://127.0.0.1:1/v1\nmodel: full-model\n"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req = adapterRequest(responseschema.CorrectionSetKey)
|
|
||||||
_, err = client.CompleteStructured(context.Background(), req, &out)
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 0 {
|
|
||||||
t.Fatalf("target drift dispatched: %v, %d", err, count.Load())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
|
func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
|
||||||
@@ -333,7 +580,9 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
|
|||||||
|
|
||||||
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
|
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
|
||||||
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
|
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
|
||||||
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
|
runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{
|
||||||
|
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n",
|
||||||
|
})
|
||||||
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
|
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
|
||||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -6,13 +6,11 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
|
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/redaction"
|
"gitea.maximumdirect.net/eric/audita/internal/core/redaction"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||||
@@ -26,8 +24,8 @@ type Runtime struct {
|
|||||||
Config config.Config
|
Config config.Config
|
||||||
Engine *pk.Engine
|
Engine *pk.Engine
|
||||||
Profiles map[string]pk.ProfileInspection
|
Profiles map[string]pk.ProfileInspection
|
||||||
// ProfilePaths contains filesystem sources actually read by PromptKit.
|
// SourcePaths contains host configuration and message files captured by PromptKit.
|
||||||
ProfilePaths []string
|
SourcePaths []string
|
||||||
nextCallID atomic.Uint64
|
nextCallID atomic.Uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,32 +37,63 @@ func (e *AssetError) Error() string { return e.Err.Error() }
|
|||||||
func (e *AssetError) Unwrap() error { return e.Err }
|
func (e *AssetError) Unwrap() error { return e.Err }
|
||||||
|
|
||||||
func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) {
|
func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) {
|
||||||
|
return newRuntimeWithPromptFS(ctx, c, mainPath, client, prompts.Filesystem())
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRuntimeWithPromptFS(ctx context.Context, c config.Config, mainPath string, client *http.Client, promptFS fs.FS) (*Runtime, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
profileDir := c.ProfilesDir
|
profileDir := c.ProfilesDir
|
||||||
if !filepath.IsAbs(profileDir) {
|
if !filepath.IsAbs(profileDir) {
|
||||||
profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
|
profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
|
||||||
}
|
}
|
||||||
profileFS := &profileSourceFS{FS: os.DirFS(profileDir), paths: make(map[string]struct{})}
|
promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot)
|
||||||
options := []pk.Option{
|
|
||||||
pk.WithPromptFS(prompts.Filesystem(), prompts.AssetRoot),
|
|
||||||
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
|
|
||||||
pk.WithProfileFS(profileFS, "."),
|
|
||||||
}
|
|
||||||
ids := make([]string, 0, len(c.Backends))
|
|
||||||
for id := range c.Backends {
|
|
||||||
ids = append(ids, id)
|
|
||||||
}
|
|
||||||
sort.Strings(ids)
|
|
||||||
for _, id := range ids {
|
|
||||||
backend := c.Backends[id]
|
|
||||||
options = append(options, pk.WithBackend(pk.Backend{ID: id, Endpoint: backend.Endpoint, ConcurrencyLimit: backend.Concurrency, QueueCapacity: &backend.QueueCapacity}))
|
|
||||||
}
|
|
||||||
engine, err := pk.NewEngine(pk.Config{HTTPClient: client, TransientRetries: &c.LLM.TransientRetries}, options...)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("assemble LLM runtime: %w", err)
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return nil, ctxErr
|
||||||
|
}
|
||||||
|
return nil, &AssetError{Err: fmt.Errorf("discover embedded prompts: %w", err)}
|
||||||
|
}
|
||||||
|
profileInputs, err := pk.DirectoryInputs(profileDir)
|
||||||
|
if err != nil {
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return nil, ctxErr
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("discover operator definitions: %w", err)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
applicationBackends := make(map[string]any, len(c.Backends))
|
||||||
|
for id := range c.Backends {
|
||||||
|
backend := c.Backends[id]
|
||||||
|
applicationBackends[id] = map[string]any{
|
||||||
|
"endpoint": backend.Endpoint,
|
||||||
|
"concurrency_limit": backend.Concurrency,
|
||||||
|
"queue_capacity": backend.QueueCapacity,
|
||||||
|
"api_key_env": nil,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sources := pk.ConfigurationSources{
|
||||||
|
Underlays: []ck.Underlay{
|
||||||
|
{Inputs: promptInputs},
|
||||||
|
{Inputs: profileInputs},
|
||||||
|
},
|
||||||
|
Main: &ck.Input{Source: ck.Mapping("application backends", map[string]any{"backends": applicationBackends})},
|
||||||
|
}
|
||||||
|
engine, err := pk.NewEngine(pk.Config{
|
||||||
|
Sources: sources,
|
||||||
|
HTTPClient: client,
|
||||||
|
TransientRetries: &c.LLM.TransientRetries,
|
||||||
|
}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
|
||||||
|
if err != nil {
|
||||||
|
return nil, embeddedAssetOrInputError(ctx, promptInputs, fmt.Errorf("assemble LLM runtime: %w", err))
|
||||||
}
|
}
|
||||||
for _, meta := range prompts.RegisteredMetadata() {
|
for _, meta := range prompts.RegisteredMetadata() {
|
||||||
if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil {
|
if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil {
|
||||||
return nil, &AssetError{Err: fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, err)}
|
cause := fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err)
|
||||||
|
return nil, embeddedAssetOrInputError(ctx, promptInputs, cause)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)}
|
runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)}
|
||||||
@@ -78,6 +107,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
|
|||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("%s profile must select a registered application backend", selection.role)
|
return nil, fmt.Errorf("%s profile must select a registered application backend", selection.role)
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(target.Model) == "" {
|
||||||
|
return nil, fmt.Errorf("%s profile must select a nonblank model", selection.role)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(target.Endpoint) == "" {
|
||||||
|
return nil, fmt.Errorf("%s profile must resolve to a nonblank endpoint", selection.role)
|
||||||
|
}
|
||||||
endpoint := func(value string) string { return strings.TrimRight(strings.TrimSpace(value), "/") }
|
endpoint := func(value string) string { return strings.TrimRight(strings.TrimSpace(value), "/") }
|
||||||
if endpoint(target.Endpoint) != endpoint(backend.Endpoint) {
|
if endpoint(target.Endpoint) != endpoint(backend.Endpoint) {
|
||||||
return nil, fmt.Errorf("%s profile endpoint must match its registered backend", selection.role)
|
return nil, fmt.Errorf("%s profile endpoint must match its registered backend", selection.role)
|
||||||
@@ -90,15 +125,39 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
|
|||||||
}
|
}
|
||||||
runtime.Profiles[selection.role] = *profile
|
runtime.Profiles[selection.role] = *profile
|
||||||
}
|
}
|
||||||
profileFS.mu.Lock()
|
runtime.SourcePaths, err = capturedSourcePaths(engine)
|
||||||
for name := range profileFS.paths {
|
if err != nil {
|
||||||
runtime.ProfilePaths = append(runtime.ProfilePaths, filepath.Join(profileDir, filepath.FromSlash(name)))
|
return nil, fmt.Errorf("inspect captured LLM sources: %w", err)
|
||||||
}
|
}
|
||||||
profileFS.mu.Unlock()
|
|
||||||
sort.Strings(runtime.ProfilePaths)
|
|
||||||
return runtime, nil
|
return runtime, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cause error) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
embeddedSources := pk.ConfigurationSources{
|
||||||
|
Underlays: []ck.Underlay{{Inputs: promptInputs}},
|
||||||
|
}
|
||||||
|
diagnosticEngine, err := pk.NewEngine(pk.Config{Sources: embeddedSources},
|
||||||
|
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
|
||||||
|
if err == nil {
|
||||||
|
for _, meta := range prompts.RegisteredMetadata() {
|
||||||
|
if _, inspectErr := diagnosticEngine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); inspectErr != nil {
|
||||||
|
err = fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, inspectErr)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return ctxErr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return &AssetError{Err: fmt.Errorf("validate embedded prompt assets: %w", err)}
|
||||||
|
}
|
||||||
|
return cause
|
||||||
|
}
|
||||||
|
|
||||||
func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) }
|
func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) }
|
||||||
|
|
||||||
// RedactError replaces every configured resolved backend secret before a
|
// RedactError replaces every configured resolved backend secret before a
|
||||||
@@ -109,21 +168,3 @@ func RedactError(err error, c config.Config) string {
|
|||||||
}
|
}
|
||||||
return redaction.New(ConfiguredSecrets(&c)).String(err.Error())
|
return redaction.New(ConfiguredSecrets(&c)).String(err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record successful profile reads at the library boundary, including inherited
|
|
||||||
// and differently named sources, without duplicating PromptKit's discovery rules.
|
|
||||||
type profileSourceFS struct {
|
|
||||||
fs.FS
|
|
||||||
mu sync.Mutex
|
|
||||||
paths map[string]struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *profileSourceFS) ReadFile(name string) ([]byte, error) {
|
|
||||||
data, err := fs.ReadFile(f.FS, name)
|
|
||||||
if err == nil {
|
|
||||||
f.mu.Lock()
|
|
||||||
f.paths[name] = struct{}{}
|
|
||||||
f.mu.Unlock()
|
|
||||||
}
|
|
||||||
return data, err
|
|
||||||
}
|
|
||||||
|
|||||||
257
internal/framework/llm/runtime_assembly_test.go
Normal file
257
internal/framework/llm/runtime_assembly_test.go
Normal file
@@ -0,0 +1,257 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
profileDir := filepath.Join(dir, "profiles")
|
||||||
|
if err := os.Mkdir(profileDir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main := filepath.Join(dir, "config.yml")
|
||||||
|
text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections
|
||||||
|
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for name, content := range documents {
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loaded, err := config.Load(ck.File(main), "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return loaded.Config, main
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
|
||||||
|
t.Helper()
|
||||||
|
source := prompts.Filesystem()
|
||||||
|
files := make(fstest.MapFS)
|
||||||
|
err := fs.WalkDir(source, prompts.AssetRoot, func(name string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil || entry.IsDir() {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err := fs.ReadFile(source, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
files[name] = &fstest.MapFile{Data: data}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return files
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) {
|
||||||
|
validProposal := " backend: offline\n model: proposal-model\n"
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
selections string
|
||||||
|
profiles string
|
||||||
|
role string
|
||||||
|
field string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "omitted proposal model with derived validation selection",
|
||||||
|
selections: " proposal_profile: proposal\n",
|
||||||
|
profiles: "profiles:\n proposal:\n backend: offline\n",
|
||||||
|
role: "proposal",
|
||||||
|
field: "model",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty validation model",
|
||||||
|
selections: " proposal_profile: proposal\n validation_profile: validation\n",
|
||||||
|
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n",
|
||||||
|
role: "validation",
|
||||||
|
field: "model",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "null proposal model",
|
||||||
|
selections: " proposal_profile: proposal\n",
|
||||||
|
profiles: "profiles:\n proposal:\n backend: offline\n model: null\n",
|
||||||
|
role: "proposal",
|
||||||
|
field: "model",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cleared validation endpoint",
|
||||||
|
selections: " proposal_profile: proposal\n validation_profile: validation\n",
|
||||||
|
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n",
|
||||||
|
role: "validation",
|
||||||
|
field: "endpoint",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles})
|
||||||
|
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) {
|
||||||
|
t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
|
||||||
|
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||||
|
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n",
|
||||||
|
})
|
||||||
|
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" {
|
||||||
|
t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeProfileInheritancePreservesControlsAndMergesProviderValues(t *testing.T) {
|
||||||
|
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: \"child profile\"\n", map[string]string{
|
||||||
|
"unrelated-name.yaml": `profiles:
|
||||||
|
"parent~/shared":
|
||||||
|
backend: offline
|
||||||
|
temperature: 0.4
|
||||||
|
top_p: 0.7
|
||||||
|
extra_params:
|
||||||
|
routing:
|
||||||
|
retained: from-parent
|
||||||
|
overridden: parent
|
||||||
|
cleared: remove-me
|
||||||
|
"child profile":
|
||||||
|
extends: "parent~/shared"
|
||||||
|
model: child-model
|
||||||
|
extra_params:
|
||||||
|
routing:
|
||||||
|
overridden: child
|
||||||
|
child_only: present
|
||||||
|
cleared: null
|
||||||
|
`,
|
||||||
|
})
|
||||||
|
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profile := runtime.Profiles["proposal"]
|
||||||
|
target := profile.EffectiveModelParams
|
||||||
|
if profile.ProfileID != "child profile" || target.Model != "child-model" || target.Temperature != 0.4 || target.TopP != 0.7 {
|
||||||
|
t.Fatalf("inherited profile target = %+v", target)
|
||||||
|
}
|
||||||
|
routing, ok := target.ExtraParams["routing"].(map[string]any)
|
||||||
|
if !ok || routing["retained"] != "from-parent" || routing["overridden"] != "child" || routing["child_only"] != "present" {
|
||||||
|
t.Fatalf("recursively inherited provider values = %#v", target.ExtraParams)
|
||||||
|
}
|
||||||
|
if value, exists := routing["cleared"]; !exists || value != nil {
|
||||||
|
t.Fatalf("null did not replace the inherited provider value: %#v", routing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
|
||||||
|
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||||
|
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
|
||||||
|
})
|
||||||
|
if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||||
|
if err == nil || IsAssetError(err) {
|
||||||
|
t.Fatalf("missing operator profile directory classification: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
document map[string]string
|
||||||
|
selection string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "malformed unselected definition",
|
||||||
|
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"},
|
||||||
|
selection: " proposal_profile: proposal\n",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "undefined selected backend",
|
||||||
|
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"},
|
||||||
|
selection: " proposal_profile: proposal\n",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "conflicting directory contributions",
|
||||||
|
document: map[string]string{
|
||||||
|
"a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n",
|
||||||
|
"b.yml": "profiles:\n proposal:\n model: second-model\n",
|
||||||
|
},
|
||||||
|
selection: " proposal_profile: proposal\n",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document)
|
||||||
|
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||||
|
if err == nil || IsAssetError(err) {
|
||||||
|
t.Fatalf("operator definition classification: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
|
||||||
|
_, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{})
|
||||||
|
if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
_, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{})
|
||||||
|
if !errors.Is(err, context.Canceled) || IsAssetError(err) {
|
||||||
|
t.Fatalf("caller cancellation classification: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
|
||||||
|
promptFS := copyEmbeddedPromptFilesystem(t)
|
||||||
|
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
|
||||||
|
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
|
||||||
|
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||||
|
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
|
||||||
|
})
|
||||||
|
_, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
|
||||||
|
if err == nil || !IsAssetError(err) {
|
||||||
|
t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
|
||||||
|
promptFS := copyEmbeddedPromptFilesystem(t)
|
||||||
|
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
|
||||||
|
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
|
||||||
|
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||||
|
"operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n",
|
||||||
|
})
|
||||||
|
runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err)
|
||||||
|
}
|
||||||
|
if runtime.Engine == nil {
|
||||||
|
t.Fatal("runtime engine was not retained")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
|
|
||||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
type rejectingTransport struct{ t *testing.T }
|
type rejectingTransport struct{ t *testing.T }
|
||||||
@@ -29,8 +30,8 @@ func profileLayout(t *testing.T, base, overlay string) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for name, content := range map[string]string{
|
for name, content := range map[string]string{
|
||||||
"base": "id: base\nbackend: offline\nmodel: full-model\ntemperature: 0.3\nmax_tokens: 500\ntop_p: 0.8\ntimeout_seconds: 12\nservice_tier: priority\nreasoning_effort: medium\n",
|
"base": "profiles:\n base:\n backend: offline\n model: full-model\n temperature: 0.3\n max_tokens: 500\n top_p: 0.8\n timeout_seconds: 12\n service_tier: priority\n reasoning_effort: medium\n",
|
||||||
"testing": "id: testing\nbackend: offline\nmodel: economy-model\n",
|
"testing": "profiles:\n testing:\n backend: offline\n model: economy-model\n",
|
||||||
} {
|
} {
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -112,14 +113,111 @@ func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSafeViewsOmitProviderAndPromptValues(t *testing.T) {
|
||||||
|
const providerValue = "private-provider-value"
|
||||||
|
const promptValue = "private-operator-prompt-text"
|
||||||
|
runtime, _, dir := adapterFixtureWithDocuments(t, nil, 0, 0, nil, "offline", map[string]string{
|
||||||
|
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n extra_params:\n vendor_extension: " + providerValue + "\n",
|
||||||
|
"operator-prompt.yaml": operatorPromptDocument(promptValue, promptValue),
|
||||||
|
})
|
||||||
|
profiles, err := runtime.SelectedProfilesYAML()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, err := config.Load(ck.File(filepath.Join(dir, "config.yml")), "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
explain, err := ExplanationYAML(loaded)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
combined := string(profiles) + string(explain)
|
||||||
|
for _, value := range []string{providerValue, promptValue, "adapter-secret", "127.0.0.1"} {
|
||||||
|
if strings.Contains(combined, value) {
|
||||||
|
t.Fatalf("safe views exposed %q: %s", value, combined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExplanationYAMLIncludesSafeInheritanceEvidence(t *testing.T) {
|
||||||
|
t.Setenv("SYNTHETIC_KEY", "synthetic-inherited-secret-never-print")
|
||||||
|
text := `version: 2
|
||||||
|
backends:
|
||||||
|
shared:
|
||||||
|
endpoint: https://shared.invalid/v1
|
||||||
|
concurrency: 3
|
||||||
|
queue_capacity: 0
|
||||||
|
credential: SYNTHETIC_KEY
|
||||||
|
offline:
|
||||||
|
extends: shared
|
||||||
|
concurrency: 1
|
||||||
|
llm:
|
||||||
|
proposal_profile: base
|
||||||
|
`
|
||||||
|
loaded, err := config.Load(ck.Bytes("config.yml", []byte(text)), "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
explanation, err := ExplanationYAML(loaded)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(explanation), "synthetic-inherited-secret-never-print") {
|
||||||
|
t.Fatal("explanation exposed a resolved inherited credential")
|
||||||
|
}
|
||||||
|
var records []struct {
|
||||||
|
Path string `yaml:"path"`
|
||||||
|
Inheritance []struct {
|
||||||
|
From string `yaml:"from"`
|
||||||
|
To string `yaml:"to"`
|
||||||
|
Reference string `yaml:"reference"`
|
||||||
|
} `yaml:"inheritance"`
|
||||||
|
}
|
||||||
|
if err := yaml.Unmarshal(explanation, &records); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var endpoint, credential, concurrency *struct {
|
||||||
|
Path string `yaml:"path"`
|
||||||
|
Inheritance []struct {
|
||||||
|
From string `yaml:"from"`
|
||||||
|
To string `yaml:"to"`
|
||||||
|
Reference string `yaml:"reference"`
|
||||||
|
} `yaml:"inheritance"`
|
||||||
|
}
|
||||||
|
for i := range records {
|
||||||
|
switch records[i].Path {
|
||||||
|
case "/backends/offline/endpoint":
|
||||||
|
endpoint = &records[i]
|
||||||
|
case "/backends/offline/credential":
|
||||||
|
credential = &records[i]
|
||||||
|
case "/backends/offline/concurrency":
|
||||||
|
concurrency = &records[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if endpoint == nil || len(endpoint.Inheritance) != 1 {
|
||||||
|
t.Fatalf("inherited endpoint evidence missing: %s", explanation)
|
||||||
|
}
|
||||||
|
step := endpoint.Inheritance[0]
|
||||||
|
if step.From != "/backends/shared/endpoint" || step.To != "/backends/offline/endpoint" || step.Reference != "/backends/offline/extends" {
|
||||||
|
t.Fatalf("inherited endpoint evidence: %+v", step)
|
||||||
|
}
|
||||||
|
if credential == nil || len(credential.Inheritance) != 1 {
|
||||||
|
t.Fatalf("inherited credential evidence missing: %s", explanation)
|
||||||
|
}
|
||||||
|
if concurrency == nil || len(concurrency.Inheritance) != 0 {
|
||||||
|
t.Fatalf("direct child override reported parent inheritance: %+v", concurrency)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) {
|
func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) {
|
||||||
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
|
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
|
||||||
dir := profileLayout(t, profileBase, "")
|
dir := profileLayout(t, profileBase, "")
|
||||||
main := filepath.Join(dir, "config.yml")
|
main := filepath.Join(dir, "config.yml")
|
||||||
for _, tc := range []struct{ name, profile string }{
|
for _, tc := range []struct{ name, profile string }{
|
||||||
{"endpoint only", "id: custom\nendpoint: https://offline.invalid/v1\nmodel: model\n"},
|
{"endpoint only", "profiles:\n custom:\n endpoint: https://offline.invalid/v1\n model: model\n"},
|
||||||
{"mismatched endpoint", "id: custom\nbackend: offline\nendpoint: https://elsewhere.invalid/v1\nmodel: model\n"},
|
{"mismatched endpoint", "profiles:\n custom:\n backend: offline\n endpoint: https://elsewhere.invalid/v1\n model: model\n"},
|
||||||
{"profile credential", "id: custom\nbackend: offline\nmodel: model\napi_key_env: OTHER_KEY\n"},
|
{"profile credential", "profiles:\n custom:\n backend: offline\n model: model\n api_key_env: OTHER_KEY\n"},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil {
|
||||||
|
|||||||
@@ -61,11 +61,17 @@ type derivationView struct {
|
|||||||
Reason string `yaml:"reason"`
|
Reason string `yaml:"reason"`
|
||||||
Dependencies []string `yaml:"dependencies,omitempty"`
|
Dependencies []string `yaml:"dependencies,omitempty"`
|
||||||
}
|
}
|
||||||
|
type inheritanceStepView struct {
|
||||||
|
From string `yaml:"from"`
|
||||||
|
To string `yaml:"to"`
|
||||||
|
Reference string `yaml:"reference"`
|
||||||
|
}
|
||||||
type explanationView struct {
|
type explanationView struct {
|
||||||
Path string `yaml:"path"`
|
Path string `yaml:"path"`
|
||||||
Source string `yaml:"source"`
|
Source string `yaml:"source"`
|
||||||
Layer ck.Layer `yaml:"layer"`
|
Layer ck.Layer `yaml:"layer"`
|
||||||
Derivations []derivationView `yaml:"derivations,omitempty"`
|
Derivations []derivationView `yaml:"derivations,omitempty"`
|
||||||
|
Inheritance []inheritanceStepView `yaml:"inheritance,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExplanationYAML reports only origins and recorded resolution operations.
|
// ExplanationYAML reports only origins and recorded resolution operations.
|
||||||
@@ -93,6 +99,11 @@ func ExplanationYAML(result *ck.Result[config.Config]) ([]byte, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
record := explanationView{Path: path.String(), Layer: info.Origin.Layer, Source: string(info.Origin.Layer)}
|
record := explanationView{Path: path.String(), Layer: info.Origin.Layer, Source: string(info.Origin.Layer)}
|
||||||
|
for _, step := range info.Origin.Inheritance {
|
||||||
|
record.Inheritance = append(record.Inheritance, inheritanceStepView{
|
||||||
|
From: step.From.String(), To: step.To.String(), Reference: step.Reference.String(),
|
||||||
|
})
|
||||||
|
}
|
||||||
if len(info.Origin.Locations) > 0 {
|
if len(info.Origin.Locations) > 0 {
|
||||||
if label := sources[info.Origin.Locations[0].SourceID]; label != "" {
|
if label := sources[info.Origin.Locations[0].SourceID]; label != "" {
|
||||||
record.Source = label
|
record.Source = label
|
||||||
|
|||||||
65
internal/framework/llm/source_paths.go
Normal file
65
internal/framework/llm/source_paths.go
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func capturedSourcePaths(engine *pk.Engine) ([]string, error) {
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("inspect configuration: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sources := inspection.Sources()
|
||||||
|
byID := make(map[string]ck.SourceInfo, len(sources))
|
||||||
|
paths := make(map[string]struct{}, len(sources))
|
||||||
|
for _, source := range sources {
|
||||||
|
byID[source.ID] = source
|
||||||
|
if source.Kind == ck.SourceKindFile && source.Path != "" {
|
||||||
|
paths[source.Path] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata := inspection.Metadata()
|
||||||
|
for _, path := range metadata.HistoricalPaths() {
|
||||||
|
segments := strings.Split(strings.TrimPrefix(path.String(), "/"), "/")
|
||||||
|
if len(segments) != 6 || segments[0] != "prompts" || segments[3] != "messages" || segments[5] != "content_file" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if messageIndex, err := strconv.Atoi(segments[4]); err != nil || messageIndex < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
field, ok := metadata.Lookup(path)
|
||||||
|
if !ok || field.Null || len(field.Origin.Locations) != 1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
source, ok := byID[field.Origin.Locations[0].SourceID]
|
||||||
|
if !ok || source.Kind != ck.SourceKindFile || source.Path == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
explanation, ok := inspection.Explain(path)
|
||||||
|
if !ok || !explanation.Effective.Present || explanation.Effective.Redacted {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var resource string
|
||||||
|
if err := yaml.Unmarshal([]byte(explanation.Effective.YAML), &resource); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode captured message resource at %s: %w", path.String(), err)
|
||||||
|
}
|
||||||
|
paths[filepath.Join(filepath.Dir(source.Path), filepath.FromSlash(resource))] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
result := make([]string, 0, len(paths))
|
||||||
|
for path := range paths {
|
||||||
|
result = append(result, path)
|
||||||
|
}
|
||||||
|
sort.Strings(result)
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
61
internal/framework/llm/source_paths_test.go
Normal file
61
internal/framework/llm/source_paths_test.go
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRuntimeTracksCapturedConfigurationAndInheritedMessageFiles(t *testing.T) {
|
||||||
|
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n validation_profile: validation\n", map[string]string{
|
||||||
|
"selected.yml": "profiles:\n proposal:\n backend: offline\n model: proposal\n validation:\n backend: offline\n model: validation\n",
|
||||||
|
})
|
||||||
|
profiles := filepath.Join(filepath.Dir(main), cfg.ProfilesDir)
|
||||||
|
parentDir := filepath.Join(profiles, "catalog")
|
||||||
|
childDir := filepath.Join(profiles, "nested", "child")
|
||||||
|
for _, directory := range []string{parentDir, childDir} {
|
||||||
|
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parent := filepath.Join(parentDir, "arbitrary-name.yaml")
|
||||||
|
child := filepath.Join(childDir, "override.yaml")
|
||||||
|
resource := filepath.Join(parentDir, "instruction.md")
|
||||||
|
parentDocument := "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./instruction.md\n output:\n format: text\n validation_mode: basic\n"
|
||||||
|
childDocument := "prompts:\n literal/~child:\n extends: source\n"
|
||||||
|
for path, content := range map[string]string{
|
||||||
|
parent: parentDocument,
|
||||||
|
child: childDocument,
|
||||||
|
resource: "Captured operator message.\n",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []string{
|
||||||
|
filepath.Join(profiles, "nested", "child", "override.yaml"),
|
||||||
|
filepath.Join(profiles, "catalog", "arbitrary-name.yaml"),
|
||||||
|
filepath.Join(profiles, "selected.yml"),
|
||||||
|
resource,
|
||||||
|
}
|
||||||
|
// Runtime paths are canonical paths from PromptKit's captured sources.
|
||||||
|
for i := range want {
|
||||||
|
want[i], err = filepath.EvalSymlinks(want[i])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(want)
|
||||||
|
if !reflect.DeepEqual(runtime.SourcePaths, want) {
|
||||||
|
t.Fatalf("captured source paths = %#v, want %#v", runtime.SourcePaths, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -41,7 +41,7 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
|
|||||||
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
|
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nmodel: local-model\n"), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
loaded, err := config.Load(ck.File(main), "")
|
loaded, err := config.Load(ck.File(main), "")
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
|
|||||||
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
|
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles/base.yaml"), []byte("id: base\nbackend: shared\nmodel: local\ntimeout_seconds: 30\n"), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "profiles/base.yaml"), []byte("profiles:\n base:\n backend: shared\n model: local\n timeout_seconds: 30\n"), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
loaded, err := config.Load(ck.File(main), "")
|
loaded, err := config.Load(ck.File(main), "")
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
|||||||
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
|
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nmodel: local-model\n"), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
loaded, err := config.Load(ck.File(main), "")
|
loaded, err := config.Load(ck.File(main), "")
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
id: modules.glossary.proposal
|
prompts:
|
||||||
version: v1
|
modules.glossary.proposal:
|
||||||
|
v1:
|
||||||
messages:
|
messages:
|
||||||
- role: system
|
- role: system
|
||||||
content: |
|
content: |
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
id: modules.grammar.proposal
|
prompts:
|
||||||
version: v1
|
modules.grammar.proposal:
|
||||||
|
v1:
|
||||||
messages:
|
messages:
|
||||||
- role: system
|
- role: system
|
||||||
content: |
|
content: |
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
id: modules.homophones.proposal
|
prompts:
|
||||||
version: v1
|
modules.homophones.proposal:
|
||||||
|
v1:
|
||||||
messages:
|
messages:
|
||||||
- role: system
|
- role: system
|
||||||
content: |
|
content: |
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
id: modules.spoken_word.proposal
|
prompts:
|
||||||
version: v1
|
modules.spoken_word.proposal:
|
||||||
|
v1:
|
||||||
messages:
|
messages:
|
||||||
- role: system
|
- role: system
|
||||||
content: |
|
content: |
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
id: validators.editorial_review
|
prompts:
|
||||||
version: v1
|
validators.editorial_review:
|
||||||
|
v1:
|
||||||
messages:
|
messages:
|
||||||
- role: system
|
- role: system
|
||||||
content: |
|
content: |
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
id: validators.meaning_reversal_review
|
prompts:
|
||||||
version: v1
|
validators.meaning_reversal_review:
|
||||||
|
v1:
|
||||||
messages:
|
messages:
|
||||||
- role: system
|
- role: system
|
||||||
content: |
|
content: |
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
id: validators.spoken_form_plausibility
|
prompts:
|
||||||
version: v1
|
validators.spoken_form_plausibility:
|
||||||
|
v1:
|
||||||
messages:
|
messages:
|
||||||
- role: system
|
- role: system
|
||||||
content: |
|
content: |
|
||||||
|
|||||||
@@ -5,15 +5,28 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
|
func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
|
||||||
engine, err := pk.NewEngine(pk.Config{},
|
promptInputs, err := pk.FilesystemInputs("embedded prompts", Filesystem(), AssetRoot)
|
||||||
pk.WithPromptFS(Filesystem(), AssetRoot),
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
endpoint := "https://example.invalid/v1"
|
||||||
|
model := "fixture-model"
|
||||||
|
sources := pk.ConfigurationSources{
|
||||||
|
Underlays: []ck.Underlay{
|
||||||
|
{Inputs: promptInputs},
|
||||||
|
{Inputs: []ck.Input{{Source: pk.ProfileSource("registry fixture profile", map[string]pk.ProfileConfig{
|
||||||
|
"fixture": {Endpoint: &endpoint, Model: &model},
|
||||||
|
})}}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
engine, err := pk.NewEngine(pk.Config{Sources: sources},
|
||||||
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
|
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
|
||||||
pk.WithProfiles(pk.Profile{ID: "fixture", Endpoint: "https://example.invalid/v1", Model: "fixture-model"}),
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
Reference in New Issue
Block a user