eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Demonstrate overlay model routing in prototype

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 00:19:51 +00:00
parent 2d8b441802
commit 228ac4e26d
5 changed files with 153 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/roadmap/implementation.md
View File

@@ -123,7 +123,10 @@ Do not yet claim that the testing models reached the HTTP transport.
## Stage 3: Demonstrate and Verify Model Switching
**Status:** Pending. **Depends on:** Stage 2.
**Status:** Complete (2026-09-24). **Depends on:** Stage 2. Nested tests inspect
outbound base/testing/override model IDs, including repairs; all three documented
offline demonstrations passed with one accepted correction and 4/0/2 attempts,
retries, and repairs.
Start with `main.go`, `offline.go`, `runtime.go`, and `workflow_test.go`. Add minimal
standard-library argument handling to the prototype executable:

12
examples/library-adoption/README.md
View File

@@ -5,8 +5,7 @@ is a standalone nested Go module, not a production Audita command. Its configura
and profile files are accepted by this prototype only. Design decisions, limitations,
and migration status belong to the [findings](../../docs/roadmap/library-boundary-findings.md).
The current prototype pins ConfigKit v0.4.0; the original v0.3.1 results remain
historical evidence in the findings. Selected overlays are not wired into this
executable yet.
historical evidence in the findings.
From this directory, acquire dependencies once (network access required):
@@ -19,6 +18,8 @@ these commands run offline and require no credentials:
```sh
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off go run .
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off go run . --overlay testing
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off go run . --overlay testing --proposal-profile proposal
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off go test ./...
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off go test -race ./...
```
@@ -28,6 +29,13 @@ never makes a network connection. Both proposal and assessment first return inva
JSON, then valid output. Expected output reports one accepted correction, four
attempts, zero transient retries, and two structural repairs. It previews a synthetic
transcript correction without writing transcript files.
The default invocation uses `config.yaml` and its base proposal/assessment profiles.
`--overlay testing` loads `overlays/testing.yml`, selecting both testing profiles
from `profiles/`. An explicitly supplied `--proposal-profile` or
`--validation-profile` replaces that role's selection. The output shows the
requested overlay and each effective profile, backend, and model. Synthetic model
IDs demonstrate operator-selected routing; they are not measurements of price,
speed, or model quality.
Tests control provider responses, capacity, and cancellation. They cover positive
repair budgets for both roles, exhausted repair, domain rejection, transient retry,

91
examples/library-adoption/main.go
View File

@@ -3,7 +3,11 @@ package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
ck "gitea.maximumdirect.net/eric/ConfigKit"
@@ -16,31 +20,70 @@ func main() {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
loaded, err := loadConfig(ck.File("config.yaml"), "")
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
runtime, err := newRuntime(loaded.Config, ".", demonstrationClient())
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
recorder := &Recorder{}
result, err := runtime.workflow(context.Background(), recorder.observer)
if err != nil {
fmt.Fprintln(os.Stderr, "offline workflow failed")
os.Exit(1)
}
attempts, retries, repairs := recorder.totals()
output := struct {
Result WorkflowResult `json:"result"`
Attempts uint64 `json:"attempts"`
Retries uint64 `json:"transient_retries"`
Repairs uint64 `json:"structural_repairs"`
}{result, attempts, retries, repairs}
if err := json.NewEncoder(os.Stdout).Encode(output); err != nil {
if err := run(os.Args[1:], os.Stdout, demonstrationClient()); err != nil {
if errors.Is(err, flag.ErrHelp) {
return
}
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run(args []string, output io.Writer, client *http.Client) error {
flags := flag.NewFlagSet("library-adoption", flag.ContinueOnError)
flags.SetOutput(os.Stderr)
overlay := flags.String("overlay", "", "ConfigKit overlay")
proposal := flags.String("proposal-profile", "", "proposal profile")
validation := flags.String("validation-profile", "", "validation profile")
if err := flags.Parse(args); err != nil {
return err
}
if flags.NArg() != 0 {
return fmt.Errorf("unexpected positional arguments: %v", flags.Args())
}
var overrides []ck.Override
flags.Visit(func(f *flag.Flag) {
switch f.Name {
case "proposal-profile":
overrides = append(overrides, ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: *proposal, Flag: "--proposal-profile"})
case "validation-profile":
overrides = append(overrides, ck.Override{Path: ck.Keys("llm", "validation_profile"), Value: *validation, Flag: "--validation-profile"})
}
})
loaded, err := loadConfig(ck.File("config.yaml"), *overlay, overrides...)
if err != nil {
return err
}
runtime, err := newRuntime(loaded.Config, ".", client)
if err != nil {
return err
}
recorder := &Recorder{}
result, err := runtime.workflow(context.Background(), recorder.observer)
if err != nil {
return fmt.Errorf("offline workflow failed: %w", err)
}
attempts, retries, repairs := recorder.totals()
type selection struct {
Profile string `json:"profile"`
Backend string `json:"backend"`
Model string `json:"model"`
}
selected := func(role string) selection {
p := runtime.Profiles[role]
return selection{p.ProfileID, p.EffectiveModelParams.BackendID, p.EffectiveModelParams.Model}
}
summary := struct {
Overlay string `json:"overlay"`
Proposal selection `json:"proposal"`
Assessment selection `json:"assessment"`
Result WorkflowResult `json:"result"`
Attempts uint64 `json:"attempts"`
Retries uint64 `json:"transient_retries"`
Repairs uint64 `json:"structural_repairs"`
}{*overlay, selected("proposal"), selected("assessment"), result, attempts, retries, repairs}
if err := json.NewEncoder(output).Encode(summary); err != nil {
return err
}
return nil
}

4
examples/library-adoption/offline.go
View File

@@ -50,9 +50,9 @@ func demonstrationClient() *http.Client {
return providerResponse(200, "not JSON", nil), nil
}
switch body.Model {
case "synthetic-grammar":
case "synthetic-grammar", "synthetic-grammar-testing":
return providerResponse(200, validProposal, nil), nil
case "synthetic-review":
case "synthetic-review", "synthetic-review-testing":
return providerResponse(200, validAssessment, nil), nil
default:
return nil, fmt.Errorf("unexpected synthetic model")

70
examples/library-adoption/workflow_test.go
View File

@@ -1,9 +1,11 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"sync"
@@ -13,6 +15,74 @@ import (
pk "gitea.maximumdirect.net/eric/promptkit"
)
func TestDemonstrationSelectsOutboundModels(t *testing.T) {
for _, tc := range []struct {
name string
args []string
overlay, proposal, assessment string
}{
{"base", nil, "", "synthetic-grammar", "synthetic-review"},
{"testing", []string{"--overlay", "testing"}, "testing", "synthetic-grammar-testing", "synthetic-review-testing"},
{"proposal override", []string{"--overlay", "testing", "--proposal-profile", "proposal"}, "testing", "synthetic-grammar", "synthetic-review-testing"},
{"assessment override", []string{"--overlay", "testing", "--validation-profile", "assessment"}, "testing", "synthetic-grammar-testing", "synthetic-review"},
} {
t.Run(tc.name, func(t *testing.T) {
t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret")
var mu sync.Mutex
var models []string
base := demonstrationClient()
client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
body, err := io.ReadAll(req.Body)
if err != nil {
return nil, err
}
var wire struct {
Model string `json:"model"`
}
if err := json.Unmarshal(body, &wire); err != nil {
return nil, err
}
mu.Lock()
models = append(models, wire.Model)
mu.Unlock()
req.Body = io.NopCloser(bytes.NewReader(body))
return base.Transport.RoundTrip(req)
})}
var out bytes.Buffer
if err := run(tc.args, &out, client); err != nil {
t.Fatal(err)
}
var got struct {
Overlay string `json:"overlay"`
Proposal struct{ Profile, Backend, Model string } `json:"proposal"`
Assessment struct{ Profile, Backend, Model string } `json:"assessment"`
Result WorkflowResult `json:"result"`
Attempts uint64 `json:"attempts"`
Retries uint64 `json:"transient_retries"`
Repairs uint64 `json:"structural_repairs"`
}
if err := json.Unmarshal(out.Bytes(), &got); err != nil {
t.Fatal(err)
}
if got.Overlay != tc.overlay || got.Proposal.Model != tc.proposal || got.Assessment.Model != tc.assessment || got.Proposal.Backend != "offline" || got.Assessment.Backend != "offline" {
t.Fatalf("selection = %+v", got)
}
profiles := map[string]string{"synthetic-grammar": "proposal", "synthetic-review": "assessment", "synthetic-grammar-testing": "proposal-testing", "synthetic-review-testing": "assessment-testing"}
if got.Proposal.Profile != profiles[tc.proposal] || got.Assessment.Profile != profiles[tc.assessment] {
t.Fatalf("reported profiles = %q, %q", got.Proposal.Profile, got.Assessment.Profile)
}
if got.Result.Accepted != 1 || got.Attempts != 4 || got.Retries != 0 || got.Repairs != 2 {
t.Fatalf("outcome = %+v", got)
}
mu.Lock()
defer mu.Unlock()
if len(models) != 4 || models[0] != tc.proposal || models[1] != tc.proposal || models[2] != tc.assessment || models[3] != tc.assessment {
t.Fatalf("outbound models = %v", models)
}
})
}
}
type step struct {
status int
content string