From 228ac4e26d439c897d08d8cc1f9eebfb15d1923a Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 24 Sep 2026 00:19:51 +0000 Subject: [PATCH] Demonstrate overlay model routing in prototype --- docs/roadmap/implementation.md | 5 +- examples/library-adoption/README.md | 12 ++- examples/library-adoption/main.go | 91 ++++++++++++++++------ examples/library-adoption/offline.go | 4 +- examples/library-adoption/workflow_test.go | 70 +++++++++++++++++ 5 files changed, 153 insertions(+), 29 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 5395fe0..c2080cb 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -123,7 +123,10 @@ Do not yet claim that the testing models reached the HTTP transport. ## Stage 3: Demonstrate and Verify Model Switching -**Status:** Pending. **Depends on:** Stage 2. +**Status:** Complete (2026-09-24). **Depends on:** Stage 2. Nested tests inspect +outbound base/testing/override model IDs, including repairs; all three documented +offline demonstrations passed with one accepted correction and 4/0/2 attempts, +retries, and repairs. Start with `main.go`, `offline.go`, `runtime.go`, and `workflow_test.go`. Add minimal standard-library argument handling to the prototype executable: diff --git a/examples/library-adoption/README.md b/examples/library-adoption/README.md index 9ea60bf..da39cfc 100644 --- a/examples/library-adoption/README.md +++ b/examples/library-adoption/README.md @@ -5,8 +5,7 @@ is a standalone nested Go module, not a production Audita command. Its configura and profile files are accepted by this prototype only. Design decisions, limitations, and migration status belong to the [findings](../../docs/roadmap/library-boundary-findings.md). The current prototype pins ConfigKit v0.4.0; the original v0.3.1 results remain -historical evidence in the findings. Selected overlays are not wired into this -executable yet. +historical evidence in the findings. From this directory, acquire dependencies once (network access required): @@ -19,6 +18,8 @@ these commands run offline and require no credentials: ```sh GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off go run . +GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off go run . --overlay testing +GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off go run . --overlay testing --proposal-profile proposal GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off go test ./... GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off go test -race ./... ``` @@ -28,6 +29,13 @@ never makes a network connection. Both proposal and assessment first return inva JSON, then valid output. Expected output reports one accepted correction, four attempts, zero transient retries, and two structural repairs. It previews a synthetic transcript correction without writing transcript files. +The default invocation uses `config.yaml` and its base proposal/assessment profiles. +`--overlay testing` loads `overlays/testing.yml`, selecting both testing profiles +from `profiles/`. An explicitly supplied `--proposal-profile` or +`--validation-profile` replaces that role's selection. The output shows the +requested overlay and each effective profile, backend, and model. Synthetic model +IDs demonstrate operator-selected routing; they are not measurements of price, +speed, or model quality. Tests control provider responses, capacity, and cancellation. They cover positive repair budgets for both roles, exhausted repair, domain rejection, transient retry, diff --git a/examples/library-adoption/main.go b/examples/library-adoption/main.go index 70d5e23..31ccf27 100644 --- a/examples/library-adoption/main.go +++ b/examples/library-adoption/main.go @@ -3,7 +3,11 @@ package main import ( "context" "encoding/json" + "errors" + "flag" "fmt" + "io" + "net/http" "os" ck "gitea.maximumdirect.net/eric/ConfigKit" @@ -16,31 +20,70 @@ func main() { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - loaded, err := loadConfig(ck.File("config.yaml"), "") - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - runtime, err := newRuntime(loaded.Config, ".", demonstrationClient()) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - recorder := &Recorder{} - result, err := runtime.workflow(context.Background(), recorder.observer) - if err != nil { - fmt.Fprintln(os.Stderr, "offline workflow failed") - os.Exit(1) - } - attempts, retries, repairs := recorder.totals() - output := struct { - Result WorkflowResult `json:"result"` - Attempts uint64 `json:"attempts"` - Retries uint64 `json:"transient_retries"` - Repairs uint64 `json:"structural_repairs"` - }{result, attempts, retries, repairs} - if err := json.NewEncoder(os.Stdout).Encode(output); err != nil { + if err := run(os.Args[1:], os.Stdout, demonstrationClient()); err != nil { + if errors.Is(err, flag.ErrHelp) { + return + } fmt.Fprintln(os.Stderr, err) os.Exit(1) } } + +func run(args []string, output io.Writer, client *http.Client) error { + flags := flag.NewFlagSet("library-adoption", flag.ContinueOnError) + flags.SetOutput(os.Stderr) + overlay := flags.String("overlay", "", "ConfigKit overlay") + proposal := flags.String("proposal-profile", "", "proposal profile") + validation := flags.String("validation-profile", "", "validation profile") + if err := flags.Parse(args); err != nil { + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("unexpected positional arguments: %v", flags.Args()) + } + var overrides []ck.Override + flags.Visit(func(f *flag.Flag) { + switch f.Name { + case "proposal-profile": + overrides = append(overrides, ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: *proposal, Flag: "--proposal-profile"}) + case "validation-profile": + overrides = append(overrides, ck.Override{Path: ck.Keys("llm", "validation_profile"), Value: *validation, Flag: "--validation-profile"}) + } + }) + loaded, err := loadConfig(ck.File("config.yaml"), *overlay, overrides...) + if err != nil { + return err + } + runtime, err := newRuntime(loaded.Config, ".", client) + if err != nil { + return err + } + recorder := &Recorder{} + result, err := runtime.workflow(context.Background(), recorder.observer) + if err != nil { + return fmt.Errorf("offline workflow failed: %w", err) + } + attempts, retries, repairs := recorder.totals() + type selection struct { + Profile string `json:"profile"` + Backend string `json:"backend"` + Model string `json:"model"` + } + selected := func(role string) selection { + p := runtime.Profiles[role] + return selection{p.ProfileID, p.EffectiveModelParams.BackendID, p.EffectiveModelParams.Model} + } + summary := struct { + Overlay string `json:"overlay"` + Proposal selection `json:"proposal"` + Assessment selection `json:"assessment"` + Result WorkflowResult `json:"result"` + Attempts uint64 `json:"attempts"` + Retries uint64 `json:"transient_retries"` + Repairs uint64 `json:"structural_repairs"` + }{*overlay, selected("proposal"), selected("assessment"), result, attempts, retries, repairs} + if err := json.NewEncoder(output).Encode(summary); err != nil { + return err + } + return nil +} diff --git a/examples/library-adoption/offline.go b/examples/library-adoption/offline.go index 43d7765..4bab811 100644 --- a/examples/library-adoption/offline.go +++ b/examples/library-adoption/offline.go @@ -50,9 +50,9 @@ func demonstrationClient() *http.Client { return providerResponse(200, "not JSON", nil), nil } switch body.Model { - case "synthetic-grammar": + case "synthetic-grammar", "synthetic-grammar-testing": return providerResponse(200, validProposal, nil), nil - case "synthetic-review": + case "synthetic-review", "synthetic-review-testing": return providerResponse(200, validAssessment, nil), nil default: return nil, fmt.Errorf("unexpected synthetic model") diff --git a/examples/library-adoption/workflow_test.go b/examples/library-adoption/workflow_test.go index fdfb343..849aca2 100644 --- a/examples/library-adoption/workflow_test.go +++ b/examples/library-adoption/workflow_test.go @@ -1,9 +1,11 @@ package main import ( + "bytes" "context" "encoding/json" "errors" + "io" "net/http" "strings" "sync" @@ -13,6 +15,74 @@ import ( pk "gitea.maximumdirect.net/eric/promptkit" ) +func TestDemonstrationSelectsOutboundModels(t *testing.T) { + for _, tc := range []struct { + name string + args []string + overlay, proposal, assessment string + }{ + {"base", nil, "", "synthetic-grammar", "synthetic-review"}, + {"testing", []string{"--overlay", "testing"}, "testing", "synthetic-grammar-testing", "synthetic-review-testing"}, + {"proposal override", []string{"--overlay", "testing", "--proposal-profile", "proposal"}, "testing", "synthetic-grammar", "synthetic-review-testing"}, + {"assessment override", []string{"--overlay", "testing", "--validation-profile", "assessment"}, "testing", "synthetic-grammar-testing", "synthetic-review"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret") + var mu sync.Mutex + var models []string + base := demonstrationClient() + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + var wire struct { + Model string `json:"model"` + } + if err := json.Unmarshal(body, &wire); err != nil { + return nil, err + } + mu.Lock() + models = append(models, wire.Model) + mu.Unlock() + req.Body = io.NopCloser(bytes.NewReader(body)) + return base.Transport.RoundTrip(req) + })} + var out bytes.Buffer + if err := run(tc.args, &out, client); err != nil { + t.Fatal(err) + } + var got struct { + Overlay string `json:"overlay"` + Proposal struct{ Profile, Backend, Model string } `json:"proposal"` + Assessment struct{ Profile, Backend, Model string } `json:"assessment"` + Result WorkflowResult `json:"result"` + Attempts uint64 `json:"attempts"` + Retries uint64 `json:"transient_retries"` + Repairs uint64 `json:"structural_repairs"` + } + if err := json.Unmarshal(out.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Overlay != tc.overlay || got.Proposal.Model != tc.proposal || got.Assessment.Model != tc.assessment || got.Proposal.Backend != "offline" || got.Assessment.Backend != "offline" { + t.Fatalf("selection = %+v", got) + } + profiles := map[string]string{"synthetic-grammar": "proposal", "synthetic-review": "assessment", "synthetic-grammar-testing": "proposal-testing", "synthetic-review-testing": "assessment-testing"} + if got.Proposal.Profile != profiles[tc.proposal] || got.Assessment.Profile != profiles[tc.assessment] { + t.Fatalf("reported profiles = %q, %q", got.Proposal.Profile, got.Assessment.Profile) + } + if got.Result.Accepted != 1 || got.Attempts != 4 || got.Retries != 0 || got.Repairs != 2 { + t.Fatalf("outcome = %+v", got) + } + mu.Lock() + defer mu.Unlock() + if len(models) != 4 || models[0] != tc.proposal || models[1] != tc.proposal || models[2] != tc.assessment || models[3] != tc.assessment { + t.Fatalf("outbound models = %v", models) + } + }) + } +} + type step struct { status int content string