Prototype ConfigKit and PromptKit adoption and record the findings

examples/library-adoption/config_test.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+
+	ck "gitea.maximumdirect.net/eric/ConfigKit"
+)
+
+func TestConfigurationContract(t *testing.T) {
+	original, err := os.ReadFile("config.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, tc := range []struct {
+		name string
+		edit func(string) string
+		key  string
+		bad  bool
+	}{
+		{"complete", func(s string) string { return s }, "synthetic-test-secret", false},
+		{"empty credential", func(s string) string { return s }, "", true},
+		{"quoted version", func(s string) string { return strings.Replace(s, "version: 2", "version: '2'", 1) }, "key", true},
+		{"version one", func(s string) string { return strings.Replace(s, "version: 2", "version: 1", 1) }, "key", true},
+		{"missing version", func(s string) string { return strings.Replace(s, "version: 2\n", "", 1) }, "key", true},
+		{"unknown field", func(s string) string { return s + "unknown: true\n" }, "key", true},
+		{"omitted validation profile", func(s string) string { return strings.Replace(s, "  validation_profile: assessment\n", "", 1) }, "key", false},
+		{"empty validation profile", func(s string) string {
+			return strings.Replace(s, "validation_profile: assessment", "validation_profile: ''", 1)
+		}, "key", true},
+		{"null validation profile", func(s string) string {
+			return strings.Replace(s, "validation_profile: assessment", "validation_profile: null", 1)
+		}, "key", true},
+		{"omitted backend limits", func(s string) string {
+			return strings.Replace(strings.Replace(s, "    concurrency: 4\n", "", 1), "    queue_capacity: 8\n", "", 1)
+		}, "key", false},
+		{"zero backend limit", func(s string) string { return strings.Replace(s, "concurrency: 4", "concurrency: 0", 1) }, "key", true},
+		{"zero waiting capacity", func(s string) string { return strings.Replace(s, "queue_capacity: 8", "queue_capacity: 0", 1) }, "key", false},
+		{"null optional credential", func(s string) string {
+			return strings.Replace(s, "credential: AUDITA_PROTOTYPE_KEY", "credential: null", 1)
+		}, "", false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Setenv("AUDITA_PROTOTYPE_KEY", tc.key)
+			loaded, err := loadConfig(ck.Bytes("example", []byte(tc.edit(string(original)))))
+			if tc.bad {
+				if err == nil {
+					t.Fatal("expected rejection")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if loaded.Config.Version != 2 {
+				t.Fatal("version not decoded")
+			}
+			if tc.name == "omitted validation profile" && loaded.Config.LLM.ValidationProfile != loaded.Config.LLM.ProposalProfile {
+				t.Fatal("inheritance failed")
+			}
+			if tc.name == "omitted backend limits" && (loaded.Config.Backends["offline"].Concurrency != 1 || loaded.Config.Backends["offline"].QueueCapacity != 1024) {
+				t.Fatal("finite defaults missing")
+			}
+			var effective bytes.Buffer
+			if err := loaded.WriteEffective(&effective); err != nil {
+				t.Fatal(err)
+			}
+			if strings.Contains(effective.String(), "synthetic-test-secret") {
+				t.Fatal("credential exposed")
+			}
+			r, err := newRuntime(loaded.Config, ".", demonstrationClient())
+			if err != nil {
+				t.Fatal(err)
+			}
+			p, err := r.Engine.InspectProfile(context.Background(), "proposal")
+			if err != nil {
+				t.Fatal(err)
+			}
+			raw, _ := json.Marshal(p)
+			if strings.Contains(string(raw), "synthetic-test-secret") {
+				t.Fatal("profile exposed credential")
+			}
+		})
+	}
+	t.Setenv("AUDITA_PROTOTYPE_KEY", "")
+	if err := os.Unsetenv("AUDITA_PROTOTYPE_KEY"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := loadConfig(ck.Bytes("example", original)); err == nil {
+		t.Fatal("missing secret accepted")
+	}
+	t.Setenv("AUDITA_PROTOTYPE_KEY", "key")
+	loaded, err := loadConfig(ck.Bytes("example", original), intOverride("transient_retries", 0), ck.Override{Path: ck.Keys("llm", "validation_profile"), Value: "proposal", Flag: "--validation-profile"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if loaded.Config.LLM.TransientRetries != 0 || loaded.Config.LLM.ValidationProfile != "proposal" {
+		t.Fatal("explicit override lost")
+	}
+}
+
+func TestProfilesRespectSharedBackendBoundary(t *testing.T) {
+	t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret")
+	loaded, err := loadConfig(ck.File("config.yaml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, tc := range []struct{ name, profile string }{
+		{"endpoint only", "id: custom\nendpoint: https://offline.invalid/v1\nmodel: example\n"},
+		{"endpoint override", "id: custom\nbackend: offline\nendpoint: https://other.invalid/v1\nmodel: example\n"},
+		{"competing secret lookup", "id: custom\nbackend: offline\nmodel: example\napi_key_env: OTHER_PROTOTYPE_KEY\n"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			if err := os.WriteFile(dir+"/custom.yaml", []byte(tc.profile), 0600); err != nil {
+				t.Fatal(err)
+			}
+			c := loaded.Config
+			c.ProfilesDir = dir
+			c.LLM.ProposalProfile = "custom"
+			c.LLM.ValidationProfile = "custom"
+			if _, err := newRuntime(c, ".", demonstrationClient()); err == nil {
+				t.Fatal("profile bypass accepted")
+			}
+		})
+	}
+}