239 lines
8.0 KiB
Go
239 lines
8.0 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
|
)
|
|
|
|
func overlayLayout(t *testing.T, base, overlay string) string {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(base), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if overlay != "" {
|
|
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, name := range []string{"proposal.yaml", "assessment.yaml", "proposal-testing.yaml", "assessment-testing.yaml"} {
|
|
data, err := os.ReadFile(filepath.Join("profiles", name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "profiles", name), data, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return dir
|
|
}
|
|
|
|
func fixtureConfig(t *testing.T) string {
|
|
t.Helper()
|
|
data, err := os.ReadFile("config.yaml")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(data)
|
|
}
|
|
|
|
func requireLoadCode(t *testing.T, err error, code ck.Code) {
|
|
t.Helper()
|
|
var loadErr *ck.LoadError
|
|
if !errors.As(err, &loadErr) {
|
|
t.Fatalf("expected ConfigKit load error, got %v", err)
|
|
}
|
|
for _, diagnostic := range loadErr.Diagnostics() {
|
|
if diagnostic.Code == code {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("expected diagnostic %s, got %+v", code, loadErr.Diagnostics())
|
|
}
|
|
|
|
func TestOverlayFailuresAndApplicationValidation(t *testing.T) {
|
|
t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-secret")
|
|
base := fixtureConfig(t)
|
|
for _, tc := range []struct {
|
|
name, selection, overlay string
|
|
code ck.Code
|
|
}{
|
|
{"missing", "missing", "", ck.CodeOverlayMissing},
|
|
{"invalid name", "../testing", "", ck.CodeOptionsInvalid},
|
|
{"version", "testing", "version: 3\n", ck.CodeOverlayVersion},
|
|
{"directive", "testing", "configkit: {}\n", ck.CodeOverlayDirective},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
dir := overlayLayout(t, base, tc.overlay)
|
|
_, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), tc.selection)
|
|
requireLoadCode(t, err, tc.code)
|
|
})
|
|
}
|
|
valid := "backends:\n offline:\n concurrency: 2\n"
|
|
dir := overlayLayout(t, base, valid)
|
|
loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := loaded.Config.Backends["offline"].Concurrency; got != 2 {
|
|
t.Fatalf("overlay concurrency = %d", got)
|
|
}
|
|
dir = overlayLayout(t, base, "backends:\n offline:\n concurrency: 0\n")
|
|
if _, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing"); err == nil {
|
|
t.Fatal("invalid effective backend limit accepted")
|
|
}
|
|
}
|
|
|
|
func TestOverlayCredentialResolutionAndInspection(t *testing.T) {
|
|
t.Setenv("AUDITA_PROTOTYPE_KEY", "")
|
|
t.Setenv("AUDITA_OVERLAY_KEY", "replacement-synthetic-secret")
|
|
base := fixtureConfig(t)
|
|
for _, tc := range []struct {
|
|
name, overlay, key string
|
|
unsetKey bool
|
|
wantError, wantAuth bool
|
|
}{
|
|
{"replacement", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "replacement-synthetic-secret", false, false, true},
|
|
{"surviving missing", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "", true, true, false},
|
|
{"surviving empty", "backends:\n offline:\n credential: AUDITA_OVERLAY_KEY\n", "", false, true, false},
|
|
{"unauthenticated null", "backends:\n offline:\n credential: null\n", "replacement-synthetic-secret", false, false, false},
|
|
{"profile switch retains credential", "llm:\n proposal_profile: proposal-testing\n validation_profile: assessment-testing\n", "replacement-synthetic-secret", false, true, false},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Setenv("AUDITA_OVERLAY_KEY", tc.key)
|
|
if tc.unsetKey {
|
|
if err := os.Unsetenv("AUDITA_OVERLAY_KEY"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
dir := overlayLayout(t, base, tc.overlay)
|
|
loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing")
|
|
if tc.wantError {
|
|
if err == nil {
|
|
t.Fatal("unresolved surviving credential accepted")
|
|
}
|
|
if strings.Contains(err.Error(), "replacement-synthetic-secret") {
|
|
t.Fatal("error exposed secret")
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var effective, explanation bytes.Buffer
|
|
if err := loaded.WriteEffective(&effective); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := loaded.WriteExplanation(&explanation, ck.Keys("backends", "offline", "credential")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(effective.String()+explanation.String(), "replacement-synthetic-secret") {
|
|
t.Fatal("inspection exposed secret")
|
|
}
|
|
observed := 0
|
|
baseClient := demonstrationClient()
|
|
client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
|
want := ""
|
|
if tc.wantAuth {
|
|
want = "Bearer replacement-synthetic-secret"
|
|
}
|
|
if req.Header.Get("Authorization") != want {
|
|
t.Errorf("authorization mismatch")
|
|
}
|
|
observed++
|
|
return baseClient.Transport.RoundTrip(req)
|
|
})}
|
|
r, err := newRuntime(loaded.Config, dir, client)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, role := range []string{"proposal", "assessment"} {
|
|
p := r.Profiles[role]
|
|
raw, _ := json.Marshal(p)
|
|
if strings.Contains(string(raw), "replacement-synthetic-secret") {
|
|
t.Fatal("profile exposed secret")
|
|
}
|
|
}
|
|
recorder := &Recorder{}
|
|
result, err := r.workflow(context.Background(), recorder.observer)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Accepted != 1 || observed != 4 {
|
|
t.Fatalf("result %+v; calls %d", result, observed)
|
|
}
|
|
raw, _ := json.Marshal(recorder.snapshot())
|
|
if strings.Contains(string(raw), "replacement-synthetic-secret") {
|
|
t.Fatal("observations exposed secret")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOverlaySelectionProvenance(t *testing.T) {
|
|
t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-secret")
|
|
dir := overlayLayout(t, fixtureConfig(t), "llm:\n proposal_profile: proposal-testing\n validation_profile: assessment-testing\n")
|
|
loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), "testing", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, tc := range []struct {
|
|
key string
|
|
layer ck.Layer
|
|
}{{"proposal_profile", ck.LayerCLI}, {"validation_profile", ck.LayerOverlay}} {
|
|
path := ck.Keys("llm", tc.key)
|
|
info, ok := loaded.Metadata().Lookup(path)
|
|
if !ok || info.Origin.Layer != tc.layer {
|
|
t.Fatalf("%s metadata = %+v", tc.key, info)
|
|
}
|
|
explain, ok := loaded.Explain(path)
|
|
if !ok || explain.Origin.Layer != tc.layer {
|
|
t.Fatalf("%s explanation = %+v", tc.key, explain)
|
|
}
|
|
}
|
|
var effective, explanation bytes.Buffer
|
|
if err := loaded.WriteEffective(&effective); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := loaded.WriteExplanation(&explanation, ck.Keys("llm", "validation_profile")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(effective.String(), "assessment-testing") || !strings.Contains(explanation.String(), "overlay") {
|
|
t.Fatal("overlay inspection missing")
|
|
}
|
|
seenOverlay := false
|
|
for _, source := range loaded.Sources() {
|
|
if source.Layer == ck.LayerOverlay {
|
|
seenOverlay = true
|
|
}
|
|
}
|
|
if !seenOverlay {
|
|
t.Fatal("overlay source missing")
|
|
}
|
|
r, err := newRuntime(loaded.Config, dir, demonstrationClient())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if r.Profiles["proposal"].ProfileID != "proposal" || r.Profiles["assessment"].ProfileID != "assessment-testing" || r.Profiles["assessment"].EffectiveModelParams.Model != "synthetic-review-testing" {
|
|
t.Fatal("inspection/runtime selection diverged")
|
|
}
|
|
if r.Profiles["proposal"].EffectiveModelParams.Model != "synthetic-grammar" || r.Profiles["proposal"].EffectiveModelParams.BackendID != "offline" || r.Profiles["assessment"].EffectiveModelParams.BackendID != "offline" {
|
|
t.Fatal("inspected profile target diverged")
|
|
}
|
|
}
|