package promptkit_test import ( "bytes" "context" "errors" "io" "strings" "sync" "sync/atomic" "testing" "testing/fstest" "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" ) func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) { engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{ {Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{ "backends": map[string]any{"local": map[string]any{ "endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY", "extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"}, }}, "profiles": map[string]any{ "base.v1": map[string]any{"backend": "local", "model": "default-model"}, "literal/id": map[string]any{"extends": "base.v1"}, }, })}}}, {Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{ "profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}}, })}}}, }, }}) if err != nil { t.Fatalf("construct layered engine: %v", err) } view, err := engine.InspectConfiguration() if err != nil { t.Fatalf("inspect configuration: %v", err) } modelPath := configkit.Keys("profiles", "base.v1", "model") model, found := view.Explain(modelPath) if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 { t.Fatalf("layered model explanation = (%+v, %t)", model, found) } inheritedPath := configkit.Keys("profiles", "literal/id", "model") inherited, found := view.Explain(inheritedPath) if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) { t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found) } if inheritedPath.String() != "/profiles/literal~1id/model" { t.Fatalf("literal-key path = %q", inheritedPath) } secretPath := configkit.Keys("backends", "local", "extra_params", "token") secret, found := view.Explain(secretPath) if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") { t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found) } var effective bytes.Buffer if err := view.WriteEffective(&effective); err != nil { t.Fatalf("write effective configuration: %v", err) } if strings.Contains(effective.String(), "private-sentinel") || !strings.Contains(effective.String(), "OPTIONAL_KEY") || !strings.Contains(effective.String(), "public-value") { t.Fatalf("unexpected effective report: %s", effective.String()) } var explanation bytes.Buffer if err := view.WriteExplanation(&explanation, secretPath); err != nil || strings.Contains(explanation.String(), "private-sentinel") { t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err) } sources := view.Sources() if len(sources) == 0 { t.Fatal("inspection returned no sources") } originalLabel := sources[0].Label sources[0].Label = "changed" if view.Sources()[0].Label != originalLabel { t.Fatal("source mutation reached retained inspection") } model.History[0].After.YAML = "changed" if again, ok := view.Explain(modelPath); !ok || again.History[0].After.YAML == "changed" { t.Fatal("explanation mutation reached retained inspection") } history := view.Metadata().History(modelPath) if len(history) == 0 { t.Fatal("metadata history is empty") } history[0].Reasons = []string{"changed"} if again := view.Metadata().History(modelPath); len(again) == 0 || (len(again[0].Reasons) > 0 && again[0].Reasons[0] == "changed") { t.Fatal("metadata mutation reached retained inspection") } } type failingInspectionWriter struct{ err error } func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err } func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) { var engine *promptkit.Engine if _, err := engine.InspectConfiguration(); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("nil engine inspection = %v", err) } var empty promptkit.ConfigurationInspection path := configkit.Keys("profiles", "missing") if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 { t.Fatal("zero inspection reported configuration") } if _, found := empty.Explain(path); found { t.Fatal("zero inspection explained an unknown path") } if err := empty.WriteEffective(io.Discard); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("zero effective writer = %v", err) } if err := empty.WriteExplanation(io.Discard, path); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("zero explanation writer = %v", err) } valid, err := promptkit.NewEngine(promptkit.Config{}) if err != nil { t.Fatalf("construct engine: %v", err) } view, err := valid.InspectConfiguration() if err != nil { t.Fatalf("inspect configuration: %v", err) } writeErr := errors.New("report writer failed") writer := failingInspectionWriter{err: writeErr} if err := view.WriteEffective(writer); !errors.Is(err, writeErr) { t.Fatalf("effective writer error = %v", err) } if err := view.WriteExplanation(writer, configkit.Keys("profiles", "mistral-small-3", "model")); !errors.Is(err, writeErr) { t.Fatalf("explanation writer error = %v", err) } } func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) { source := &unreadableAfterConstructionFS{FS: fstest.MapFS{ "config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)}, "config/body.tmpl": &fstest.MapFile{Data: []byte("hello")}, }} inputs, err := promptkit.FilesystemInputs("inspection source", source, "config") if err != nil { t.Fatalf("select inputs: %v", err) } otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{ "config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)}, "config/body.tmpl": &fstest.MapFile{Data: []byte("hello")}, }} otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config") if err != nil { t.Fatalf("select comparison inputs: %v", err) } var modelCalls atomic.Int64 client := countingLLMClient{calls: &modelCalls} endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model" engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{ "offline": {Endpoint: &endpoint, Model: &model}, })}}}}, }}, promptkit.WithLLMClient(client)) if err != nil { t.Fatalf("construct engine: %v", err) } otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{ "offline": {Endpoint: &endpoint, Model: &otherModel}, })}}}}, }}, promptkit.WithLLMClient(client)) if err != nil { t.Fatalf("construct comparison engine: %v", err) } source.closed.Store(true) otherSource.closed.Store(true) view, err := engine.InspectConfiguration() if err != nil { t.Fatalf("inspect configuration: %v", err) } otherView, err := otherEngine.InspectConfiguration() if err != nil { t.Fatalf("inspect comparison configuration: %v", err) } var expected configkit.Difference for index := 0; index < 3; index++ { differences, err := view.Diff(otherView) if err != nil || len(differences) != 1 { t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err) } if differences[0].Path.String() != "/profiles/offline/model" || !strings.Contains(differences[0].Left.YAML, model) || !strings.Contains(differences[0].Right.YAML, otherModel) { t.Fatalf("captured difference = %+v", differences[0]) } if index == 0 { expected = differences[0] } else if differences[0].Path.String() != expected.Path.String() || differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML { t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0]) } } if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found { t.Fatal("captured prompt content path was not inspectable") } var report bytes.Buffer if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") { t.Fatalf("write captured report = (%q, %v)", report.String(), err) } if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 { t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load()) } const calls = 24 var wait sync.WaitGroup errs := make(chan error, calls) for index := 0; index < calls; index++ { wait.Add(1) go func(index int) { defer wait.Done() switch index % 3 { case 0: _, err := engine.InspectConfiguration() errs <- err case 1: _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"}) errs <- err case 2: differences, err := view.Diff(otherView) if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() || differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) { err = errors.New("concurrent comparison returned an unexpected difference") } errs <- err } }(index) } wait.Wait() close(errs) for err := range errs { if err != nil { t.Fatal(err) } } if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 { t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load()) } }