281 lines
12 KiB
Go
281 lines
12 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
|
Underlays: []configkit.Underlay{
|
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
|
"backends": map[string]any{"local": map[string]any{
|
|
"endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
|
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
|
}},
|
|
"collections": map[string]any{"literal": map[string]any{
|
|
"backends": map[string]any{"local": map[string]any{
|
|
"endpoint": "https://example.invalid/v1",
|
|
"extra_params": map[string]any{"token": "namespaced-private-sentinel"},
|
|
}},
|
|
"profiles": map[string]any{
|
|
"base.v1": map[string]any{"backend": "local", "model": "default-model"},
|
|
"id": map[string]any{"extends": "base.v1"},
|
|
},
|
|
}},
|
|
})}}},
|
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
|
"collections": map[string]any{"literal": map[string]any{"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}}}},
|
|
})}}},
|
|
},
|
|
}})
|
|
if err != nil {
|
|
t.Fatalf("construct layered engine: %v", err)
|
|
}
|
|
view, err := engine.InspectConfiguration()
|
|
if err != nil {
|
|
t.Fatalf("inspect configuration: %v", err)
|
|
}
|
|
modelPath := configkit.Keys("collections", "literal", "profiles", "base.v1", "model")
|
|
model, found := view.Explain(modelPath)
|
|
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 {
|
|
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
|
}
|
|
inheritedPath := configkit.Keys("collections", "literal", "profiles", "id", "model")
|
|
inherited, found := view.Explain(inheritedPath)
|
|
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
|
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
|
}
|
|
if inheritedPath.String() != "/collections/literal/profiles/id/model" {
|
|
t.Fatalf("literal-key path = %q", inheritedPath)
|
|
}
|
|
secretPath := configkit.Keys("backends", "local", "extra_params", "token")
|
|
secret, found := view.Explain(secretPath)
|
|
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
|
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
|
}
|
|
namespacedSecretPath := configkit.Keys("collections", "literal", "backends", "local", "extra_params", "token")
|
|
namespacedSecret, found := view.Explain(namespacedSecretPath)
|
|
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
|
|
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
|
|
}
|
|
var effective bytes.Buffer
|
|
if err := view.WriteEffective(&effective); err != nil {
|
|
t.Fatalf("write effective configuration: %v", err)
|
|
}
|
|
if strings.Contains(effective.String(), "private-sentinel") || !strings.Contains(effective.String(), "OPTIONAL_KEY") || !strings.Contains(effective.String(), "public-value") {
|
|
t.Fatalf("unexpected effective report: %s", effective.String())
|
|
}
|
|
var explanation bytes.Buffer
|
|
if err := view.WriteExplanation(&explanation, namespacedSecretPath); err != nil || strings.Contains(explanation.String(), "namespaced-private-sentinel") {
|
|
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
|
|
}
|
|
|
|
sources := view.Sources()
|
|
if len(sources) == 0 {
|
|
t.Fatal("inspection returned no sources")
|
|
}
|
|
originalLabel := sources[0].Label
|
|
sources[0].Label = "changed"
|
|
if view.Sources()[0].Label != originalLabel {
|
|
t.Fatal("source mutation reached retained inspection")
|
|
}
|
|
model.History[0].After.YAML = "changed"
|
|
if again, ok := view.Explain(modelPath); !ok || again.History[0].After.YAML == "changed" {
|
|
t.Fatal("explanation mutation reached retained inspection")
|
|
}
|
|
history := view.Metadata().History(modelPath)
|
|
if len(history) == 0 {
|
|
t.Fatal("metadata history is empty")
|
|
}
|
|
history[0].Reasons = []string{"changed"}
|
|
if again := view.Metadata().History(modelPath); len(again) == 0 || (len(again[0].Reasons) > 0 && again[0].Reasons[0] == "changed") {
|
|
t.Fatal("metadata mutation reached retained inspection")
|
|
}
|
|
}
|
|
|
|
func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) {
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
|
Source: configkit.Mapping("main", map[string]any{"version": "1"}),
|
|
}}})
|
|
if err != nil {
|
|
t.Fatalf("construct engine with accepted declaration: %v", err)
|
|
}
|
|
inspection, err := engine.InspectConfiguration()
|
|
if err != nil {
|
|
t.Fatalf("inspect configuration: %v", err)
|
|
}
|
|
path := configkit.Keys("version")
|
|
field, found := inspection.Explain(path)
|
|
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "1") {
|
|
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
|
|
}
|
|
var effective bytes.Buffer
|
|
if err := inspection.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), "version") {
|
|
t.Fatalf("effective inspection = (%q, %v), want retained version", effective.String(), err)
|
|
}
|
|
}
|
|
|
|
type failingInspectionWriter struct{ err error }
|
|
|
|
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
|
|
|
|
func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|
var engine *promptkit.Engine
|
|
if _, err := engine.InspectConfiguration(); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
|
t.Fatalf("nil engine inspection = %v", err)
|
|
}
|
|
var empty promptkit.ConfigurationInspection
|
|
path := configkit.Keys("profiles", "missing")
|
|
if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
|
|
t.Fatal("zero inspection reported configuration")
|
|
}
|
|
if _, found := empty.Explain(path); found {
|
|
t.Fatal("zero inspection explained an unknown path")
|
|
}
|
|
if err := empty.WriteEffective(io.Discard); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
|
t.Fatalf("zero effective writer = %v", err)
|
|
}
|
|
if err := empty.WriteExplanation(io.Discard, path); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
|
t.Fatalf("zero explanation writer = %v", err)
|
|
}
|
|
|
|
valid, err := promptkit.NewEngine(promptkit.Config{})
|
|
if err != nil {
|
|
t.Fatalf("construct engine: %v", err)
|
|
}
|
|
view, err := valid.InspectConfiguration()
|
|
if err != nil {
|
|
t.Fatalf("inspect configuration: %v", err)
|
|
}
|
|
writeErr := errors.New("report writer failed")
|
|
writer := failingInspectionWriter{err: writeErr}
|
|
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
|
t.Fatalf("effective writer error = %v", err)
|
|
}
|
|
if err := view.WriteExplanation(writer, configkit.Keys("profiles", "mistral-small-3", "model")); !errors.Is(err, writeErr) {
|
|
t.Fatalf("explanation writer error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) {
|
|
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
|
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
|
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
|
}}
|
|
inputs, err := promptkit.FilesystemInputs("inspection source", source, "config")
|
|
if err != nil {
|
|
t.Fatalf("select inputs: %v", err)
|
|
}
|
|
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
|
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
|
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
|
}}
|
|
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
|
|
if err != nil {
|
|
t.Fatalf("select comparison inputs: %v", err)
|
|
}
|
|
var modelCalls atomic.Int64
|
|
client := countingLLMClient{calls: &modelCalls}
|
|
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
|
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
|
"offline": {Endpoint: &endpoint, Model: &model},
|
|
})}}}},
|
|
}}, promptkit.WithLLMClient(client))
|
|
if err != nil {
|
|
t.Fatalf("construct engine: %v", err)
|
|
}
|
|
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
|
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
|
|
"offline": {Endpoint: &endpoint, Model: &otherModel},
|
|
})}}}},
|
|
}}, promptkit.WithLLMClient(client))
|
|
if err != nil {
|
|
t.Fatalf("construct comparison engine: %v", err)
|
|
}
|
|
source.closed.Store(true)
|
|
otherSource.closed.Store(true)
|
|
view, err := engine.InspectConfiguration()
|
|
if err != nil {
|
|
t.Fatalf("inspect configuration: %v", err)
|
|
}
|
|
otherView, err := otherEngine.InspectConfiguration()
|
|
if err != nil {
|
|
t.Fatalf("inspect comparison configuration: %v", err)
|
|
}
|
|
var expected configkit.Difference
|
|
for index := 0; index < 3; index++ {
|
|
differences, err := view.Diff(otherView)
|
|
if err != nil || len(differences) != 1 {
|
|
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
|
}
|
|
if differences[0].Path.String() != "/profiles/offline/model" ||
|
|
!strings.Contains(differences[0].Left.YAML, model) ||
|
|
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
|
t.Fatalf("captured difference = %+v", differences[0])
|
|
}
|
|
if index == 0 {
|
|
expected = differences[0]
|
|
} else if differences[0].Path.String() != expected.Path.String() ||
|
|
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
|
|
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
|
}
|
|
}
|
|
if _, found := view.Explain(configkit.Keys("prompts", "captured", "messages").Index(0).Key("content_file")); !found {
|
|
t.Fatal("captured prompt content path was not inspectable")
|
|
}
|
|
var report bytes.Buffer
|
|
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
|
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
|
}
|
|
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
|
|
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
|
}
|
|
|
|
const calls = 24
|
|
var wait sync.WaitGroup
|
|
errs := make(chan error, calls)
|
|
for index := 0; index < calls; index++ {
|
|
wait.Add(1)
|
|
go func(index int) {
|
|
defer wait.Done()
|
|
switch index % 3 {
|
|
case 0:
|
|
_, err := engine.InspectConfiguration()
|
|
errs <- err
|
|
case 1:
|
|
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
|
errs <- err
|
|
case 2:
|
|
differences, err := view.Diff(otherView)
|
|
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
|
|
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
|
|
err = errors.New("concurrent comparison returned an unexpected difference")
|
|
}
|
|
errs <- err
|
|
}
|
|
}(index)
|
|
}
|
|
wait.Wait()
|
|
close(errs)
|
|
for err := range errs {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
|
|
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
|
}
|
|
}
|