diff --git a/configuration_inspection_contract_test.go b/configuration_inspection_contract_test.go index 552933c..41e6761 100644 --- a/configuration_inspection_contract_test.go +++ b/configuration_inspection_contract_test.go @@ -144,9 +144,17 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi if err != nil { t.Fatalf("select inputs: %v", err) } + otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{ + "config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)}, + "config/body.tmpl": &fstest.MapFile{Data: []byte("hello")}, + }} + otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config") + if err != nil { + t.Fatalf("select comparison inputs: %v", err) + } var modelCalls atomic.Int64 client := countingLLMClient{calls: &modelCalls} - endpoint, model := "https://example.invalid/v1", "offline-model" + endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model" engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{ "offline": {Endpoint: &endpoint, Model: &model}, @@ -155,11 +163,42 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi if err != nil { t.Fatalf("construct engine: %v", err) } + otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{ + "offline": {Endpoint: &endpoint, Model: &otherModel}, + })}}}}, + }}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatalf("construct comparison engine: %v", err) + } source.closed.Store(true) + otherSource.closed.Store(true) view, err := engine.InspectConfiguration() if err != nil { t.Fatalf("inspect configuration: %v", err) } + otherView, err := otherEngine.InspectConfiguration() + if err != nil { + t.Fatalf("inspect comparison configuration: %v", err) + } + var expected configkit.Difference + for index := 0; index < 3; index++ { + differences, err := view.Diff(otherView) + if err != nil || len(differences) != 1 { + t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err) + } + if differences[0].Path.String() != "/profiles/offline/model" || + !strings.Contains(differences[0].Left.YAML, model) || + !strings.Contains(differences[0].Right.YAML, otherModel) { + t.Fatalf("captured difference = %+v", differences[0]) + } + if index == 0 { + expected = differences[0] + } else if differences[0].Path.String() != expected.Path.String() || + differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML { + t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0]) + } + } if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found { t.Fatal("captured prompt content path was not inspectable") } @@ -167,24 +206,32 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") { t.Fatalf("write captured report = (%q, %v)", report.String(), err) } - if source.reads.Load() != 0 || modelCalls.Load() != 0 { - t.Fatalf("inspection touched source or provider: reads=%d requests=%d", source.reads.Load(), modelCalls.Load()) + if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 { + t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load()) } - const calls = 16 + const calls = 24 var wait sync.WaitGroup errs := make(chan error, calls) for index := 0; index < calls; index++ { wait.Add(1) go func(index int) { defer wait.Done() - if index%2 == 0 { + switch index % 3 { + case 0: _, err := engine.InspectConfiguration() errs <- err - return + case 1: + _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"}) + errs <- err + case 2: + differences, err := view.Diff(otherView) + if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() || + differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) { + err = errors.New("concurrent comparison returned an unexpected difference") + } + errs <- err } - _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"}) - errs <- err }(index) } wait.Wait() @@ -194,7 +241,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi t.Fatal(err) } } - if source.reads.Load() != 0 || modelCalls.Load() != calls/2 { - t.Fatalf("concurrent inspection/execution = source reads %d, model calls %d", source.reads.Load(), modelCalls.Load()) + if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 { + t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load()) } } diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 7c2632f..36fef47 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -159,6 +159,13 @@ sequence, sensitivity, and ordering cases. ## Stage 4: Verify Frozen State And Concurrent Inspection +**Status: Complete.** The existing captured-source fixture now compares two +views repeatedly after both sources become unreadable, and concurrently runs +inspection, comparison, and offline execution against those captured engines. +It confirms stable differences without source reads or comparison-triggered +model calls. `GOTOOLCHAIN=local go test -race . ./internal/configuration +./internal/engine` and `git diff --check` passed. + Scope: lifecycle and concurrency integration tests. 1. Reuse or extend the existing captured-source inspection fixture and counting