diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 12e5d6f..6bbf5b4 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -150,6 +150,8 @@ and existing handle ownership remains intact. ## Stage 3 — Verify Public Content Semantics And Exclusions +Status: complete. + Add focused `promptkit_test` coverage, preferably in `message_content_hash_contract_test.go`, using real engines with small synthetic definitions and an injected model client only where history or execution is diff --git a/message_content_hash_contract_test.go b/message_content_hash_contract_test.go new file mode 100644 index 0000000..8011812 --- /dev/null +++ b/message_content_hash_contract_test.go @@ -0,0 +1,135 @@ +package promptkit_test + +import ( + "context" + "testing" + "testing/fstest" + + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +func contentHashEngine(t *testing.T, promptID, profileID, backendID, content string, output map[string]any, cache bool, opts ...promptkit.Option) *promptkit.Engine { + t.Helper() + message := map[string]any{"role": "user", "content": content} + if cache { + message["cache_control"] = map[string]any{"type": "ephemeral"} + } + config := promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("hash test", map[string]any{ + "backends": []any{map[string]any{"id": backendID, "endpoint": "https://example.test/v1"}}, + "profiles": []any{map[string]any{"id": profileID, "backend": backendID, "model": "model-" + profileID}}, + "prompts": []any{map[string]any{"id": promptID, "default_profile": profileID, "inputs": []any{map[string]any{"name": "input", "required": false}}, "messages": []any{message}, "output": output}}, + })}}} + engine, err := promptkit.NewEngine(config, opts...) + if err != nil { + t.Fatal(err) + } + return engine +} + +func prepareContentHash(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest) *promptkit.PreparedRun { + t.Helper() + prepared, err := engine.Prepare(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if prepared.MessageContentHash == "" { + t.Fatal("missing content hash") + } + return prepared +} + +func TestMessageContentHashIgnoresPreparationMetadata(t *testing.T) { + textOutput := map[string]any{"format": "text", "validation_mode": "none"} + baseEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, false) + base := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first"}) + repeated := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first"}) + other := contentHashEngine(t, "second", "beta", "two", "same", textOutput, false) + otherPrepared := prepareContentHash(t, other, promptkit.RunRequest{PromptID: "second", APIKey: "synthetic-key", Execution: &promptkit.ExecutionTargetOverride{Model: "other-model"}}) + session := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", SessionID: "other-session"}) + cacheEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, true) + cached := prepareContentHash(t, cacheEngine, promptkit.RunRequest{PromptID: "first"}) + schemas := fstest.MapFS{"schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}} + schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}, false, promptkit.WithSchemaFS(schemas, ".")) + schema := prepareContentHash(t, schemaEngine, promptkit.RunRequest{PromptID: "first"}) + for name, prepared := range map[string]*promptkit.PreparedRun{"repeat": repeated, "identifiers and model": otherPrepared, "session": session, "cache": cached, "schema": schema} { + if prepared.MessageContentHash != base.MessageContentHash { + t.Errorf("%s changed content hash", name) + } + } + if session.RenderedPromptHash == base.RenderedPromptHash || cached.RenderedPromptHash == base.RenderedPromptHash { + t.Fatal("session and cache should each change rendered prompt hash") + } +} + +func TestMessageContentHashUsesRenderedInputBytes(t *testing.T) { + reader := &mutablePreparedArtifactReader{body: "body", hash: "first-hash"} + engine := contentHashEngine(t, "prompt", "profile", "backend", `{{input "input"}} {{.label}}`, map[string]any{"format": "text", "validation_mode": "none"}, false, promptkit.WithArtifactReader(reader)) + request := promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"input": promptkit.Inline("first")}, Vars: map[string]string{"label": "label", "unused": "first"}} + base := prepareContentHash(t, engine, request) + reader.set("body", "other-hash") + request.Inputs["input"] = promptkit.Inline("other") + request.Vars["unused"] = "other" + unchanged := prepareContentHash(t, engine, request) + if unchanged.MessageContentHash != base.MessageContentHash { + t.Fatal("unused variable or artifact metadata changed hash") + } + request.Vars["label"] = "changed" + if prepareContentHash(t, engine, request).MessageContentHash == base.MessageContentHash { + t.Fatal("rendered variable did not change hash") + } + request.Vars["label"] = "label" + reader.set("changed body", "other-hash") + if prepareContentHash(t, engine, request).MessageContentHash == base.MessageContentHash { + t.Fatal("rendered input did not change hash") + } +} + +func TestMessageContentHashIncludesHistoryAndAppendedMessages(t *testing.T) { + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}} + engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client)) + initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true}) + if err != nil { + t.Fatal(err) + } + request := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, AppendedMessages: []promptkit.RenderedMessage{{Role: "user", Content: "extra"}}} + continued := prepareContentHash(t, engine, request) + explicit := prepareContentHash(t, engine, promptkit.RunRequest{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: "assistant", Content: "answer"}, {Role: "user", Content: "question"}, {Role: "user", Content: "extra"}}}) + if continued.MessageContentHash != explicit.MessageContentHash { + t.Fatal("equivalent effective messages hashed differently") + } + for _, msg := range []promptkit.RenderedMessage{{Role: "assistant", Content: "extra"}, {Role: "user", Content: "changed"}} { + request.AppendedMessages = []promptkit.RenderedMessage{msg} + if prepareContentHash(t, engine, request).MessageContentHash == continued.MessageContentHash { + t.Fatal("appended message change did not affect hash") + } + } + otherClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "different answer"}} + otherEngine := newSimpleEngine(t, "question", promptkit.WithLLMClient(otherClient)) + other, err := otherEngine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true}) + if err != nil { + t.Fatal(err) + } + request.ContinueFrom = other.Exchange + request.AppendedMessages = []promptkit.RenderedMessage{{Role: "user", Content: "extra"}} + if prepareContentHash(t, engine, request).MessageContentHash == continued.MessageContentHash { + t.Fatal("history text change did not affect hash") + } + firstHistory, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, AppendedMessages: []promptkit.RenderedMessage{{Role: "user", Content: "history"}}}) + if err != nil { + t.Fatal(err) + } + secondHistory, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, AppendedMessages: []promptkit.RenderedMessage{{Role: "assistant", Content: "history"}}}) + if err != nil { + t.Fatal(err) + } + request.ContinueFrom = firstHistory.Exchange + firstHash := prepareContentHash(t, engine, request).MessageContentHash + request.ContinueFrom = secondHistory.Exchange + if prepareContentHash(t, engine, request).MessageContentHash == firstHash { + t.Fatal("history role change did not affect hash") + } + if len(client.requests) != 3 || len(otherClient.requests) != 1 { + t.Fatal("preparation invoked the model") + } +}