Carry optional prompt labels through results
This commit is contained in:
@@ -128,7 +128,7 @@ func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixtu
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
if prepared.PromptID != "workflow" || prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
|
if prepared.PromptID != "workflow" || prepared.PromptVersion == nil || *prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
|
||||||
prepared.Messages[0].Content != fixture.expectedMessage {
|
prepared.Messages[0].Content != fixture.expectedMessage {
|
||||||
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
|||||||
}
|
}
|
||||||
return &PreparedRun{
|
return &PreparedRun{
|
||||||
PromptID: prepared.PromptID,
|
PromptID: prepared.PromptID,
|
||||||
PromptVersion: prepared.PromptVersion,
|
PromptVersion: copyStringPtr(prepared.PromptVersion),
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
SelectedBackendID: prepared.SelectedBackendID,
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
@@ -99,7 +99,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
|||||||
RawOutput: result.RawOutput,
|
RawOutput: result.RawOutput,
|
||||||
Validation: fromDomainValidationResult(result.Validation),
|
Validation: fromDomainValidationResult(result.Validation),
|
||||||
PromptID: result.PromptID,
|
PromptID: result.PromptID,
|
||||||
PromptVersion: result.PromptVersion,
|
PromptVersion: copyStringPtr(result.PromptVersion),
|
||||||
PromptHash: result.PromptHash,
|
PromptHash: result.PromptHash,
|
||||||
SessionID: result.SessionID,
|
SessionID: result.SessionID,
|
||||||
RenderedPromptHash: result.RenderedPromptHash,
|
RenderedPromptHash: result.RenderedPromptHash,
|
||||||
@@ -257,7 +257,7 @@ func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInsp
|
|||||||
}
|
}
|
||||||
return &PromptInspection{
|
return &PromptInspection{
|
||||||
PromptID: inspection.PromptID,
|
PromptID: inspection.PromptID,
|
||||||
PromptVersion: inspection.PromptVersion,
|
PromptVersion: copyStringPtr(inspection.PromptVersion),
|
||||||
PromptHash: inspection.PromptHash,
|
PromptHash: inspection.PromptHash,
|
||||||
DefaultProfileID: inspection.DefaultProfileID,
|
DefaultProfileID: inspection.DefaultProfileID,
|
||||||
Inputs: inputs,
|
Inputs: inputs,
|
||||||
|
|||||||
@@ -444,6 +444,14 @@ Validation: domain, promptdef, configuration, observation, usecase, and root tes
|
|||||||
`GOWORK=off go test ./...` and the offline examples. Verify the existing selected
|
`GOWORK=off go test ./...` and the offline examples. Verify the existing selected
|
||||||
definition hash still agrees between inspection and preparation.
|
definition hash still agrees between inspection and preparation.
|
||||||
|
|
||||||
|
Completion: prompt labels are presence-aware from configuration capture through
|
||||||
|
inspection, listing, requirements, prepared and completed runs, observation
|
||||||
|
events, and smoke targets. Nil and declared-empty labels remain distinct in JSON;
|
||||||
|
null overlays clear inherited labels. Returned label pointers are independently
|
||||||
|
owned, and inspection/preparation retain matching definition hashes. Validation
|
||||||
|
passed: `GOWORK=off go test ./...`, `GOWORK=off go vet ./...`,
|
||||||
|
`GOWORK=off python3 scripts/check-examples.py`, and `git diff --check`.
|
||||||
|
|
||||||
## Stage 7: Enable Shared Specification Checking
|
## Stage 7: Enable Shared Specification Checking
|
||||||
|
|
||||||
Scope: root declaration policy through ConfigKit's released version API.
|
Scope: root declaration policy through ConfigKit's released version API.
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
|
|
||||||
type summary struct {
|
type summary struct {
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version"`
|
PromptVersion *string `json:"prompt_version,omitempty"`
|
||||||
SelectedProfile string `json:"selected_profile"`
|
SelectedProfile string `json:"selected_profile"`
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
MessageCount int `json:"message_count"`
|
MessageCount int `json:"message_count"`
|
||||||
|
|||||||
@@ -228,7 +228,7 @@ func BenchmarkInspectionSelection(b *testing.B) {
|
|||||||
}
|
}
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
|
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
|
||||||
if err != nil || definition.ID != "inspection" || definition.Version != "1" || hash == "" ||
|
if err != nil || definition.ID != "inspection" || definition.Version == nil || *definition.Version != "1" || hash == "" ||
|
||||||
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
|
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
|
||||||
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
|
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
|
||||||
}
|
}
|
||||||
@@ -238,7 +238,7 @@ func BenchmarkInspectionSelection(b *testing.B) {
|
|||||||
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
||||||
}
|
}
|
||||||
inspection, err := result.GetPromptInspection(ctx, "inspection")
|
inspection, err := result.GetPromptInspection(ctx, "inspection")
|
||||||
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
|
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" ||
|
||||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
||||||
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,9 @@ func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity,
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
||||||
|
for index := range prompts {
|
||||||
|
prompts[index].Version = stringPointerValue(prompts[index].Version)
|
||||||
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -45,9 +45,9 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
wantPrompts := []domain.PromptIdentity{
|
wantPrompts := []domain.PromptIdentity{
|
||||||
{ID: " z/prompt ", Version: "v1"},
|
{ID: " z/prompt ", Version: stringPointer("v1")},
|
||||||
{ID: "a/prompt", Version: "v1"},
|
{ID: "a/prompt", Version: stringPointer("v1")},
|
||||||
{ID: "z/prompt", Version: "v1"},
|
{ID: "z/prompt", Version: stringPointer("v1")},
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||||
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
|
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
|
||||||
@@ -75,7 +75,7 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
prompts[0].ID = "changed prompt"
|
prompts[0].ID = "changed prompt"
|
||||||
prompts[0].Version = "changed version"
|
*prompts[0].Version = "changed version"
|
||||||
|
|
||||||
profiles, err = state.ListProfiles(context.Background())
|
profiles, err = state.ListProfiles(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -88,7 +88,7 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
wantPrompts := []domain.PromptIdentity{{ID: "example", Version: "v1"}}
|
wantPrompts := []domain.PromptIdentity{{ID: "example", Version: stringPointer("v1")}}
|
||||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||||
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
|
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -181,7 +181,7 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
|||||||
}
|
}
|
||||||
state.profileIDs = sortedKeys(state.indexedProfiles)
|
state.profileIDs = sortedKeys(state.indexedProfiles)
|
||||||
for _, id := range sortedKeys(state.prompts) {
|
for _, id := range sortedKeys(state.prompts) {
|
||||||
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id, Version: state.prompts[id].definition.Version})
|
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id, Version: stringPointerValue(state.prompts[id].definition.Version)})
|
||||||
}
|
}
|
||||||
return state, nil
|
return state, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -96,11 +96,7 @@ func (state *loaded) materializePrompts(prompts map[string]promptConfig, basePat
|
|||||||
if strings.TrimSpace(id) == "" {
|
if strings.TrimSpace(id) == "" {
|
||||||
return fmt.Errorf("%s: prompt ID must not be blank", promptBase)
|
return fmt.Errorf("%s: prompt ID must not be blank", promptBase)
|
||||||
}
|
}
|
||||||
version := ""
|
entry, err := state.materializePrompt(id, prompts[id].Version, prompts[id], promptBase, lookupOrigin)
|
||||||
if prompts[id].Version != nil {
|
|
||||||
version = *prompts[id].Version
|
|
||||||
}
|
|
||||||
entry, err := state.materializePrompt(id, version, prompts[id], promptBase, lookupOrigin)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -109,11 +105,11 @@ func (state *loaded) materializePrompts(prompts map[string]promptConfig, basePat
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) materializePrompt(id, version string, raw promptConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
|
func (state *loaded) materializePrompt(id string, version *string, raw promptConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
|
||||||
if len(raw.Messages) == 0 {
|
if len(raw.Messages) == 0 {
|
||||||
return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages"))
|
return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages"))
|
||||||
}
|
}
|
||||||
definition := domain.PromptDefinition{ID: id, Version: version}
|
definition := domain.PromptDefinition{ID: id, Version: stringPointerValue(version)}
|
||||||
if raw.DefaultProfile != nil {
|
if raw.DefaultProfile != nil {
|
||||||
if strings.TrimSpace(*raw.DefaultProfile) == "" {
|
if strings.TrimSpace(*raw.DefaultProfile) == "" {
|
||||||
return nil, fmt.Errorf("%s: default profile must not be blank", base.Key("default_profile"))
|
return nil, fmt.Errorf("%s: default profile must not be blank", base.Key("default_profile"))
|
||||||
@@ -242,7 +238,7 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, bas
|
|||||||
entry.definition = definition
|
entry.definition = definition
|
||||||
entry.hash = hash
|
entry.hash = hash
|
||||||
entry.inspection = domain.PromptInspection{
|
entry.inspection = domain.PromptInspection{
|
||||||
PromptID: id, PromptVersion: version, PromptHash: hash,
|
PromptID: id, PromptVersion: stringPointerValue(version), PromptHash: hash,
|
||||||
DefaultProfileID: definition.DefaultProfile, Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
DefaultProfileID: definition.DefaultProfile, Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
||||||
OutputContract: contract,
|
OutputContract: contract,
|
||||||
}
|
}
|
||||||
@@ -254,6 +250,13 @@ func stringPointer(value string) *string {
|
|||||||
return ©
|
return ©
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func stringPointerValue(value *string) *string {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return stringPointer(*value)
|
||||||
|
}
|
||||||
|
|
||||||
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
|
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
|
||||||
if strings.TrimSpace(name) == "" {
|
if strings.TrimSpace(name) == "" {
|
||||||
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
|
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
|
||||||
@@ -298,6 +301,7 @@ func (state *loaded) lookupPromptEntry(id string) (*promptEntry, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
|
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
|
||||||
|
definition.Version = stringPointerValue(definition.Version)
|
||||||
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
|
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
|
||||||
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
|
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
|
||||||
for index := range definition.Templates {
|
for index := range definition.Templates {
|
||||||
@@ -310,6 +314,7 @@ func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefin
|
|||||||
}
|
}
|
||||||
|
|
||||||
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
|
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
|
||||||
|
inspection.PromptVersion = stringPointerValue(inspection.PromptVersion)
|
||||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||||
return &inspection
|
return &inspection
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -131,11 +131,112 @@ func TestLoadRejectsMalformedStaticSyntaxAndCapturesOptionalVersion(t *testing.T
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition, inspection, err := state.prompt("example")
|
definition, inspection, err := state.prompt("example")
|
||||||
if err != nil || definition.Version != "release label" || inspection.PromptVersion != "release label" {
|
if err != nil || definition.Version == nil || *definition.Version != "release label" ||
|
||||||
|
inspection.PromptVersion == nil || *inspection.PromptVersion != "release label" {
|
||||||
t.Fatalf("optional version metadata was not retained: definition=%+v inspection=%+v err=%v", definition, inspection, err)
|
t.Fatalf("optional version metadata was not retained: definition=%+v inspection=%+v err=%v", definition, inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptVersionLabelPresenceAndOwnership(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
version any
|
||||||
|
include bool
|
||||||
|
wantPresent bool
|
||||||
|
wantValue string
|
||||||
|
}{
|
||||||
|
{name: "absent"},
|
||||||
|
{name: "null", include: true},
|
||||||
|
{name: "empty", version: "", include: true, wantPresent: true},
|
||||||
|
{name: "exact text", version: " Release/v1 ", include: true, wantPresent: true, wantValue: " Release/v1 "},
|
||||||
|
}
|
||||||
|
|
||||||
|
hashes := make(map[string]string, len(tests))
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
definition := map[string]any{
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}
|
||||||
|
if test.include {
|
||||||
|
definition["version"] = test.version
|
||||||
|
}
|
||||||
|
state, err := load(mapping("prompt version label", map[string]any{
|
||||||
|
"prompts": map[string]any{"example": definition},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
captured, inspection, err := state.prompt("example")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
identities, err := state.ListPrompts(context.Background())
|
||||||
|
if err != nil || len(identities) != 1 {
|
||||||
|
t.Fatalf("list prompt labels: %#v, %v", identities, err)
|
||||||
|
}
|
||||||
|
check := func(name string, value *string) {
|
||||||
|
t.Helper()
|
||||||
|
if !test.wantPresent {
|
||||||
|
if value != nil {
|
||||||
|
t.Errorf("%s label = %q, want absent", name, *value)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if value == nil || *value != test.wantValue {
|
||||||
|
t.Errorf("%s label = %v, want %q", name, value, test.wantValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check("definition", captured.Version)
|
||||||
|
check("inspection", inspection.PromptVersion)
|
||||||
|
check("listing", identities[0].Version)
|
||||||
|
if captured.Version != nil && (captured.Version == inspection.PromptVersion || captured.Version == identities[0].Version || inspection.PromptVersion == identities[0].Version) {
|
||||||
|
t.Fatal("returned prompt labels share mutable pointers")
|
||||||
|
}
|
||||||
|
hashes[test.name] = inspection.PromptHash
|
||||||
|
|
||||||
|
if captured.Version != nil {
|
||||||
|
*captured.Version = "mutated definition"
|
||||||
|
}
|
||||||
|
if inspection.PromptVersion != nil {
|
||||||
|
*inspection.PromptVersion = "mutated inspection"
|
||||||
|
}
|
||||||
|
if identities[0].Version != nil {
|
||||||
|
*identities[0].Version = "mutated listing"
|
||||||
|
}
|
||||||
|
again, againInspection, err := state.prompt("example")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
check("captured definition after mutation", again.Version)
|
||||||
|
check("captured inspection after mutation", againInspection.PromptVersion)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if hashes["absent"] == hashes["empty"] {
|
||||||
|
t.Fatal("absent and declared-empty labels produced the same definition hash")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptVersionNullClearsInheritedLabel(t *testing.T) {
|
||||||
|
prompt := map[string]any{
|
||||||
|
"version": "base-label",
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}
|
||||||
|
state, err := load(
|
||||||
|
mapping("clear inherited label", map[string]any{"prompts": map[string]any{"example": map[string]any{"version": nil}}}),
|
||||||
|
[]configkit.Underlay{{Inputs: []configkit.Input{mapping("base prompt", map[string]any{"prompts": map[string]any{"example": prompt}})}}},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
definition, inspection, err := state.prompt("example")
|
||||||
|
if err != nil || definition.Version != nil || inspection.PromptVersion != nil {
|
||||||
|
t.Fatalf("null overlay retained inherited label: definition=%+v inspection=%+v err=%v", definition, inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.T) {
|
func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.T) {
|
||||||
makePrompt := func(content string) map[string]any {
|
makePrompt := func(content string) map[string]any {
|
||||||
return map[string]any{
|
return map[string]any{
|
||||||
@@ -169,7 +270,8 @@ func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("snapshot prompt ID: %v", err)
|
t.Fatalf("snapshot prompt ID: %v", err)
|
||||||
}
|
}
|
||||||
if inspection.PromptID != definition.ID || inspection.PromptVersion != definition.Version ||
|
if inspection.PromptID != definition.ID || inspection.PromptVersion == nil || definition.Version == nil ||
|
||||||
|
*inspection.PromptVersion != *definition.Version ||
|
||||||
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
|
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
|
||||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
|
||||||
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
|
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ type RunResult struct {
|
|||||||
RawOutput string
|
RawOutput string
|
||||||
Validation ValidationResult
|
Validation ValidationResult
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
SessionID string
|
SessionID string
|
||||||
RenderedPromptHash string
|
RenderedPromptHash string
|
||||||
@@ -100,7 +100,7 @@ type RunResult struct {
|
|||||||
// It must never include resolved API key values, model output, or validation data.
|
// It must never include resolved API key values, model output, or validation data.
|
||||||
type PreparedRun struct {
|
type PreparedRun struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
SelectedProfileID string
|
SelectedProfileID string
|
||||||
SelectedBackendID string
|
SelectedBackendID string
|
||||||
@@ -137,13 +137,13 @@ type Artifact struct {
|
|||||||
// PromptIdentity identifies one configured prompt and its descriptive version label.
|
// PromptIdentity identifies one configured prompt and its descriptive version label.
|
||||||
type PromptIdentity struct {
|
type PromptIdentity struct {
|
||||||
ID string
|
ID string
|
||||||
Version string
|
Version *string
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptDefinition represents a configured prompt execution definition.
|
// PromptDefinition represents a configured prompt execution definition.
|
||||||
type PromptDefinition struct {
|
type PromptDefinition struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
Version string `yaml:"version"`
|
Version *string `yaml:"version"`
|
||||||
DefaultProfile string `yaml:"default_profile"`
|
DefaultProfile string `yaml:"default_profile"`
|
||||||
Description string `yaml:"description"`
|
Description string `yaml:"description"`
|
||||||
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
||||||
@@ -156,7 +156,7 @@ type PromptDefinition struct {
|
|||||||
// PromptInspection is the resolved result of exact prompt inspection.
|
// PromptInspection is the resolved result of exact prompt inspection.
|
||||||
type PromptInspection struct {
|
type PromptInspection struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
DefaultProfileID string
|
DefaultProfileID string
|
||||||
Inputs []PromptInput
|
Inputs []PromptInput
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ type ExecutionEvent struct {
|
|||||||
RunID string
|
RunID string
|
||||||
Time time.Time
|
Time time.Time
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
BackendID string
|
BackendID string
|
||||||
Model string
|
Model string
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ type RequirementIssue struct {
|
|||||||
// RequirementInspection contains the inventories discovered for one prompt.
|
// RequirementInspection contains the inventories discovered for one prompt.
|
||||||
type RequirementInspection struct {
|
type RequirementInspection struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
Inputs []InputRequirement
|
Inputs []InputRequirement
|
||||||
Variables []VariableRequirement
|
Variables []VariableRequirement
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ type SmokeTestTarget struct {
|
|||||||
BackendID string
|
BackendID string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
DefaultSourceBackendID string
|
DefaultSourceBackendID string
|
||||||
SelectionSource string
|
SelectionSource string
|
||||||
Endpoint string
|
Endpoint string
|
||||||
@@ -180,6 +180,7 @@ func CloneSmokeTestRequest(request SmokeTestRequest) (SmokeTestRequest, error) {
|
|||||||
func CloneSmokeTestReport(report SmokeTestReport) (SmokeTestReport, error) {
|
func CloneSmokeTestReport(report SmokeTestReport) (SmokeTestReport, error) {
|
||||||
cloned := report
|
cloned := report
|
||||||
cloned.Applicable = cloneBool(report.Applicable)
|
cloned.Applicable = cloneBool(report.Applicable)
|
||||||
|
cloned.Target.PromptVersion = cloneString(report.Target.PromptVersion)
|
||||||
cloned.Facts = cloneSmokeTestFacts(report.Facts)
|
cloned.Facts = cloneSmokeTestFacts(report.Facts)
|
||||||
cloned.Timing = cloneSmokeTestTiming(report.Timing)
|
cloned.Timing = cloneSmokeTestTiming(report.Timing)
|
||||||
var err error
|
var err error
|
||||||
|
|||||||
@@ -77,11 +77,15 @@ func TestCloneSmokeTestReportOwnsMutableValues(t *testing.T) {
|
|||||||
reachable := true
|
reachable := true
|
||||||
status := 200
|
status := 200
|
||||||
wait := 3 * time.Millisecond
|
wait := 3 * time.Millisecond
|
||||||
|
promptVersion := "v1"
|
||||||
report := SmokeTestReport{
|
report := SmokeTestReport{
|
||||||
Applicable: &applicable,
|
Applicable: &applicable,
|
||||||
Target: SmokeTestTarget{EffectiveSettings: ExecutionTarget{
|
Target: SmokeTestTarget{
|
||||||
|
PromptVersion: &promptVersion,
|
||||||
|
EffectiveSettings: ExecutionTarget{
|
||||||
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
||||||
}},
|
},
|
||||||
|
},
|
||||||
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
||||||
Attempts: []SmokeTestAttempt{{
|
Attempts: []SmokeTestAttempt{{
|
||||||
HTTPStatus: &status,
|
HTTPStatus: &status,
|
||||||
@@ -99,9 +103,10 @@ func TestCloneSmokeTestReportOwnsMutableValues(t *testing.T) {
|
|||||||
*cloned.Attempts[0].HTTPStatus = 503
|
*cloned.Attempts[0].HTTPStatus = 503
|
||||||
*cloned.Attempts[0].Facts.EndpointReachable = false
|
*cloned.Attempts[0].Facts.EndpointReachable = false
|
||||||
*cloned.Attempts[0].Timing.CapacityWait = time.Second
|
*cloned.Attempts[0].Timing.CapacityWait = time.Second
|
||||||
|
*cloned.Target.PromptVersion = "changed"
|
||||||
cloned.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
cloned.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||||
|
|
||||||
if !applicable || !reachable || status != 200 || wait != 3*time.Millisecond {
|
if !applicable || !reachable || status != 200 || wait != 3*time.Millisecond || promptVersion != "v1" {
|
||||||
t.Fatal("cloned report shared optional scalar pointers")
|
t.Fatal("cloned report shared optional scalar pointers")
|
||||||
}
|
}
|
||||||
if got := report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
if got := report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ type Recorder struct {
|
|||||||
// Selection contains resolved identities only. Unresolved fields stay empty.
|
// Selection contains resolved identities only. Unresolved fields stay empty.
|
||||||
type Selection struct {
|
type Selection struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
BackendID string
|
BackendID string
|
||||||
Model string
|
Model string
|
||||||
@@ -103,7 +103,10 @@ func (r *Recorder) Start() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// UpdateSelection replaces the known resolved identities for later events.
|
// UpdateSelection replaces the known resolved identities for later events.
|
||||||
func (r *Recorder) UpdateSelection(selection Selection) { r.selection = selection }
|
func (r *Recorder) UpdateSelection(selection Selection) {
|
||||||
|
selection.PromptVersion = copyStringPointer(selection.PromptVersion)
|
||||||
|
r.selection = selection
|
||||||
|
}
|
||||||
|
|
||||||
// Attempt records one completed acquisition and optional dispatch.
|
// Attempt records one completed acquisition and optional dispatch.
|
||||||
func (r *Recorder) Attempt(data AttemptData) {
|
func (r *Recorder) Attempt(data AttemptData) {
|
||||||
@@ -177,11 +180,19 @@ func (r *Recorder) Finish(data FinishData) {
|
|||||||
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
|
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
|
||||||
return domain.ExecutionEvent{
|
return domain.ExecutionEvent{
|
||||||
Kind: kind, RunID: r.runID, Time: r.now(), PromptID: r.selection.PromptID,
|
Kind: kind, RunID: r.runID, Time: r.now(), PromptID: r.selection.PromptID,
|
||||||
PromptVersion: r.selection.PromptVersion, ProfileID: r.selection.ProfileID,
|
PromptVersion: copyStringPointer(r.selection.PromptVersion), ProfileID: r.selection.ProfileID,
|
||||||
BackendID: r.selection.BackendID, Model: r.selection.Model,
|
BackendID: r.selection.BackendID, Model: r.selection.Model,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func copyStringPointer(value *string) *string {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copy := *value
|
||||||
|
return ©
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Recorder) deliver(event domain.ExecutionEvent) {
|
func (r *Recorder) deliver(event domain.ExecutionEvent) {
|
||||||
if r.observer != nil {
|
if r.observer != nil {
|
||||||
r.observer(event)
|
r.observer(event)
|
||||||
|
|||||||
@@ -50,7 +50,8 @@ func TestRecorderAccountsDispatchedAttemptsAndReportedUsage(t *testing.T) {
|
|||||||
var events []domain.ExecutionEvent
|
var events []domain.ExecutionEvent
|
||||||
recorder := newRecorder("run-1", now, func(event domain.ExecutionEvent) { events = append(events, event) })
|
recorder := newRecorder("run-1", now, func(event domain.ExecutionEvent) { events = append(events, event) })
|
||||||
recorder.Start()
|
recorder.Start()
|
||||||
recorder.UpdateSelection(Selection{PromptID: "prompt", PromptVersion: "v1", ProfileID: "profile", BackendID: "backend", Model: "model"})
|
version := "v1"
|
||||||
|
recorder.UpdateSelection(Selection{PromptID: "prompt", PromptVersion: &version, ProfileID: "profile", BackendID: "backend", Model: "model"})
|
||||||
recorder.Attempt(AttemptData{
|
recorder.Attempt(AttemptData{
|
||||||
Generation: 1, Attempt: 1, Dispatched: true,
|
Generation: 1, Attempt: 1, Dispatched: true,
|
||||||
CapacityWait: 2 * time.Millisecond, GenerationDuration: 10 * time.Millisecond,
|
CapacityWait: 2 * time.Millisecond, GenerationDuration: 10 * time.Millisecond,
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ func BenchmarkCompiledRendering(b *testing.B) {
|
|||||||
small := newCompiledRenderingBenchmarkCase(b,
|
small := newCompiledRenderingBenchmarkCase(b,
|
||||||
"small_message", "", []string{"Summary for {{.audience}}."},
|
"small_message", "", []string{"Summary for {{.audience}}."},
|
||||||
&domain.PromptDefinition{
|
&domain.PromptDefinition{
|
||||||
ID: "compiled-render-benchmark", Version: "v1",
|
ID: "compiled-render-benchmark", Version: stringPointerForTest("v1"),
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
|
||||||
}, nil, map[string]string{"audience": "project leads"},
|
}, nil, map[string]string{"audience": "project leads"},
|
||||||
len("Summary for project leads."),
|
len("Summary for project leads."),
|
||||||
@@ -68,7 +68,7 @@ func BenchmarkCompiledRendering(b *testing.B) {
|
|||||||
}
|
}
|
||||||
largeContent := `{{input "document"}}`
|
largeContent := `{{input "document"}}`
|
||||||
largeDefinition := &domain.PromptDefinition{
|
largeDefinition := &domain.PromptDefinition{
|
||||||
ID: "compiled-render-benchmark", Version: "v1",
|
ID: "compiled-render-benchmark", Version: stringPointerForTest("v1"),
|
||||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
|
||||||
}
|
}
|
||||||
@@ -91,7 +91,7 @@ func BenchmarkCompiledRendering(b *testing.B) {
|
|||||||
`{{input "document"}}|{{input "document"}}`,
|
`{{input "document"}}|{{input "document"}}`,
|
||||||
}
|
}
|
||||||
repeatedDefinition := &domain.PromptDefinition{
|
repeatedDefinition := &domain.PromptDefinition{
|
||||||
ID: "compiled-render-benchmark", Version: "v1", SessionID: repeatedSession,
|
ID: "compiled-render-benchmark", Version: stringPointerForTest("v1"), SessionID: repeatedSession,
|
||||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
Templates: []domain.PromptMessageTemplate{
|
Templates: []domain.PromptMessageTemplate{
|
||||||
{Role: "system", Content: repeatedMessages[0]},
|
{Role: "system", Content: repeatedMessages[0]},
|
||||||
@@ -141,7 +141,7 @@ func BenchmarkRetainedRenderedOutput(b *testing.B) {
|
|||||||
bodyText := string(body)
|
bodyText := string(body)
|
||||||
content := `{{input "document"}}`
|
content := `{{input "document"}}`
|
||||||
definition := &domain.PromptDefinition{
|
definition := &domain.PromptDefinition{
|
||||||
ID: "retained-render-benchmark", Version: "v1",
|
ID: "retained-render-benchmark", Version: stringPointerForTest("v1"),
|
||||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -525,7 +525,7 @@ func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
definition := &domain.PromptDefinition{
|
definition := &domain.PromptDefinition{
|
||||||
ID: promptID, Version: version, SessionID: "session-{{.request}}",
|
ID: promptID, Version: stringPointerForTest(version), SessionID: "session-{{.request}}",
|
||||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: `request={{.request}};document={{input "document"}}`}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: `request={{.request}};document={{input "document"}}`}},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -384,7 +384,7 @@ func AnalyzeRequirements(
|
|||||||
result := domain.RequirementInspection{Complete: true}
|
result := domain.RequirementInspection{Complete: true}
|
||||||
if definition != nil {
|
if definition != nil {
|
||||||
result.PromptID = definition.ID
|
result.PromptID = definition.ID
|
||||||
result.PromptVersion = definition.Version
|
result.PromptVersion = copyString(definition.Version)
|
||||||
}
|
}
|
||||||
walker := &requirementWalker{
|
walker := &requirementWalker{
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
|
|||||||
@@ -108,7 +108,7 @@ func TestRequirementWalkerDoesNotRescanNestedUnsupportedNames(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
|
func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
|
||||||
description := "retained metadata"
|
description := "retained metadata"
|
||||||
definition := &domain.PromptDefinition{ID: "capture", Version: "v1"}
|
definition := &domain.PromptDefinition{ID: "capture", Version: stringPointerForTest("v1")}
|
||||||
snapshot := RequirementSnapshot{
|
snapshot := RequirementSnapshot{
|
||||||
Inputs: []InputMetadata{
|
Inputs: []InputMetadata{
|
||||||
{Name: "optional", ContentType: stringPointerForTest("text/plain"), Description: &description},
|
{Name: "optional", ContentType: stringPointerForTest("text/plain"), Description: &description},
|
||||||
@@ -123,7 +123,7 @@ func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if result.PromptID != "capture" || result.PromptVersion != "v1" || !result.Complete || len(result.Issues) != 0 {
|
if result.PromptID != "capture" || result.PromptVersion == nil || *result.PromptVersion != "v1" || !result.Complete || len(result.Issues) != 0 {
|
||||||
t.Fatalf("unexpected result identity or completeness: %+v", result)
|
t.Fatalf("unexpected result identity or completeness: %+v", result)
|
||||||
}
|
}
|
||||||
wantNames := []string{"extra", "optional", "transcript", "unused"}
|
wantNames := []string{"extra", "optional", "transcript", "unused"}
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ func (r *Runner) resolvePromptInspection(ctx context.Context, promptID string) (
|
|||||||
definition := selection.definition
|
definition := selection.definition
|
||||||
return &domain.PromptInspection{
|
return &domain.PromptInspection{
|
||||||
PromptID: definition.ID,
|
PromptID: definition.ID,
|
||||||
PromptVersion: definition.Version,
|
PromptVersion: copyStringPointer(definition.Version),
|
||||||
PromptHash: selection.hash,
|
PromptHash: selection.hash,
|
||||||
DefaultProfileID: definition.DefaultProfile,
|
DefaultProfileID: definition.DefaultProfile,
|
||||||
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ func (r *promptMetadataRepository) GetPromptInspection(
|
|||||||
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
||||||
definition := &domain.PromptDefinition{
|
definition := &domain.PromptDefinition{
|
||||||
ID: "normalized.prompt",
|
ID: "normalized.prompt",
|
||||||
Version: "1.2.3",
|
Version: stringPtr("1.2.3"),
|
||||||
DefaultProfile: "not-resolved",
|
DefaultProfile: "not-resolved",
|
||||||
Inputs: []domain.PromptInput{
|
Inputs: []domain.PromptInput{
|
||||||
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
|
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
|
||||||
@@ -79,8 +79,8 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
if repository.calls != 1 || repository.id != " prompt-id " {
|
if repository.calls != 1 || repository.id != " prompt-id " {
|
||||||
t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id)
|
t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id)
|
||||||
}
|
}
|
||||||
if inspection.PromptID != definition.ID ||
|
if inspection.PromptID != definition.ID || inspection.PromptVersion == nil || definition.Version == nil ||
|
||||||
inspection.PromptVersion != definition.Version ||
|
*inspection.PromptVersion != *definition.Version ||
|
||||||
inspection.PromptHash != wantHash ||
|
inspection.PromptHash != wantHash ||
|
||||||
inspection.DefaultProfileID != definition.DefaultProfile ||
|
inspection.DefaultProfileID != definition.DefaultProfile ||
|
||||||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
||||||
@@ -100,7 +100,7 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
|
|
||||||
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
|
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
|
||||||
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
||||||
PromptID: "captured.prompt", PromptVersion: "v2", PromptHash: "cached-prompt-hash",
|
PromptID: "captured.prompt", PromptVersion: stringPtr("v2"), PromptHash: "cached-prompt-hash",
|
||||||
DefaultProfileID: "partial-profile",
|
DefaultProfileID: "partial-profile",
|
||||||
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
|
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
|
||||||
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
|
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
|
||||||
@@ -108,7 +108,7 @@ func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing
|
|||||||
runner := &Runner{promptDefs: repository}
|
runner := &Runner{promptDefs: repository}
|
||||||
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ")
|
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ")
|
||||||
if err != nil || repository.lookupID != " requested-id " ||
|
if err != nil || repository.lookupID != " requested-id " ||
|
||||||
inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v2" ||
|
inspection.PromptID != "captured.prompt" || inspection.PromptVersion == nil || *inspection.PromptVersion != "v2" ||
|
||||||
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
|
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
|
||||||
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
|
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
|
||||||
t.Fatalf("metadata inspection=(%+v, %v), lookup=%q", inspection, err, repository.lookupID)
|
t.Fatalf("metadata inspection=(%+v, %v), lookup=%q", inspection, err, repository.lookupID)
|
||||||
@@ -124,11 +124,11 @@ func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing
|
|||||||
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
|
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
|
||||||
repository := &requirementInspectionPromptRepository{
|
repository := &requirementInspectionPromptRepository{
|
||||||
definition: &domain.PromptDefinition{
|
definition: &domain.PromptDefinition{
|
||||||
ID: "captured.prompt", Version: "v1", Inputs: []domain.PromptInput{{Name: "document"}},
|
ID: "captured.prompt", Version: stringPtr("v1"), Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ")
|
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ")
|
||||||
if err != nil || inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v1" ||
|
if err != nil || inspection.PromptID != "captured.prompt" || inspection.PromptVersion == nil || *inspection.PromptVersion != "v1" ||
|
||||||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
||||||
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
|
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ func (r *Runner) InspectRequirements(
|
|||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: selection.PromptID, Version: selection.PromptVersion}
|
definition := &domain.PromptDefinition{ID: selection.PromptID, Version: copyStringPointer(selection.PromptVersion)}
|
||||||
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ func TestRunnerInspectRequirementsUsesCapturedSelectionWithoutExecutionWork(t *t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v2", DefaultProfile: "default-profile"}
|
definition := &domain.PromptDefinition{ID: "selected", Version: stringPtr("v2"), DefaultProfile: "default-profile"}
|
||||||
repository := &requirementInspectionPromptRepository{
|
repository := &requirementInspectionPromptRepository{
|
||||||
definition: definition,
|
definition: definition,
|
||||||
snapshotFound: true,
|
snapshotFound: true,
|
||||||
@@ -113,7 +113,7 @@ func TestRunnerInspectRequirementsUsesCapturedSelectionWithoutExecutionWork(t *t
|
|||||||
repository.snapshotCalls, repository.snapshotID)
|
repository.snapshotCalls, repository.snapshotID)
|
||||||
}
|
}
|
||||||
if profiles.calls != 1 || profiles.id != "partial-profile" || inspection.PromptID != definition.ID ||
|
if profiles.calls != 1 || profiles.id != "partial-profile" || inspection.PromptID != definition.ID ||
|
||||||
inspection.PromptVersion != definition.Version || inspection.ProfileID != "partial-profile" || !inspection.Complete ||
|
inspection.PromptVersion == nil || *inspection.PromptVersion != *definition.Version || inspection.ProfileID != "partial-profile" || !inspection.Complete ||
|
||||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" || len(inspection.Variables) != 1 ||
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" || len(inspection.Variables) != 1 ||
|
||||||
inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
||||||
t.Fatalf("unexpected requirement inspection: %+v", inspection)
|
t.Fatalf("unexpected requirement inspection: %+v", inspection)
|
||||||
@@ -138,7 +138,7 @@ func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(
|
|||||||
promptRepository := &requirementPromptMetadataRepository{
|
promptRepository := &requirementPromptMetadataRepository{
|
||||||
requirementInspectionPromptRepository: baseRepository,
|
requirementInspectionPromptRepository: baseRepository,
|
||||||
inspection: domain.PromptInspection{
|
inspection: domain.PromptInspection{
|
||||||
PromptID: "captured-id", PromptVersion: "captured-version", PromptHash: "captured-hash",
|
PromptID: "captured-id", PromptVersion: stringPtr("captured-version"), PromptHash: "captured-hash",
|
||||||
DefaultProfileID: "default-profile",
|
DefaultProfileID: "default-profile",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -155,7 +155,7 @@ func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(
|
|||||||
if err != nil || promptRepository.lookupID != " requested-id " ||
|
if err != nil || promptRepository.lookupID != " requested-id " ||
|
||||||
baseRepository.snapshotID != "captured-id" ||
|
baseRepository.snapshotID != "captured-id" ||
|
||||||
profiles.identityID != "default-profile" || inspection.PromptID != "captured-id" ||
|
profiles.identityID != "default-profile" || inspection.PromptID != "captured-id" ||
|
||||||
inspection.PromptVersion != "captured-version" || inspection.ProfileID != "default-profile" ||
|
inspection.PromptVersion == nil || *inspection.PromptVersion != "captured-version" || inspection.ProfileID != "default-profile" ||
|
||||||
!inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" ||
|
!inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" ||
|
||||||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
||||||
t.Fatalf("metadata-based requirement inspection=(%+v, %v), prompt lookup=%q, snapshot=%q, profile=%q",
|
t.Fatalf("metadata-based requirement inspection=(%+v, %v), prompt lookup=%q, snapshot=%q, profile=%q",
|
||||||
@@ -168,7 +168,7 @@ func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *tes
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v1"}
|
definition := &domain.PromptDefinition{ID: "selected", Version: stringPtr("v1")}
|
||||||
makePromptRepository := func() *requirementInspectionPromptRepository {
|
makePromptRepository := func() *requirementInspectionPromptRepository {
|
||||||
return &requirementInspectionPromptRepository{
|
return &requirementInspectionPromptRepository{
|
||||||
definition: definition, snapshotFound: true,
|
definition: definition, snapshotFound: true,
|
||||||
@@ -234,7 +234,7 @@ func TestRunnerInspectRequirementsUsesPromptDefaultAndIncludesSessionWhenNotBypa
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "default-profile"}
|
definition := &domain.PromptDefinition{ID: "selected", Version: stringPtr("v1"), DefaultProfile: "default-profile"}
|
||||||
repository := &requirementInspectionPromptRepository{
|
repository := &requirementInspectionPromptRepository{
|
||||||
definition: definition, snapshotFound: true,
|
definition: definition, snapshotFound: true,
|
||||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}, Session: session}},
|
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}, Session: session}},
|
||||||
@@ -260,7 +260,7 @@ func TestRunnerInspectRequirementsReturnsIncompleteDiscoveryAsSuccess(t *testing
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
repository := &requirementInspectionPromptRepository{
|
repository := &requirementInspectionPromptRepository{
|
||||||
definition: &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "profile"},
|
definition: &domain.PromptDefinition{ID: "selected", Version: stringPtr("v1"), DefaultProfile: "profile"},
|
||||||
snapshotFound: true,
|
snapshotFound: true,
|
||||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{compiled}}},
|
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{compiled}}},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T)
|
|||||||
Temperature: 0, ExtraParams: map[string]any{"configured": true},
|
Temperature: 0, ExtraParams: map[string]any{"configured": true},
|
||||||
}, TargetPresence: domain.ExecutionTargetPresence{Temperature: true}}
|
}, TargetPresence: domain.ExecutionTargetPresence{Temperature: true}}
|
||||||
runner := &Runner{
|
runner := &Runner{
|
||||||
promptDefs: staticPromptRepo{def: &domain.PromptDefinition{ID: "prompt", Version: "v1", Validation: domain.OutputContract{
|
promptDefs: staticPromptRepo{def: &domain.PromptDefinition{ID: "prompt", Version: stringPtr("v1"), Validation: domain.OutputContract{
|
||||||
Format: domain.FormatText, ValidationMode: domain.ValidationNone,
|
Format: domain.FormatText, ValidationMode: domain.ValidationNone,
|
||||||
}}},
|
}}},
|
||||||
profiles: staticExecutionProfileRepo{profile: profile}, renderer: staticRenderer{},
|
profiles: staticExecutionProfileRepo{profile: profile}, renderer: staticRenderer{},
|
||||||
@@ -63,7 +63,7 @@ func TestResolvedTargetCredentialAvailability(t *testing.T) {
|
|||||||
Endpoint: "https://provider.example/v1", Model: "model", APIKeyRequired: true,
|
Endpoint: "https://provider.example/v1", Model: "model", APIKeyRequired: true,
|
||||||
}}
|
}}
|
||||||
runner := &Runner{
|
runner := &Runner{
|
||||||
promptDefs: staticPromptRepo{def: &domain.PromptDefinition{ID: "prompt", Version: "v1", Validation: domain.OutputContract{
|
promptDefs: staticPromptRepo{def: &domain.PromptDefinition{ID: "prompt", Version: stringPtr("v1"), Validation: domain.OutputContract{
|
||||||
Format: domain.FormatText, ValidationMode: domain.ValidationNone,
|
Format: domain.FormatText, ValidationMode: domain.ValidationNone,
|
||||||
}}},
|
}}},
|
||||||
profiles: staticExecutionProfileRepo{profile: profile}, renderer: staticRenderer{},
|
profiles: staticExecutionProfileRepo{profile: profile}, renderer: staticRenderer{},
|
||||||
|
|||||||
@@ -252,7 +252,7 @@ func (r *Runner) executePreparedRun(
|
|||||||
RawOutput: genResp.Content,
|
RawOutput: genResp.Content,
|
||||||
Validation: validationResult,
|
Validation: validationResult,
|
||||||
PromptID: prepared.PromptID,
|
PromptID: prepared.PromptID,
|
||||||
PromptVersion: prepared.PromptVersion,
|
PromptVersion: copyStringPointer(prepared.PromptVersion),
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
SessionID: prepared.SessionID,
|
SessionID: prepared.SessionID,
|
||||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||||
@@ -321,7 +321,7 @@ func (r *Runner) resolvePreparation(
|
|||||||
}
|
}
|
||||||
def := promptSelection.definition
|
def := promptSelection.definition
|
||||||
if recorder := observation.FromContext(ctx); recorder != nil {
|
if recorder := observation.FromContext(ctx); recorder != nil {
|
||||||
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: def.Version})
|
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: copyStringPointer(def.Version)})
|
||||||
}
|
}
|
||||||
promptDefinitionHash := promptSelection.hash
|
promptDefinitionHash := promptSelection.hash
|
||||||
effectiveContract, err := resolveOutputContract(def, req.Validation)
|
effectiveContract, err := resolveOutputContract(def, req.Validation)
|
||||||
@@ -342,7 +342,7 @@ func (r *Runner) resolvePreparation(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if recorder := observation.FromContext(ctx); recorder != nil {
|
if recorder := observation.FromContext(ctx); recorder != nil {
|
||||||
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: def.Version, ProfileID: selection.id})
|
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: copyStringPointer(def.Version), ProfileID: selection.id})
|
||||||
}
|
}
|
||||||
|
|
||||||
effectiveModel, targetPresence := resolveExecutionTarget(selection.profile, req.Execution)
|
effectiveModel, targetPresence := resolveExecutionTarget(selection.profile, req.Execution)
|
||||||
@@ -355,7 +355,7 @@ func (r *Runner) resolvePreparation(
|
|||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
if recorder := observation.FromContext(ctx); recorder != nil {
|
if recorder := observation.FromContext(ctx); recorder != nil {
|
||||||
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: def.Version,
|
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: copyStringPointer(def.Version),
|
||||||
ProfileID: selection.id, BackendID: effectiveModel.BackendID, Model: effectiveModel.Model})
|
ProfileID: selection.id, BackendID: effectiveModel.BackendID, Model: effectiveModel.Model})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -481,7 +481,7 @@ func (r *Runner) completePreparationWithStructuredOutput(
|
|||||||
effectiveModel.APIKey = ""
|
effectiveModel.APIKey = ""
|
||||||
return &domain.PreparedRun{
|
return &domain.PreparedRun{
|
||||||
PromptID: state.definition.ID,
|
PromptID: state.definition.ID,
|
||||||
PromptVersion: state.definition.Version,
|
PromptVersion: copyStringPointer(state.definition.Version),
|
||||||
PromptHash: state.promptDefinitionHash,
|
PromptHash: state.promptDefinitionHash,
|
||||||
SelectedProfileID: state.selectedProfileID,
|
SelectedProfileID: state.selectedProfileID,
|
||||||
SelectedBackendID: state.effectiveModel.BackendID,
|
SelectedBackendID: state.effectiveModel.BackendID,
|
||||||
@@ -713,3 +713,11 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti
|
|||||||
func hashPromptDefinition(def *domain.PromptDefinition) (string, error) {
|
func hashPromptDefinition(def *domain.PromptDefinition) (string, error) {
|
||||||
return promptdef.HashDefinition(def)
|
return promptdef.HashDefinition(def)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func copyStringPointer(value *string) *string {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copy := *value
|
||||||
|
return ©
|
||||||
|
}
|
||||||
|
|||||||
@@ -366,7 +366,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
if prepared.PromptID != "p" || prepared.PromptVersion != "1" {
|
if prepared.PromptID != "p" || prepared.PromptVersion == nil || *prepared.PromptVersion != "1" {
|
||||||
t.Fatalf("unexpected prepared prompt metadata: %+v", prepared)
|
t.Fatalf("unexpected prepared prompt metadata: %+v", prepared)
|
||||||
}
|
}
|
||||||
if prepared.SelectedProfileID != "exec" {
|
if prepared.SelectedProfileID != "exec" {
|
||||||
@@ -1257,7 +1257,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
if res.PromptID != "p" || res.PromptVersion != "1" {
|
if res.PromptID != "p" || res.PromptVersion == nil || *res.PromptVersion != "1" {
|
||||||
t.Fatalf("unexpected prompt metadata: %+v", res)
|
t.Fatalf("unexpected prompt metadata: %+v", res)
|
||||||
}
|
}
|
||||||
if res.SelectedProfileID != "exec" {
|
if res.SelectedProfileID != "exec" {
|
||||||
@@ -2504,7 +2504,7 @@ func TestBuildOutputArtifactDefaults(t *testing.T) {
|
|||||||
func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts int) *domain.PromptDefinition {
|
func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts int) *domain.PromptDefinition {
|
||||||
return &domain.PromptDefinition{
|
return &domain.PromptDefinition{
|
||||||
ID: "p",
|
ID: "p",
|
||||||
Version: "1",
|
Version: stringPtr("1"),
|
||||||
DefaultProfile: "exec",
|
DefaultProfile: "exec",
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
|
||||||
|
|||||||
@@ -12,6 +12,10 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func stringPointerForTest(value string) *string {
|
||||||
|
return &value
|
||||||
|
}
|
||||||
|
|
||||||
func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
|
func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
|
||||||
value := promptkit.GenerateResponse{
|
value := promptkit.GenerateResponse{
|
||||||
Content: "ok",
|
Content: "ok",
|
||||||
@@ -42,6 +46,45 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptVersionJSONDistinguishesAbsenceAndDeclaredEmpty(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value *string
|
||||||
|
wantPresent bool
|
||||||
|
wantValue string
|
||||||
|
}{
|
||||||
|
{name: "absent"},
|
||||||
|
{name: "declared empty", value: stringPointerForTest(""), wantPresent: true},
|
||||||
|
{name: "declared text", value: stringPointerForTest("release/v1"), wantPresent: true, wantValue: "release/v1"},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
check := func(label string, marshal func() ([]byte, error)) {
|
||||||
|
t.Helper()
|
||||||
|
payload, err := marshal()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
object := decodeJSONObject(t, payload)
|
||||||
|
value, present := object["prompt_version"]
|
||||||
|
if present != test.wantPresent {
|
||||||
|
t.Errorf("%s prompt_version present=%t, want %t: %s", label, present, test.wantPresent, payload)
|
||||||
|
}
|
||||||
|
var decoded string
|
||||||
|
if test.wantPresent && (json.Unmarshal(value, &decoded) != nil || decoded != test.wantValue) {
|
||||||
|
t.Errorf("%s prompt_version=%#v, want %q", label, value, test.wantValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check("prepared", func() ([]byte, error) {
|
||||||
|
return json.Marshal(promptkit.PreparedRun{PromptVersion: test.value})
|
||||||
|
})
|
||||||
|
check("result", func() ([]byte, error) {
|
||||||
|
return json.Marshal(promptkit.RunResult{PromptVersion: test.value})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPreparedRunJSONContractRoundTripsAllFields(t *testing.T) {
|
func TestPreparedRunJSONContractRoundTripsAllFields(t *testing.T) {
|
||||||
value := fullyPopulatedPreparedRun()
|
value := fullyPopulatedPreparedRun()
|
||||||
payload, err := json.Marshal(value)
|
payload, err := json.Marshal(value)
|
||||||
@@ -216,7 +259,7 @@ func fullyPopulatedPreparedRun() promptkit.PreparedRun {
|
|||||||
start := time.Date(2026, time.August, 11, 12, 13, 14, 150_000_000, time.UTC)
|
start := time.Date(2026, time.August, 11, 12, 13, 14, 150_000_000, time.UTC)
|
||||||
return promptkit.PreparedRun{
|
return promptkit.PreparedRun{
|
||||||
PromptID: "prompt.prepared",
|
PromptID: "prompt.prepared",
|
||||||
PromptVersion: "2.1.0",
|
PromptVersion: stringPointerForTest("2.1.0"),
|
||||||
PromptHash: "prompt-hash",
|
PromptHash: "prompt-hash",
|
||||||
SelectedProfileID: "profile-prepared",
|
SelectedProfileID: "profile-prepared",
|
||||||
SelectedBackendID: "backend-prepared",
|
SelectedBackendID: "backend-prepared",
|
||||||
@@ -283,7 +326,7 @@ func fullyPopulatedRunResult() promptkit.RunResult {
|
|||||||
IsValid: false,
|
IsValid: false,
|
||||||
},
|
},
|
||||||
PromptID: "prompt.result",
|
PromptID: "prompt.result",
|
||||||
PromptVersion: "3.2.1",
|
PromptVersion: stringPointerForTest("3.2.1"),
|
||||||
PromptHash: "result-prompt-hash",
|
PromptHash: "result-prompt-hash",
|
||||||
SessionID: "session-result",
|
SessionID: "session-result",
|
||||||
RenderedPromptHash: "result-rendered-hash",
|
RenderedPromptHash: "result-rendered-hash",
|
||||||
|
|||||||
11
listing.go
11
listing.go
@@ -7,10 +7,11 @@ import (
|
|||||||
|
|
||||||
// PromptIdentity identifies one prompt ID and its optional descriptive version
|
// PromptIdentity identifies one prompt ID and its optional descriptive version
|
||||||
// label in the effective configuration captured when an Engine is constructed.
|
// label in the effective configuration captured when an Engine is constructed.
|
||||||
// It has no stable JSON representation.
|
// A nil Version means no label was declared; a non-nil empty string is an
|
||||||
|
// explicit label. It has no stable JSON representation.
|
||||||
type PromptIdentity struct {
|
type PromptIdentity struct {
|
||||||
ID string
|
ID string
|
||||||
Version string
|
Version *string
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListProfiles returns profile IDs in the effective configuration captured
|
// ListProfiles returns profile IDs in the effective configuration captured
|
||||||
@@ -38,8 +39,8 @@ func (e *Engine) ListProfiles(ctx context.Context) ([]string, error) {
|
|||||||
|
|
||||||
// ListPrompts returns every prompt ID in the effective configuration captured
|
// ListPrompts returns every prompt ID in the effective configuration captured
|
||||||
// when the engine was constructed, sorted lexically by ID. Each identity may
|
// when the engine was constructed, sorted lexically by ID. Each identity may
|
||||||
// include its descriptive version label. The
|
// include its descriptive version label. The returned slice and each Version
|
||||||
// returned slice is caller-owned and safe to mutate while the engine is used
|
// pointer are caller-owned and safe to mutate while the engine is used
|
||||||
// concurrently. Empty inventories succeed.
|
// concurrently. Empty inventories succeed.
|
||||||
//
|
//
|
||||||
// Listing reports membership, not execution readiness. It does not reload
|
// Listing reports membership, not execution readiness. It does not reload
|
||||||
@@ -58,7 +59,7 @@ func (e *Engine) ListPrompts(ctx context.Context) ([]PromptIdentity, error) {
|
|||||||
}
|
}
|
||||||
public := make([]PromptIdentity, len(identities))
|
public := make([]PromptIdentity, len(identities))
|
||||||
for i, identity := range identities {
|
for i, identity := range identities {
|
||||||
public[i] = PromptIdentity{ID: identity.ID, Version: identity.Version}
|
public[i] = PromptIdentity{ID: identity.ID, Version: copyStringPtr(identity.Version)}
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
|
|||||||
@@ -38,13 +38,13 @@ func TestListIdentitiesBasicContract(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
want := []promptkit.PromptIdentity{
|
want := []promptkit.PromptIdentity{
|
||||||
{ID: "example-prompt", Version: "10"},
|
{ID: "example-prompt", Version: stringPointerForTest("10")},
|
||||||
}
|
}
|
||||||
if len(prompts) != len(want) {
|
if len(prompts) != len(want) {
|
||||||
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
|
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
|
||||||
}
|
}
|
||||||
for i := range want {
|
for i := range want {
|
||||||
if prompts[i] != want[i] {
|
if prompts[i].ID != want[i].ID || prompts[i].Version == nil || *prompts[i].Version != *want[i].Version {
|
||||||
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
|
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -54,7 +54,8 @@ func TestListIdentitiesBasicContract(t *testing.T) {
|
|||||||
t.Fatalf("listed profile could not be inspected: (%+v, %v)", profile, err)
|
t.Fatalf("listed profile could not be inspected: (%+v, %v)", profile, err)
|
||||||
}
|
}
|
||||||
prompt, err := engine.InspectPrompt(context.Background(), prompts[0].ID)
|
prompt, err := engine.InspectPrompt(context.Background(), prompts[0].ID)
|
||||||
if err != nil || prompt == nil || prompt.PromptID != prompts[0].ID || prompt.PromptVersion != prompts[0].Version {
|
if err != nil || prompt == nil || prompt.PromptID != prompts[0].ID || prompt.PromptVersion == nil ||
|
||||||
|
*prompt.PromptVersion != *prompts[0].Version {
|
||||||
t.Fatalf("listed prompt could not be inspected: (%+v, %v)", prompt, err)
|
t.Fatalf("listed prompt could not be inspected: (%+v, %v)", prompt, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -169,9 +170,9 @@ func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
wantPrompts := []promptkit.PromptIdentity{
|
wantPrompts := []promptkit.PromptIdentity{
|
||||||
{ID: "main-only", Version: "v3"},
|
{ID: "main-only", Version: stringPointerForTest("v3")},
|
||||||
{ID: "shared/prompt", Version: "v2"},
|
{ID: "shared/prompt", Version: stringPointerForTest("v2")},
|
||||||
{ID: "underlay-only", Version: "v1"},
|
{ID: "underlay-only", Version: stringPointerForTest("v1")},
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||||
t.Fatalf("effective prompt identities = %#v, want %#v", prompts, wantPrompts)
|
t.Fatalf("effective prompt identities = %#v, want %#v", prompts, wantPrompts)
|
||||||
@@ -225,7 +226,9 @@ func TestListIdentitiesAreOwnedAndSafeForConcurrentUse(t *testing.T) {
|
|||||||
for i := range prompts {
|
for i := range prompts {
|
||||||
if prompts[i].ID == "prompt" {
|
if prompts[i].ID == "prompt" {
|
||||||
prompts[i].ID = "caller mutation"
|
prompts[i].ID = "caller mutation"
|
||||||
prompts[i].Version = "caller mutation"
|
if prompts[i].Version != nil {
|
||||||
|
*prompts[i].Version = "caller mutation"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
profiles, err = engine.ListProfiles(context.Background())
|
profiles, err = engine.ListProfiles(context.Background())
|
||||||
@@ -269,7 +272,9 @@ func TestListIdentitiesAreOwnedAndSafeForConcurrentUse(t *testing.T) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
prompts[0].ID = "worker mutation"
|
prompts[0].ID = "worker mutation"
|
||||||
prompts[0].Version = "worker mutation"
|
if prompts[0].Version != nil {
|
||||||
|
*prompts[0].Version = "worker mutation"
|
||||||
|
}
|
||||||
|
|
||||||
if _, err := engine.InspectProfile(context.Background(), "profile"); err != nil {
|
if _, err := engine.InspectProfile(context.Background(), "profile"); err != nil {
|
||||||
results <- fmt.Errorf("inspect profile: %w", err)
|
results <- fmt.Errorf("inspect profile: %w", err)
|
||||||
@@ -345,7 +350,7 @@ func countListingProfile(profiles []string, id string) int {
|
|||||||
func countListingPrompt(prompts []promptkit.PromptIdentity, id, version string) int {
|
func countListingPrompt(prompts []promptkit.PromptIdentity, id, version string) int {
|
||||||
count := 0
|
count := 0
|
||||||
for _, prompt := range prompts {
|
for _, prompt := range prompts {
|
||||||
if prompt.ID == id && prompt.Version == version {
|
if prompt.ID == id && prompt.Version != nil && *prompt.Version == version {
|
||||||
count++
|
count++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,9 +56,17 @@ func TestRunObservationHasResolvedIdentitiesAndAccounting(t *testing.T) {
|
|||||||
if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished {
|
if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished {
|
||||||
t.Fatalf("events=%+v", events)
|
t.Fatalf("events=%+v", events)
|
||||||
}
|
}
|
||||||
if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[1].PromptVersion != "1" || events[1].ProfileID != "profile" || events[1].Model != "model" {
|
if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[1].PromptVersion == nil ||
|
||||||
|
*events[1].PromptVersion != "1" || events[1].ProfileID != "profile" || events[1].Model != "model" {
|
||||||
t.Fatalf("selection=%+v", events)
|
t.Fatalf("selection=%+v", events)
|
||||||
}
|
}
|
||||||
|
if events[1].PromptVersion == events[2].PromptVersion {
|
||||||
|
t.Fatal("execution events share a mutable version label")
|
||||||
|
}
|
||||||
|
*events[1].PromptVersion = "mutated event"
|
||||||
|
if events[2].PromptVersion == nil || *events[2].PromptVersion != "1" {
|
||||||
|
t.Fatal("mutating one event changed another event's version label")
|
||||||
|
}
|
||||||
if result.RunID == "" || events[0].RunID != result.RunID || events[2].RunID != result.RunID || events[2].Attempts != 1 || events[2].TransientRetries != 0 || events[2].RepairCalls != 0 || events[2].Outcome != promptkit.ExecutionOutcomeSuccess {
|
if result.RunID == "" || events[0].RunID != result.RunID || events[2].RunID != result.RunID || events[2].Attempts != 1 || events[2].TransientRetries != 0 || events[2].RepairCalls != 0 || events[2].Outcome != promptkit.ExecutionOutcomeSuccess {
|
||||||
t.Fatalf("result=%+v terminal=%+v", result, events[2])
|
t.Fatalf("result=%+v terminal=%+v", result, events[2])
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -104,7 +104,9 @@ type ExecutionEvent struct {
|
|||||||
RunID string
|
RunID string
|
||||||
Time time.Time
|
Time time.Time
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
// PromptVersion is the selected definition's descriptive label. Each event
|
||||||
|
// owns its pointer; nil means absent while a non-nil empty string is explicit.
|
||||||
|
PromptVersion *string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
BackendID string
|
BackendID string
|
||||||
Model string
|
Model string
|
||||||
@@ -131,7 +133,7 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
|
|||||||
}
|
}
|
||||||
return ExecutionEvent{
|
return ExecutionEvent{
|
||||||
Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, Time: value.Time,
|
Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, Time: value.Time,
|
||||||
PromptID: value.PromptID, PromptVersion: value.PromptVersion, ProfileID: value.ProfileID,
|
PromptID: value.PromptID, PromptVersion: copyStringPtr(value.PromptVersion), ProfileID: value.ProfileID,
|
||||||
BackendID: value.BackendID, Model: value.Model, Generation: value.Generation,
|
BackendID: value.BackendID, Model: value.Model, Generation: value.Generation,
|
||||||
Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait,
|
Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait,
|
||||||
GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration,
|
GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration,
|
||||||
|
|||||||
@@ -87,13 +87,18 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.
|
|||||||
reader.set("changed artifact", "changed-input-hash")
|
reader.set("changed artifact", "changed-input-hash")
|
||||||
|
|
||||||
first := prepared.Details()
|
first := prepared.Details()
|
||||||
|
if first.PromptVersion == nil || *first.PromptVersion != "1" {
|
||||||
|
t.Fatalf("prepared label = %v, want 1", *first.PromptVersion)
|
||||||
|
}
|
||||||
|
*first.PromptVersion = "mutated details"
|
||||||
first.Messages[0].Content = "changed details"
|
first.Messages[0].Content = "changed details"
|
||||||
first.InputHashes["input"] = "changed-details-hash"
|
first.InputHashes["input"] = "changed-details-hash"
|
||||||
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "changed details"
|
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "changed details"
|
||||||
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "changed details"
|
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "changed details"
|
||||||
|
|
||||||
second := prepared.Details()
|
second := prepared.Details()
|
||||||
if second.Messages[0].Content != "Input=original artifact Label=original variable" {
|
if second.PromptVersion == nil || *second.PromptVersion != "1" ||
|
||||||
|
second.Messages[0].Content != "Input=original artifact Label=original variable" {
|
||||||
t.Fatalf("details message changed: %q", second.Messages[0].Content)
|
t.Fatalf("details message changed: %q", second.Messages[0].Content)
|
||||||
}
|
}
|
||||||
if second.InputHashes["input"] != "original-input-hash" {
|
if second.InputHashes["input"] != "original-input-hash" {
|
||||||
@@ -132,7 +137,7 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.
|
|||||||
t.Fatalf("generation did not use frozen details:\nrequest=%+v\ndetails=%+v", generated, second)
|
t.Fatalf("generation did not use frozen details:\nrequest=%+v\ndetails=%+v", generated, second)
|
||||||
}
|
}
|
||||||
if result.PromptID != second.PromptID ||
|
if result.PromptID != second.PromptID ||
|
||||||
result.PromptVersion != second.PromptVersion ||
|
result.PromptVersion == nil || second.PromptVersion == nil || *result.PromptVersion != *second.PromptVersion ||
|
||||||
result.PromptHash != second.PromptHash ||
|
result.PromptHash != second.PromptHash ||
|
||||||
result.SessionID != second.SessionID ||
|
result.SessionID != second.SessionID ||
|
||||||
result.RenderedPromptHash != second.RenderedPromptHash ||
|
result.RenderedPromptHash != second.RenderedPromptHash ||
|
||||||
@@ -142,6 +147,13 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.
|
|||||||
!reflect.DeepEqual(result.InputHashes, second.InputHashes) {
|
!reflect.DeepEqual(result.InputHashes, second.InputHashes) {
|
||||||
t.Fatalf("result provenance does not match details:\nresult=%+v\ndetails=%+v", result, second)
|
t.Fatalf("result provenance does not match details:\nresult=%+v\ndetails=%+v", result, second)
|
||||||
}
|
}
|
||||||
|
if result.PromptVersion == second.PromptVersion {
|
||||||
|
t.Fatal("run result and prepared details share a mutable version label")
|
||||||
|
}
|
||||||
|
*result.PromptVersion = "mutated result"
|
||||||
|
if *second.PromptVersion != "1" {
|
||||||
|
t.Fatal("run result label mutation reached prepared details")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
|
func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
|
||||||
|
|||||||
@@ -116,7 +116,8 @@ type RequirementIssue struct {
|
|||||||
// prompt context. Input and variable entries are sorted by exact name. Issues
|
// prompt context. Input and variable entries are sorted by exact name. Issues
|
||||||
// are deduplicated by code and associated names, then sorted by that tuple.
|
// are deduplicated by code and associated names, then sorted by that tuple.
|
||||||
// Their descriptions and entry explanations are safe summaries, not
|
// Their descriptions and entry explanations are safe summaries, not
|
||||||
// machine-readable details.
|
// machine-readable details. PromptVersion is caller-owned: nil means no label
|
||||||
|
// was declared; a non-nil empty string is explicit.
|
||||||
//
|
//
|
||||||
// Complete describes the supported analysis's discovery coverage, not whether
|
// Complete describes the supported analysis's discovery coverage, not whether
|
||||||
// requirements are supplied, templates will render, artifacts or schemas are
|
// requirements are supplied, templates will render, artifacts or schemas are
|
||||||
@@ -128,8 +129,9 @@ type RequirementIssue struct {
|
|||||||
type RequirementInspection struct {
|
type RequirementInspection struct {
|
||||||
// PromptID is the selected exact prompt ID.
|
// PromptID is the selected exact prompt ID.
|
||||||
PromptID string
|
PromptID string
|
||||||
// PromptVersion is the selected definition's descriptive version label.
|
// PromptVersion is the selected definition's descriptive version label. Nil
|
||||||
PromptVersion string
|
// means no label was declared; a non-nil empty string is an explicit label.
|
||||||
|
PromptVersion *string
|
||||||
// ProfileID is the selected profile identity; it does not indicate
|
// ProfileID is the selected profile identity; it does not indicate
|
||||||
// execution readiness.
|
// execution readiness.
|
||||||
ProfileID string
|
ProfileID string
|
||||||
@@ -200,7 +202,7 @@ func fromDomainRequirementInspection(inspection *domain.RequirementInspection) *
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
result := &RequirementInspection{
|
result := &RequirementInspection{
|
||||||
PromptID: inspection.PromptID, PromptVersion: inspection.PromptVersion,
|
PromptID: inspection.PromptID, PromptVersion: copyRequirementString(inspection.PromptVersion),
|
||||||
ProfileID: inspection.ProfileID, Complete: inspection.Complete,
|
ProfileID: inspection.ProfileID, Complete: inspection.Complete,
|
||||||
Inputs: make([]InputRequirement, len(inspection.Inputs)),
|
Inputs: make([]InputRequirement, len(inspection.Inputs)),
|
||||||
Variables: make([]VariableRequirement, len(inspection.Variables)),
|
Variables: make([]VariableRequirement, len(inspection.Variables)),
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ func BenchmarkRequirementInspection(b *testing.B) {
|
|||||||
PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"},
|
PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"},
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectRequirements(ctx, request)
|
inspection, err := engine.InspectRequirements(ctx, request)
|
||||||
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
|
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" ||
|
||||||
inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 ||
|
inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 ||
|
||||||
inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired ||
|
inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired ||
|
||||||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {
|
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *te
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect before artifacts, credentials, and schemas exist: %v", err)
|
t.Fatalf("inspect before artifacts, credentials, and schemas exist: %v", err)
|
||||||
}
|
}
|
||||||
if inspection.PromptID != "requirements" || inspection.PromptVersion != "v1" || inspection.ProfileID != "partial" || !inspection.Complete {
|
if inspection.PromptID != "requirements" || inspection.PromptVersion == nil || *inspection.PromptVersion != "v1" || inspection.ProfileID != "partial" || !inspection.Complete {
|
||||||
t.Fatalf("selected context or completeness is wrong: %+v", inspection)
|
t.Fatalf("selected context or completeness is wrong: %+v", inspection)
|
||||||
}
|
}
|
||||||
inputs := make(map[string]promptkit.InputRequirement, len(inspection.Inputs))
|
inputs := make(map[string]promptkit.InputRequirement, len(inspection.Inputs))
|
||||||
@@ -146,13 +146,13 @@ func TestInspectRequirementsUsesPromptIDsAndProfileIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
engine := requirementsEngine(t, prompts, profiles)
|
engine := requirementsEngine(t, prompts, profiles)
|
||||||
first, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "first-prompt"})
|
first, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "first-prompt"})
|
||||||
if err != nil || first.PromptVersion != "v1" || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" {
|
if err != nil || first.PromptVersion == nil || *first.PromptVersion != "v1" || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" {
|
||||||
t.Fatalf("first prompt ID/default profile=(%+v, %v)", first, err)
|
t.Fatalf("first prompt ID/default profile=(%+v, %v)", first, err)
|
||||||
}
|
}
|
||||||
second, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
second, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||||
PromptID: "second-prompt", ProfileID: "first",
|
PromptID: "second-prompt", ProfileID: "first",
|
||||||
})
|
})
|
||||||
if err != nil || second.PromptVersion != "v2" || second.ProfileID != "first" || second.Inputs[0].Name != "two" {
|
if err != nil || second.PromptVersion == nil || *second.PromptVersion != "v2" || second.ProfileID != "first" || second.Inputs[0].Name != "two" {
|
||||||
t.Fatalf("second prompt ID/request profile=(%+v, %v)", second, err)
|
t.Fatalf("second prompt ID/request profile=(%+v, %v)", second, err)
|
||||||
}
|
}
|
||||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "First-Prompt"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "First-Prompt"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
||||||
@@ -288,6 +288,10 @@ func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
if first.PromptVersion == nil || *first.PromptVersion != "v1" {
|
||||||
|
t.Fatalf("requirement inspection label = %v, want v1", first.PromptVersion)
|
||||||
|
}
|
||||||
|
*first.PromptVersion = "mutated label"
|
||||||
for index := range first.Inputs {
|
for index := range first.Inputs {
|
||||||
if first.Inputs[index].Name == "declared" {
|
if first.Inputs[index].Name == "declared" {
|
||||||
first.Inputs[index].Name = "mutated"
|
first.Inputs[index].Name = "mutated"
|
||||||
@@ -308,7 +312,8 @@ func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *
|
|||||||
description = input.Description
|
description = input.Description
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" ||
|
if second.PromptVersion == nil || *second.PromptVersion != "v1" ||
|
||||||
|
second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" ||
|
||||||
description == nil || *description != "captured description" ||
|
description == nil || *description != "captured description" ||
|
||||||
len(second.Issues) != 1 || second.Issues[0].Description == "mutated issue" {
|
len(second.Issues) != 1 || second.Issues[0].Description == "mutated issue" {
|
||||||
t.Fatalf("result mutation reached captured state or later result: (%+v, %v)", second, err)
|
t.Fatalf("result mutation reached captured state or later result: (%+v, %v)", second, err)
|
||||||
|
|||||||
@@ -116,9 +116,10 @@ type SmokeTestTarget struct {
|
|||||||
// ProfileID is the selected profile, if any.
|
// ProfileID is the selected profile, if any.
|
||||||
ProfileID string
|
ProfileID string
|
||||||
// PromptID identifies the selected prompt; PromptVersion carries its
|
// PromptID identifies the selected prompt; PromptVersion carries its
|
||||||
// descriptive version label, if any.
|
// descriptive version label. Nil means absent while a non-nil empty string is
|
||||||
|
// explicit. The pointer belongs to this returned target.
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion *string
|
||||||
// DefaultSourceBackendID identifies the backend whose configured default
|
// DefaultSourceBackendID identifies the backend whose configured default
|
||||||
// selected the profile, when that selection path was used.
|
// selected the profile, when that selection path was used.
|
||||||
DefaultSourceBackendID string
|
DefaultSourceBackendID string
|
||||||
@@ -305,7 +306,7 @@ func fromDomainSmokeTestReport(report domain.SmokeTestReport) (*SmokeTestReport,
|
|||||||
BackendID: report.Target.BackendID,
|
BackendID: report.Target.BackendID,
|
||||||
ProfileID: report.Target.ProfileID,
|
ProfileID: report.Target.ProfileID,
|
||||||
PromptID: report.Target.PromptID,
|
PromptID: report.Target.PromptID,
|
||||||
PromptVersion: report.Target.PromptVersion,
|
PromptVersion: copyStringPtr(report.Target.PromptVersion),
|
||||||
DefaultSourceBackendID: report.Target.DefaultSourceBackendID,
|
DefaultSourceBackendID: report.Target.DefaultSourceBackendID,
|
||||||
SelectionSource: report.Target.SelectionSource,
|
SelectionSource: report.Target.SelectionSource,
|
||||||
Endpoint: report.Target.Endpoint,
|
Endpoint: report.Target.Endpoint,
|
||||||
|
|||||||
@@ -291,6 +291,9 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
|
|||||||
if err != nil || report == nil || !report.Completed {
|
if err != nil || report == nil || !report.Completed {
|
||||||
t.Fatalf("structured smoke test=(%+v, %v)", report, err)
|
t.Fatalf("structured smoke test=(%+v, %v)", report, err)
|
||||||
}
|
}
|
||||||
|
if report.Target.PromptVersion == nil || *report.Target.PromptVersion != "1" {
|
||||||
|
t.Fatalf("smoke target label = %v, want 1", report.Target.PromptVersion)
|
||||||
|
}
|
||||||
if reader.calls != 0 || schemaSource.openCount() != schemaOpensBefore {
|
if reader.calls != 0 || schemaSource.openCount() != schemaOpensBefore {
|
||||||
t.Fatalf("application resources used: artifact reads=%d schema opens before=%d after=%d", reader.calls, schemaOpensBefore, schemaSource.openCount())
|
t.Fatalf("application resources used: artifact reads=%d schema opens before=%d after=%d", reader.calls, schemaOpensBefore, schemaSource.openCount())
|
||||||
}
|
}
|
||||||
@@ -322,6 +325,15 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
|
|||||||
t.Errorf("smoke report exposed %q", private)
|
t.Errorf("smoke report exposed %q", private)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
firstLabel := report.Target.PromptVersion
|
||||||
|
*firstLabel = "mutated report"
|
||||||
|
secondReport, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
|
||||||
|
Kind: promptkit.SmokeTestStructuredOutput, PromptID: "application-prompt", APIKey: "synthetic-smoke-key-marker",
|
||||||
|
})
|
||||||
|
if err != nil || secondReport == nil || !secondReport.Completed || secondReport.Target.PromptVersion == nil ||
|
||||||
|
*secondReport.Target.PromptVersion != "1" || secondReport.Target.PromptVersion == firstLabel {
|
||||||
|
t.Fatalf("smoke target label was not independently owned: (%+v, %v)", secondReport, err)
|
||||||
|
}
|
||||||
profilesAfter, err := engine.ListProfiles(context.Background())
|
profilesAfter, err := engine.ListProfiles(context.Background())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
21
types.go
21
types.go
@@ -159,8 +159,10 @@ type RunRequest struct {
|
|||||||
type PreparedRun struct {
|
type PreparedRun struct {
|
||||||
// PromptID is the selected prompt identifier.
|
// PromptID is the selected prompt identifier.
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
// PromptVersion is the selected definition's descriptive version label.
|
// PromptVersion is the selected definition's descriptive version label. Nil
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
// means no label was declared; a non-nil empty string is explicit. JSON omits
|
||||||
|
// nil and preserves a declared empty string.
|
||||||
|
PromptVersion *string `json:"prompt_version,omitempty"`
|
||||||
// PromptHash is an opaque equality value for the selected definition.
|
// PromptHash is an opaque equality value for the selected definition.
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
// SelectedProfileID is the explicit request profile or prompt default that
|
// SelectedProfileID is the explicit request profile or prompt default that
|
||||||
@@ -217,8 +219,10 @@ type RunResult struct {
|
|||||||
Validation ValidationResult `json:"validation"`
|
Validation ValidationResult `json:"validation"`
|
||||||
// PromptID is the selected prompt identifier.
|
// PromptID is the selected prompt identifier.
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
// PromptVersion is the selected definition's descriptive version label.
|
// PromptVersion is the selected definition's descriptive version label. Nil
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
// means no label was declared; a non-nil empty string is explicit. JSON omits
|
||||||
|
// nil and preserves a declared empty string.
|
||||||
|
PromptVersion *string `json:"prompt_version,omitempty"`
|
||||||
// PromptHash is the same opaque definition equality value exposed by
|
// PromptHash is the same opaque definition equality value exposed by
|
||||||
// PreparedRun.
|
// PreparedRun.
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
@@ -394,14 +398,17 @@ type PromptInputDefinition struct {
|
|||||||
// PromptInspection is the caller-owned result of [Engine.InspectPrompt].
|
// PromptInspection is the caller-owned result of [Engine.InspectPrompt].
|
||||||
// It has no stable JSON representation.
|
// It has no stable JSON representation.
|
||||||
//
|
//
|
||||||
// Inputs contains copied declared input metadata in definition order.
|
// PromptVersion is caller-owned metadata: nil means no label was declared and
|
||||||
|
// a non-nil empty string is explicit. Inputs contains copied declared input
|
||||||
|
// metadata in definition order.
|
||||||
// OutputContract is the normalized contract declared by the prompt definition,
|
// OutputContract is the normalized contract declared by the prompt definition,
|
||||||
// rather than a request-level effective override. PromptHash is opaque.
|
// rather than a request-level effective override. PromptHash is opaque.
|
||||||
type PromptInspection struct {
|
type PromptInspection struct {
|
||||||
// PromptID is the normalized ID of the selected prompt definition.
|
// PromptID is the normalized ID of the selected prompt definition.
|
||||||
PromptID string
|
PromptID string
|
||||||
// PromptVersion is the selected definition's descriptive version label.
|
// PromptVersion is the selected definition's descriptive version label. Nil
|
||||||
PromptVersion string
|
// means no label was declared; a non-nil empty string is an explicit label.
|
||||||
|
PromptVersion *string
|
||||||
// PromptHash is the opaque equality value for the selected definition.
|
// PromptHash is the opaque equality value for the selected definition.
|
||||||
PromptHash string
|
PromptHash string
|
||||||
// DefaultProfileID is declared metadata and is not resolved by inspection.
|
// DefaultProfileID is declared metadata and is not resolved by inspection.
|
||||||
|
|||||||
Reference in New Issue
Block a user