From 44ee389334e55af40a354bb8fa9355240e612752 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 13 Aug 2026 02:30:40 +0000 Subject: [PATCH] Reconcile prompt execution provenance --- docs/integrations/promptkit.md | 6 +- docs/internal/app-orchestration.md | 11 +-- docs/internal/prepared-report.md | 8 ++ internal/app/comparison_execution_test.go | 14 ++- internal/app/generation_test.go | 32 ++++++- internal/app/profile_execution.go | 77 ++++++++++++++++- internal/app/profile_execution_test.go | 86 ++++++++++++++++++- internal/app/prompt_inspection.go | 6 ++ internal/app/prompt_inspection_test.go | 15 ++++ .../app/prompt_profile_integration_test.go | 3 +- 10 files changed, 237 insertions(+), 21 deletions(-) diff --git a/docs/integrations/promptkit.md b/docs/integrations/promptkit.md index b72e1d7..9a87770 100644 --- a/docs/integrations/promptkit.md +++ b/docs/integrations/promptkit.md @@ -16,7 +16,7 @@ The `~` prefix is part of each OpenRouter rolling-alias model ID. The embedded p ## Selection And Active Execution -Before weather collection, Weatherreporter validates the report's exact generated-text report/schema/template catalog binding, prompt version, output contract, and selected profile. A nonblank `promptkit.profile` selects one profile ID for every report in the command; otherwise the prompt's declared default selects it. Promptkit resolves the selected definition in this order: +Before weather collection, Weatherreporter validates the report's exact generated-text report/schema/template catalog binding, prompt version and hash, output contract, and selected profile. Active profiles must resolve a nonblank backend and model identity. A nonblank `promptkit.profile` selects one profile ID for every report in the command; otherwise the prompt's declared default selects it. Promptkit resolves the selected definition in this order: 1. explicit in-memory profiles used by an embedding consumer or test; 2. the configured `profile_file` or `profile_dir`; @@ -27,7 +27,7 @@ A source falls through only when the selected ID is absent. Each source supplies Profiles that require a direct API key are unsupported; a profile that reports `APIKeyEnv` requires a nonblank value in that environment variable. Active results retain the selected logical profile ID and resolved backend and model. Ordinary errors, summaries, logs, and outputs exclude endpoints, credentials, rendered messages, schemas, request bodies, response bodies, and complete parameter maps. -Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions. +Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Before accepting that JSON, Weatherreporter requires exactly one preparation callback and reconciles its prompt/profile/backend/model and rendered/input hashes with the inspected identity and completed result. The callback output contract and completed validation must use the report's expected JSON Schema mode and path. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions. When capture is enabled, its preparation artifact projects a provider endpoint to its scheme and host and retains only reviewed execution settings. Provider @@ -42,7 +42,7 @@ binding, one exact prompt, and every explicitly selected profile before weather collection. It prepares one deterministic YAML data package, retains immutable copies of the report inputs, and executes every profile against the same exact data-package bytes. Each profile remains an -independent Promptkit execution: one provider or validation failure does not +independent Promptkit execution: one provider, provenance, or validation failure does not stop its peers, while caller cancellation applies to every in-flight execution. Weatherreporter starts selected profile executions concurrently and does not diff --git a/docs/internal/app-orchestration.md b/docs/internal/app-orchestration.md index e9d0c62..348d330 100644 --- a/docs/internal/app-orchestration.md +++ b/docs/internal/app-orchestration.md @@ -9,7 +9,7 @@ is owned by the [CLI reference](../cli.md) and [operations guide](../operations. `GenerateDetailed` resolves the requested report and output destination before initializing an optional explicit debug writer. An explicit output file wins; otherwise the configured output directory is used, falling back to the captured working directory. It validates the report's generated-text catalog binding, exact Promptkit prompt, and selected profile before collecting weather data. The resolved profile, backend, and model are carried in the active result. -The workflow builds facts, a module snapshot, briefing metadata, and the YAML prompt package in memory. It executes Promptkit, validates the returned generated text, builds a render context, and renders Markdown. `fileutil` atomically writes the completed Markdown to the selected output path. Only after that write succeeds does single-report notification run. +The workflow builds facts, a module snapshot, briefing metadata, and the YAML prompt package in memory. It executes Promptkit only against the inspected prompt and profile, reconciles the preparation callback and completed result with that identity and the prepared report schema, validates the returned generated text, builds a render context, and renders Markdown. `fileutil` atomically writes the completed Markdown to the selected output path. Only after that write succeeds does single-report notification run. Failures return an active partial result with safe identity, profile, warning, validation, debug, and output information when available. After rendering and immediately before publication, the workflow checks for cancellation or deadline expiry. Any failure before publication leaves an existing destination unchanged. A notification failure retains the newly published output. @@ -36,10 +36,11 @@ inspection fails, the partial result retains the resolved prompt ID, version, and hash. Artifact paths are added only after publication commits. The comparison execution core starts each inspected profile independently, -keeps results in selection order, and waits for all started work. Independent -profile failures are recorded and do not stop peers. Context cancellation marks -unfinished work and prevents publication. Details of prepared values, execution -and debugging, and publication are documented in [prepared report +keeps results in selection order, and waits for all started work. Every profile +reconciles its callback and completion provenance before its JSON can be +rendered. Independent profile failures are recorded and do not stop peers. +Context cancellation marks unfinished work and prevents publication. Details of +prepared values, execution and debugging, and publication are documented in [prepared report internals](prepared-report.md), [comparison execution internals](comparison-execution.md), and [comparison publication internals](comparison-publication.md). diff --git a/docs/internal/prepared-report.md b/docs/internal/prepared-report.md index 0e49d94..5db286a 100644 --- a/docs/internal/prepared-report.md +++ b/docs/internal/prepared-report.md @@ -17,6 +17,14 @@ Preparation deep-copies mutable facts, snapshots, identity, and data-package bytes before returning them. Consumers receive independent copies so one execution cannot change another's input or rendering context. +Before accepting generated JSON, the execution boundary reconciles the prepared +report definition, inspected prompt hash and selected profile identity, the one +preparation callback, and the completed Promptkit result. The callback and +completion must agree on prompt, profile, backend, model, and rendered/input +hashes; the callback output and completed validation must name the prepared +report's JSON Schema. A mismatch produces no rendered Markdown and leaves +results with only the inspected safe identity. + Single-report generation executes one prepared profile and publishes its Markdown. Comparison prepares once, gives every selected profile the same YAML bytes, and only then assembles the resulting logical bundle. The prompt-input diff --git a/internal/app/comparison_execution_test.go b/internal/app/comparison_execution_test.go index 5cf4143..b44f1c9 100644 --- a/internal/app/comparison_execution_test.go +++ b/internal/app/comparison_execution_test.go @@ -148,18 +148,21 @@ type barrierExecutor struct { releases map[string]chan struct{} requests map[string]promptexec.ExecuteRequest errors map[string]error + profiles map[string]ComparisonProfileInspection inFlight int maximum int } func newBarrierExecutor(profiles []ComparisonProfileInspection) *barrierExecutor { releases := make(map[string]chan struct{}, len(profiles)) + identities := make(map[string]ComparisonProfileInspection, len(profiles)) for _, profile := range profiles { releases[profile.ProfileID] = make(chan struct{}) + identities[profile.ProfileID] = profile } return &barrierExecutor{ started: make(chan string, len(profiles)), callbackFailures: make(chan error, len(profiles)), releases: releases, - requests: make(map[string]promptexec.ExecuteRequest, len(profiles)), errors: map[string]error{}, + requests: make(map[string]promptexec.ExecuteRequest, len(profiles)), errors: map[string]error{}, profiles: identities, } } @@ -173,7 +176,10 @@ func (e *barrierExecutor) InspectProfile(context.Context, string) (promptexec.Pr func (e *barrierExecutor) Execute(ctx context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) - if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", ProfileID: req.ProfileID, BackendID: "backend-" + req.ProfileID, ModelName: "model-" + req.ProfileID, StartedAt: stamp, EndedAt: stamp}, nil); err != nil { + e.mu.Lock() + profile := e.profiles[req.ProfileID] + e.mu.Unlock() + if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, ProfileID: req.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID + ".generated_text.schema.json"}, StartedAt: stamp, EndedAt: stamp}, nil); err != nil { e.callbackFailures <- err return nil, err } @@ -202,8 +208,8 @@ func (e *barrierExecutor) Execute(ctx context.Context, req promptexec.ExecuteReq return nil, err } return &promptexec.Execution{ - PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", - ProfileID: req.ProfileID, BackendID: "backend-" + req.ProfileID, ModelName: "model-" + req.ProfileID, + PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, + ProfileID: req.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName, StartedAt: stamp, EndedAt: stamp, RawOutput: comparisonRawOutput(), Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil), }, nil diff --git a/internal/app/generation_test.go b/internal/app/generation_test.go index 8848b59..3ee9bc8 100644 --- a/internal/app/generation_test.go +++ b/internal/app/generation_test.go @@ -48,6 +48,10 @@ type generationExecutor struct { validation promptexec.ValidationStatus rawOutput []byte failedPrompt string + skipPreparation bool + preparationCalls int + prepare func(*promptexec.Preparation) + complete func(*promptexec.Execution) } var generationExecutorMu sync.Mutex @@ -73,8 +77,25 @@ func (e *generationExecutor) InspectProfile(_ context.Context, id string) (promp } func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) - if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil { - return nil, err + generationExecutorMu.Lock() + skipPreparation := e.skipPreparation + prepare := e.prepare + preparationCalls := e.preparationCalls + generationExecutorMu.Unlock() + if !skipPreparation { + calls := preparationCalls + if calls == 0 { + calls = 1 + } + for range calls { + preparation := promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID + ".generated_text.schema.json"}, StartedAt: stamp, EndedAt: stamp} + if prepare != nil { + prepare(&preparation) + } + if err := callback(preparation, nil); err != nil { + return nil, err + } + } } generationExecutorMu.Lock() e.called = true @@ -86,6 +107,7 @@ func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRe rawOutput := append([]byte(nil), e.rawOutput...) failedPrompt := e.failedPrompt cancelBeforeReturn := e.cancelBeforeReturn + complete := e.complete generationExecutorMu.Unlock() if beforeExecute != nil { beforeExecute(req) @@ -108,7 +130,11 @@ func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRe if cancelBeforeReturn != nil { cancelBeforeReturn() } - return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil + execution := &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)} + if complete != nil { + complete(execution) + } + return execution, nil } const generationPromptHash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" diff --git a/internal/app/profile_execution.go b/internal/app/profile_execution.go index 743a83a..86aa338 100644 --- a/internal/app/profile_execution.go +++ b/internal/app/profile_execution.go @@ -3,6 +3,7 @@ package app import ( "context" "fmt" + "reflect" "gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug" @@ -49,10 +50,22 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p if req.Executor == nil { return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)} } + if err := validatePreparedExecutionRequest(req); err != nil { + return outcome, nil, &profileExecutionError{operation: "validate prompt provenance", err: err} + } callbackFailed := false - preparationCallback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { - outcome.ProfileID, outcome.BackendID, outcome.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName + preparationCount := 0 + var preparation promptexec.Preparation + preparationCallback := func(value promptexec.Preparation, debug *promptexec.PreparationDebug) error { + preparationCount++ + if preparationCount != 1 { + return promptProvenanceError() + } + if err := validatePreparationProvenance(req, value); err != nil { + return err + } + preparation = clonePreparation(value) if req.DebugWriter == nil || !req.DebugWriter.Enabled() { return nil } @@ -60,7 +73,7 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p callbackFailed = true return promptDebugWriteError(fmt.Errorf("prompt debug reference is required")) } - path, err := req.DebugWriter.WritePreparation(*req.DebugRef, preparation, debug) + path, err := req.DebugWriter.WritePreparation(*req.DebugRef, value, debug) if err != nil { callbackFailed = true return promptDebugWriteError(err) @@ -86,7 +99,12 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p if execution == nil { return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)} } - + if preparationCount != 1 { + return outcome, nil, &profileExecutionError{operation: "validate prompt provenance", err: promptProvenanceError()} + } + if err := validateExecutionProvenance(req, preparation, *execution); err != nil { + return outcome, nil, &profileExecutionError{operation: "validate prompt provenance", err: err} + } outcome.ValidationStatus = execution.Validation.Status if err := generatedtext.ValidateRawOutput(execution.RawOutput); err != nil { return outcome, nil, &profileExecutionError{operation: "validate generated text", err: err} @@ -129,3 +147,54 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p } return outcome, rendered, nil } + +func validatePreparedExecutionRequest(req profileExecutionRequest) error { + definition := req.Prepared.resolved.Definition + if definition.PromptID != req.Prompt.PromptID || definition.PromptVersion != req.Prompt.PromptVersion || + definition.GeneratedTextSchemaID != req.Prepared.handler.SchemaID() { + return promptProvenanceError() + } + if req.Prompt.ProfileID != "" && (req.Prompt.ProfileID != req.Profile.ProfileID || req.Prompt.BackendID != req.Profile.BackendID || req.Prompt.ModelName != req.Profile.ModelName) { + return promptProvenanceError() + } + if req.Prompt.PromptHash == "" || req.Profile.ProfileID == "" || req.Profile.BackendID == "" || req.Profile.ModelName == "" { + return promptProvenanceError() + } + return nil +} + +func validatePreparationProvenance(req profileExecutionRequest, preparation promptexec.Preparation) error { + definition := req.Prepared.resolved.Definition + if preparation.PromptID != req.Prompt.PromptID || preparation.PromptVersion != req.Prompt.PromptVersion || preparation.PromptHash != req.Prompt.PromptHash || + preparation.ProfileID != req.Profile.ProfileID || preparation.BackendID != req.Profile.BackendID || preparation.ModelName != req.Profile.ModelName || + !validPromptOutput(definition, preparation.Output) { + return promptProvenanceError() + } + return nil +} + +func validateExecutionProvenance(req profileExecutionRequest, preparation promptexec.Preparation, execution promptexec.Execution) error { + definition := req.Prepared.resolved.Definition + if execution.PromptID != preparation.PromptID || execution.PromptVersion != preparation.PromptVersion || execution.PromptHash != preparation.PromptHash || + execution.RenderedPromptHash != preparation.RenderedPromptHash || !reflect.DeepEqual(execution.InputHashes, preparation.InputHashes) || + execution.ProfileID != preparation.ProfileID || execution.BackendID != preparation.BackendID || execution.ModelName != preparation.ModelName || + execution.Validation.Mode != "json_schema" || execution.Validation.SchemaPath != definition.GeneratedTextSchemaID+".generated_text.schema.json" { + return promptProvenanceError() + } + return nil +} + +func promptProvenanceError() error { + return promptexec.NewError(promptexec.InvalidConfiguration, "prompt execution provenance is inconsistent", nil) +} + +func clonePreparation(value promptexec.Preparation) promptexec.Preparation { + if value.InputHashes != nil { + inputHashes := make(map[string]string, len(value.InputHashes)) + for name, hash := range value.InputHashes { + inputHashes[name] = hash + } + value.InputHashes = inputHashes + } + return value +} diff --git a/internal/app/profile_execution_test.go b/internal/app/profile_execution_test.go index 1b4d17f..0b793ba 100644 --- a/internal/app/profile_execution_test.go +++ b/internal/app/profile_execution_test.go @@ -70,6 +70,90 @@ func TestExecutePreparedProfileBoundsOversizedExecutorOutput(t *testing.T) { } } +func TestExecutePreparedProfileRejectsInconsistentProvenance(t *testing.T) { + tests := []struct { + name string + mutate func(*preparedReport, *PromptInspectionResult, *promptexec.ProfileInspection, *generationExecutor) + invoked bool + }{ + { + name: "prepared definition", mutate: func(prepared *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, _ *generationExecutor) { + prepared.resolved.Definition.PromptVersion = "different-version" + }, + }, + { + name: "missing callback", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) { + executor.skipPreparation = true + }, invoked: true, + }, + { + name: "duplicate callback", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) { + executor.preparationCalls = 2 + }, + }, + { + name: "callback prompt hash", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) { + executor.prepare = func(value *promptexec.Preparation) { value.PromptHash = "different-hash" } + }, + }, + { + name: "callback output schema", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) { + executor.prepare = func(value *promptexec.Preparation) { value.Output.SchemaPath = "other.generated_text.schema.json" } + }, + }, + { + name: "completed profile", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) { + executor.complete = func(value *promptexec.Execution) { value.ProfileID = "different-profile" } + }, invoked: true, + }, + { + name: "completed rendered prompt hash", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) { + executor.complete = func(value *promptexec.Execution) { value.RenderedPromptHash = "different-rendered-hash" } + }, invoked: true, + }, + { + name: "completed input hashes", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) { + executor.prepare = func(value *promptexec.Preparation) { + value.InputHashes = map[string]string{"data_package": "prepared-hash"} + } + executor.complete = func(value *promptexec.Execution) { + value.InputHashes = map[string]string{"data_package": "completed-hash"} + } + }, invoked: true, + }, + { + name: "completed validation mode", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) { + executor.complete = func(value *promptexec.Execution) { value.Validation.Mode = "other" } + }, invoked: true, + }, + { + name: "completed validation schema", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) { + executor.complete = func(value *promptexec.Execution) { value.Validation.SchemaPath = "other.generated_text.schema.json" } + }, invoked: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prepared, inspection := preparedDailyProfile(t) + profile := promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName} + executor := &generationExecutor{} + tt.mutate(&prepared, &inspection, &profile, executor) + + outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{Prepared: prepared, Prompt: inspection, Profile: profile, Executor: executor}) + if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || len(rendered) != 0 { + t.Fatalf("outcome/rendered/error = %#v/%q/%v", outcome, rendered, err) + } + if outcome.ProfileID != profile.ProfileID || outcome.BackendID != profile.BackendID || outcome.ModelName != profile.ModelName || outcome.ValidationStatus != "" { + t.Fatalf("outcome retained unverified provenance: %#v", outcome) + } + if (executor.executeCalls == 1) != tt.invoked { + t.Fatalf("executor calls = %d, want invoked=%t", executor.executeCalls, tt.invoked) + } + }) + } +} + func preparedDailyProfile(t *testing.T) (preparedReport, PromptInspectionResult) { t.Helper() cfg := generationConfig() @@ -85,5 +169,5 @@ func preparedDailyProfile(t *testing.T) (preparedReport, PromptInspectionResult) if err != nil { t.Fatalf("prepareReport() error = %v", err) } - return prepared, PromptInspectionResult{PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, PromptHash: "prompt-hash", ProfileID: "fixture", BackendID: "fixture", ModelName: "fixture-model"} + return prepared, PromptInspectionResult{PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, PromptHash: generationPromptHash, ProfileID: "fixture", BackendID: "fixture", ModelName: "fixture-model"} } diff --git a/internal/app/prompt_inspection.go b/internal/app/prompt_inspection.go index 7d7b271..7e30ab1 100644 --- a/internal/app/prompt_inspection.go +++ b/internal/app/prompt_inspection.go @@ -178,6 +178,9 @@ func inspectPromptContract(ctx context.Context, executor promptexec.Executor, de if inspection.PromptID != definition.PromptID || inspection.PromptVersion != definition.PromptVersion { return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt inspection did not return the requested prompt version", nil) } + if strings.TrimSpace(inspection.PromptHash) == "" { + return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt inspection did not return a prompt hash", nil) + } if !validPromptInput(inspection.Inputs) { return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare exactly one required application/yaml data_package input", nil) } @@ -207,6 +210,9 @@ func inspectPromptProfile(ctx context.Context, executor promptexec.Executor, pro return promptexec.ProfileInspection{}, promptexec.NewError(promptexec.MissingCredential, "selected profile credential is unavailable", nil) } } + if strings.TrimSpace(profile.BackendID) == "" || strings.TrimSpace(profile.ModelName) == "" { + return promptexec.ProfileInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "profile inspection did not return a complete execution identity", nil) + } return profile, nil } diff --git a/internal/app/prompt_inspection_test.go b/internal/app/prompt_inspection_test.go index 9426783..28789a7 100644 --- a/internal/app/prompt_inspection_test.go +++ b/internal/app/prompt_inspection_test.go @@ -71,6 +71,21 @@ func TestInspectPromptExecutionRejectsInvalidContractsAndCredentials(t *testing. }(), wantCategory: promptexec.InvalidConfiguration, }, + { + name: "missing prompt hash", + prompt: func() promptexec.PromptInspection { + value := basePrompt + value.PromptHash = "" + return value + }(), + wantCategory: promptexec.InvalidConfiguration, + }, + { + name: "missing profile backend", + prompt: basePrompt, + profile: promptexec.ProfileInspection{ProfileID: "default-profile", ModelName: "model"}, + wantCategory: promptexec.InvalidConfiguration, + }, { name: "direct key", prompt: basePrompt, diff --git a/internal/app/prompt_profile_integration_test.go b/internal/app/prompt_profile_integration_test.go index 6ed6b54..a885dba 100644 --- a/internal/app/prompt_profile_integration_test.go +++ b/internal/app/prompt_profile_integration_test.go @@ -41,12 +41,13 @@ func TestPromptInspectionResolvesEmbeddedAndOverriddenProfilesOffline(t *testing override, err := promptkitadapter.New(promptkitadapter.Config{ProfileFile: writeProfileFile(t, `id: weather-light endpoint: https://local.example/v1 +backend: openrouter model: local-weather `)}) if err != nil { t.Fatalf("New(override) error = %v", err) } - inspect(t, override, report.Hourly, "", "weather-light", "", "local-weather") + inspect(t, override, report.Hourly, "", "weather-light", "openrouter", "local-weather") } func resolvedPromptProfile(t *testing.T, id report.ID) report.Resolved {