Simplify prompt report orchestration
This commit is contained in:
@@ -568,8 +568,9 @@ func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportReque
|
|||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
result.OutputPath = req.OutputPath
|
result.OutputPath = req.OutputPath
|
||||||
req.ExecutionArtifact.Paths.OutputPath = req.OutputPath
|
if err := persistReachedPromptPath(ctx, req.Store, req.Resolved, req.ExecutionArtifact, func(paths *state.PromptExecutionPaths) {
|
||||||
if _, err := req.Store.SavePromptExecution(ctx, req.Resolved, *req.ExecutionArtifact); err != nil {
|
paths.OutputPath = req.OutputPath
|
||||||
|
}); err != nil {
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -598,8 +599,9 @@ func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportReque
|
|||||||
result.Notification = notification
|
result.Notification = notification
|
||||||
metadata.NotificationPath = notificationPath
|
metadata.NotificationPath = notificationPath
|
||||||
result.Metadata = metadata
|
result.Metadata = metadata
|
||||||
req.ExecutionArtifact.Paths.NotificationPath = notificationPath
|
if saveErr := persistReachedPromptPath(ctx, req.Store, req.Resolved, req.ExecutionArtifact, func(paths *state.PromptExecutionPaths) {
|
||||||
if _, saveErr := req.Store.SavePromptExecution(ctx, req.Resolved, *req.ExecutionArtifact); saveErr != nil {
|
paths.NotificationPath = notificationPath
|
||||||
|
}); saveErr != nil {
|
||||||
return result, saveErr
|
return result, saveErr
|
||||||
}
|
}
|
||||||
metadataPath, saveErr := req.Store.SaveMetadata(ctx, metadata)
|
metadataPath, saveErr := req.Store.SaveMetadata(ctx, metadata)
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/fileutil"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/fileutil"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
@@ -25,8 +26,70 @@ type promptReportRequest struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
|
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
|
||||||
bundle := req.Collection.Bundle
|
workflow, err := newPromptReportWorkflow(ctx, req)
|
||||||
if bundle == nil {
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := workflow.buildInputs(); err != nil {
|
||||||
|
return workflow.result, err
|
||||||
|
}
|
||||||
|
|
||||||
|
execution, executeErr := workflow.executePrompt()
|
||||||
|
if executeErr != nil {
|
||||||
|
return workflow.result, workflow.handleExecutionFailure(executeErr)
|
||||||
|
}
|
||||||
|
if execution == nil {
|
||||||
|
err := promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)
|
||||||
|
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
|
||||||
|
return workflow.result, saveErr
|
||||||
|
}
|
||||||
|
return workflow.result, workflow.reportError("execute prompt", err)
|
||||||
|
}
|
||||||
|
if err := workflow.persistExecutionDebug(*execution); err != nil {
|
||||||
|
return workflow.result, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
|
||||||
|
err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)
|
||||||
|
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
|
||||||
|
return workflow.result, saveErr
|
||||||
|
}
|
||||||
|
return workflow.result, workflow.reportError("validate prompt execution", err)
|
||||||
|
}
|
||||||
|
if err := workflow.persistCompletedExecution(*execution); err != nil {
|
||||||
|
return workflow.result, err
|
||||||
|
}
|
||||||
|
if execution.Validation.Status == promptexec.ValidationFailed {
|
||||||
|
err := promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)
|
||||||
|
return workflow.result, workflow.reportError("validate prompt execution", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rendered, err := workflow.persistGeneratedContent(execution.RawOutput)
|
||||||
|
if err != nil {
|
||||||
|
return workflow.result, err
|
||||||
|
}
|
||||||
|
return workflow.finalizeReport(rendered)
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptReportWorkflow struct {
|
||||||
|
ctx context.Context
|
||||||
|
req promptReportRequest
|
||||||
|
store state.Store
|
||||||
|
result *ReportResult
|
||||||
|
metadata state.Metadata
|
||||||
|
briefingMetadata briefing.Metadata
|
||||||
|
reportFacts ReportFacts
|
||||||
|
moduleSnapshot module.Snapshot
|
||||||
|
dataPackageBytes []byte
|
||||||
|
handler generatedtext.Handler
|
||||||
|
executionArtifact state.PromptExecutionArtifact
|
||||||
|
debugRef state.PromptDebugRef
|
||||||
|
prepared bool
|
||||||
|
callbackFailed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) {
|
||||||
|
if req.Collection.Bundle == nil {
|
||||||
return nil, fmt.Errorf("collected weather bundle is required")
|
return nil, fmt.Errorf("collected weather bundle is required")
|
||||||
}
|
}
|
||||||
store := req.Store
|
store := req.Store
|
||||||
@@ -37,316 +100,303 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
paths, err := store.Paths(req.Resolved)
|
return &promptReportWorkflow{ctx: ctx, req: req, store: store}, nil
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
result := &ReportResult{}
|
|
||||||
|
|
||||||
priorSnapshot, err := store.FindPriorSnapshot(ctx, req.Resolved)
|
func (w *promptReportWorkflow) buildInputs() error {
|
||||||
|
paths, err := w.store.Paths(w.req.Resolved)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return err
|
||||||
}
|
}
|
||||||
reportFacts, err := BuildReportFacts(ModuleSnapshotRequest{Config: req.Config, Resolved: req.Resolved}, bundle)
|
w.result = &ReportResult{}
|
||||||
|
priorSnapshot, err := w.store.FindPriorSnapshot(w.ctx, w.req.Resolved)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, generatedReportError(req.Resolved, req.Resolved.Metadata().RunID, "build report facts", err)
|
return err
|
||||||
}
|
}
|
||||||
moduleSnapshot, err := BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: req.Config, Resolved: req.Resolved}, reportFacts)
|
w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, generatedReportError(req.Resolved, req.Resolved.Metadata().RunID, "build module snapshot", err)
|
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build report facts", err)
|
||||||
}
|
}
|
||||||
moduleSnapshotPath, err := store.SaveModuleSnapshot(ctx, req.Resolved, moduleSnapshot)
|
w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build module snapshot", err)
|
||||||
}
|
}
|
||||||
result.ModuleSnapshot = moduleSnapshot
|
moduleSnapshotPath, err := w.store.SaveModuleSnapshot(w.ctx, w.req.Resolved, w.moduleSnapshot)
|
||||||
result.ModuleSnapshotPath = moduleSnapshotPath
|
if err != nil {
|
||||||
result.PriorSnapshot = priorSnapshot
|
return err
|
||||||
|
}
|
||||||
|
w.result.ModuleSnapshot = w.moduleSnapshot
|
||||||
|
w.result.ModuleSnapshotPath = moduleSnapshotPath
|
||||||
|
w.result.PriorSnapshot = priorSnapshot
|
||||||
|
|
||||||
recent, err := recentChanges(ctx, store, priorSnapshot, req.Resolved.Definition.ID, moduleSnapshot, req.Config.RecentChange)
|
recent, err := recentChanges(w.ctx, w.store, priorSnapshot, w.req.Resolved.Definition.ID, w.moduleSnapshot, w.req.Config.RecentChange)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return err
|
||||||
}
|
}
|
||||||
result.RecentChanges = recent
|
w.result.RecentChanges = recent
|
||||||
briefingMetadata := briefing.BuildMetadata(briefingBuildContext(req.Config, req.Resolved, reportFacts.Collected))
|
w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected))
|
||||||
metadata := state.BuildPromptMetadataFromBriefingMetadata(req.Resolved, briefingMetadata, state.ArtifactPaths{
|
w.metadata = state.BuildPromptMetadataFromBriefingMetadata(w.req.Resolved, w.briefingMetadata, state.ArtifactPaths{
|
||||||
ModuleSnapshot: moduleSnapshotPath,
|
ModuleSnapshot: moduleSnapshotPath,
|
||||||
Metadata: paths.Metadata,
|
Metadata: paths.Metadata,
|
||||||
})
|
})
|
||||||
result.Metadata = metadata
|
w.result.Metadata = w.metadata
|
||||||
dataPackage, err := promptinput.Build(promptinput.BuildRequest{
|
dataPackage, err := promptinput.Build(promptinput.BuildRequest{
|
||||||
Metadata: promptMetadata(metadata),
|
Metadata: promptMetadata(w.metadata), Modules: w.moduleSnapshot, RecentChanges: recent,
|
||||||
Modules: moduleSnapshot,
|
|
||||||
RecentChanges: recent,
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "build data package", err)
|
return w.reportError("build data package", err)
|
||||||
}
|
}
|
||||||
data, err := promptinput.MarshalYAML(dataPackage)
|
w.dataPackageBytes, err = promptinput.MarshalYAML(dataPackage)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
dataPackagePath, err := store.SaveDataPackageBytes(ctx, req.Resolved, data)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
metadata.DataPackagePath = dataPackagePath
|
|
||||||
result.DataPackage = dataPackage
|
|
||||||
result.DataPackagePath = dataPackagePath
|
|
||||||
result.Metadata = metadata
|
|
||||||
|
|
||||||
handler, err := generatedtext.LookupDefinition(req.Resolved.Definition)
|
|
||||||
if err != nil {
|
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "lookup generated text catalog", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prepared := false
|
|
||||||
callbackFailed := false
|
|
||||||
debugRef := state.PromptDebugRef{
|
|
||||||
ReportID: req.Resolved.Definition.ID,
|
|
||||||
ValidDate: req.Resolved.ValidPeriod.Start.Format("2006-01-02"),
|
|
||||||
RunID: metadata.RunID,
|
|
||||||
}
|
|
||||||
callback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
|
||||||
artifact := state.PromptPreparationArtifact{
|
|
||||||
SchemaVersion: state.PromptPreparationSchemaVersion,
|
|
||||||
Status: state.PromptPreparationSucceeded,
|
|
||||||
ReportID: req.Resolved.Definition.ID,
|
|
||||||
RunID: metadata.RunID,
|
|
||||||
PromptID: req.Inspection.PromptID,
|
|
||||||
PromptVersion: req.Inspection.PromptVersion,
|
|
||||||
DataPackagePath: dataPackagePath,
|
|
||||||
Preparation: &preparation,
|
|
||||||
StartedAt: preparation.StartedAt,
|
|
||||||
EndedAt: preparation.EndedAt,
|
|
||||||
Duration: preparation.Duration,
|
|
||||||
}
|
|
||||||
path, err := store.SavePromptPreparation(ctx, req.Resolved, artifact)
|
|
||||||
if err != nil {
|
|
||||||
callbackFailed = true
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
prepared = true
|
dataPackagePath, err := w.store.SaveDataPackageBytes(w.ctx, w.req.Resolved, w.dataPackageBytes)
|
||||||
result.PreparationPath = path
|
|
||||||
metadata.PreparationPath = path
|
|
||||||
result.Metadata = metadata
|
|
||||||
debugPath, err := req.DebugWriter.WritePreparation(debugRef, preparation, debug)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
callbackFailed = true
|
return err
|
||||||
|
}
|
||||||
|
w.metadata.DataPackagePath = dataPackagePath
|
||||||
|
w.result.DataPackage = dataPackage
|
||||||
|
w.result.DataPackagePath = dataPackagePath
|
||||||
|
w.result.Metadata = w.metadata
|
||||||
|
w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition)
|
||||||
|
if err != nil {
|
||||||
|
return w.reportError("lookup generated text catalog", err)
|
||||||
|
}
|
||||||
|
w.debugRef = state.PromptDebugRef{
|
||||||
|
ReportID: w.req.Resolved.Definition.ID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.metadata.RunID,
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) {
|
||||||
|
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{
|
||||||
|
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
||||||
|
ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackageBytes,
|
||||||
|
DataPackagePath: w.result.DataPackagePath, CaptureDebug: w.req.DebugWriter.Enabled(),
|
||||||
|
}, w.persistPreparation)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *promptReportWorkflow) persistPreparation(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
||||||
|
artifact := state.PromptPreparationArtifact{
|
||||||
|
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationSucceeded,
|
||||||
|
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
|
||||||
|
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
||||||
|
DataPackagePath: w.result.DataPackagePath, Preparation: &preparation,
|
||||||
|
StartedAt: preparation.StartedAt, EndedAt: preparation.EndedAt, Duration: preparation.Duration,
|
||||||
|
}
|
||||||
|
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
|
||||||
|
if err != nil {
|
||||||
|
w.callbackFailed = true
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
w.prepared = true
|
||||||
|
w.result.PreparationPath = path
|
||||||
|
w.metadata.PreparationPath = path
|
||||||
|
w.result.Metadata = w.metadata
|
||||||
|
debugPath, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
|
||||||
|
if err != nil {
|
||||||
|
w.callbackFailed = true
|
||||||
return promptDebugWriteError(err)
|
return promptDebugWriteError(err)
|
||||||
}
|
}
|
||||||
if debugPath != "" {
|
if debugPath != "" {
|
||||||
result.LLMDebugPath = debugPath
|
w.result.LLMDebugPath = debugPath
|
||||||
}
|
}
|
||||||
metadataPath, err := store.SaveMetadata(ctx, metadata)
|
if err := w.saveMetadata(); err != nil {
|
||||||
if err != nil {
|
w.callbackFailed = true
|
||||||
callbackFailed = true
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
result.Metadata = metadata
|
|
||||||
result.MetadataPath = metadataPath
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
execution, executeErr := req.Executor.Execute(ctx, promptexec.ExecuteRequest{
|
|
||||||
PromptID: req.Inspection.PromptID,
|
func (w *promptReportWorkflow) handleExecutionFailure(executeErr error) error {
|
||||||
PromptVersion: req.Inspection.PromptVersion,
|
if w.callbackFailed {
|
||||||
ProfileID: req.Inspection.ProfileID,
|
return executeErr
|
||||||
DataPackage: data,
|
|
||||||
DataPackagePath: dataPackagePath,
|
|
||||||
CaptureDebug: req.DebugWriter.Enabled(),
|
|
||||||
}, callback)
|
|
||||||
if executeErr != nil {
|
|
||||||
if callbackFailed {
|
|
||||||
return result, executeErr
|
|
||||||
}
|
}
|
||||||
executeErr = classifiedPromptError("prompt execution failed", executeErr)
|
executeErr = classifiedPromptError("prompt execution failed", executeErr)
|
||||||
if !prepared {
|
if !w.prepared {
|
||||||
artifact := state.PromptPreparationArtifact{
|
if err := w.persistPreparationFailure(executeErr); err != nil {
|
||||||
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationFailed,
|
return err
|
||||||
ReportID: req.Resolved.Definition.ID, RunID: metadata.RunID, PromptID: req.Inspection.PromptID,
|
|
||||||
PromptVersion: req.Inspection.PromptVersion, DataPackagePath: dataPackagePath,
|
|
||||||
StartedAt: time.Now(), EndedAt: time.Now(), Error: state.NewPromptArtifactError(executeErr),
|
|
||||||
}
|
}
|
||||||
path, saveErr := store.SavePromptPreparation(ctx, req.Resolved, artifact)
|
return w.reportError("prepare prompt", executeErr)
|
||||||
if saveErr != nil {
|
|
||||||
return result, saveErr
|
|
||||||
}
|
|
||||||
metadata.PreparationPath = path
|
|
||||||
result.PreparationPath = path
|
|
||||||
result.Metadata = metadata
|
|
||||||
metadataPath, saveErr := store.SaveMetadata(ctx, metadata)
|
|
||||||
if saveErr != nil {
|
|
||||||
return result, saveErr
|
|
||||||
}
|
|
||||||
result.MetadataPath = metadataPath
|
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "prepare prompt", executeErr)
|
|
||||||
}
|
}
|
||||||
if promptexec.CategoryOf(executeErr) != "" {
|
if promptexec.CategoryOf(executeErr) != "" {
|
||||||
artifact := failedPromptExecutionArtifact(req.Resolved, metadata, req.Inspection, executeErr)
|
if err := w.persistOperationalExecutionFailure(executeErr); err != nil {
|
||||||
path, saveErr := store.SavePromptExecution(ctx, req.Resolved, artifact)
|
return err
|
||||||
if saveErr != nil {
|
|
||||||
return result, saveErr
|
|
||||||
}
|
}
|
||||||
metadata.ExecutionPath = path
|
|
||||||
result.ExecutionPath = path
|
|
||||||
result.Metadata = metadata
|
|
||||||
metadataPath, saveErr := store.SaveMetadata(ctx, metadata)
|
|
||||||
if saveErr != nil {
|
|
||||||
return result, saveErr
|
|
||||||
}
|
}
|
||||||
result.MetadataPath = metadataPath
|
return w.reportError("execute prompt", executeErr)
|
||||||
}
|
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", executeErr)
|
|
||||||
}
|
|
||||||
if execution == nil {
|
|
||||||
err := promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)
|
|
||||||
artifact := failedPromptExecutionArtifact(req.Resolved, metadata, req.Inspection, err)
|
|
||||||
executionPath, saveErr := store.SavePromptExecution(ctx, req.Resolved, artifact)
|
|
||||||
if saveErr != nil {
|
|
||||||
return result, saveErr
|
|
||||||
}
|
|
||||||
metadata.ExecutionPath = executionPath
|
|
||||||
result.ExecutionPath = executionPath
|
|
||||||
result.Metadata = metadata
|
|
||||||
metadataPath, saveErr := store.SaveMetadata(ctx, metadata)
|
|
||||||
if saveErr != nil {
|
|
||||||
return result, saveErr
|
|
||||||
}
|
|
||||||
result.MetadataPath = metadataPath
|
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", err)
|
|
||||||
}
|
|
||||||
debugPath, err := req.DebugWriter.WriteExecution(debugRef, *execution)
|
|
||||||
if err != nil {
|
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "write prompt debug", promptDebugWriteError(err))
|
|
||||||
}
|
|
||||||
if debugPath != "" {
|
|
||||||
result.LLMDebugPath = debugPath
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
|
func (w *promptReportWorkflow) persistPreparationFailure(executeErr error) error {
|
||||||
err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)
|
startedAt, endedAt := time.Now(), time.Now()
|
||||||
artifact := failedPromptExecutionArtifact(req.Resolved, metadata, req.Inspection, err)
|
artifact := state.PromptPreparationArtifact{
|
||||||
executionPath, saveErr := store.SavePromptExecution(ctx, req.Resolved, artifact)
|
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationFailed,
|
||||||
if saveErr != nil {
|
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
|
||||||
return result, saveErr
|
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
||||||
|
DataPackagePath: w.result.DataPackagePath, StartedAt: startedAt, EndedAt: endedAt,
|
||||||
|
Error: state.NewPromptArtifactError(executeErr),
|
||||||
}
|
}
|
||||||
metadata.ExecutionPath = executionPath
|
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
|
||||||
result.ExecutionPath = executionPath
|
|
||||||
result.Metadata = metadata
|
|
||||||
metadataPath, saveErr := store.SaveMetadata(ctx, metadata)
|
|
||||||
if saveErr != nil {
|
|
||||||
return result, saveErr
|
|
||||||
}
|
|
||||||
result.MetadataPath = metadataPath
|
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "validate prompt execution", err)
|
|
||||||
}
|
|
||||||
rawPath, err := store.SaveGeneratedTextRaw(ctx, req.Resolved, execution.RawOutput)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return err
|
||||||
}
|
}
|
||||||
result.GeneratedTextRawPath = rawPath
|
w.result.PreparationPath = path
|
||||||
metadata.GeneratedTextRawPath = rawPath
|
w.metadata.PreparationPath = path
|
||||||
result.Metadata = metadata
|
w.result.Metadata = w.metadata
|
||||||
executionArtifact := state.PromptExecutionArtifact{
|
return w.saveMetadata()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *promptReportWorkflow) persistOperationalExecutionFailure(executeErr error) error {
|
||||||
|
artifact := failedPromptExecutionArtifact(w.req.Resolved, w.metadata, w.req.Inspection, executeErr)
|
||||||
|
path, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, artifact)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
w.result.ExecutionPath = path
|
||||||
|
w.metadata.ExecutionPath = path
|
||||||
|
w.result.Metadata = w.metadata
|
||||||
|
return w.saveMetadata()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *promptReportWorkflow) persistExecutionDebug(execution promptexec.Execution) error {
|
||||||
|
debugPath, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
|
||||||
|
if err != nil {
|
||||||
|
return w.reportError("write prompt debug", promptDebugWriteError(err))
|
||||||
|
}
|
||||||
|
if debugPath != "" {
|
||||||
|
w.result.LLMDebugPath = debugPath
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *promptReportWorkflow) persistCompletedExecution(execution promptexec.Execution) error {
|
||||||
|
rawPath, err := w.store.SaveGeneratedTextRaw(w.ctx, w.req.Resolved, execution.RawOutput)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
w.result.GeneratedTextRawPath = rawPath
|
||||||
|
w.metadata.GeneratedTextRawPath = rawPath
|
||||||
|
w.result.Metadata = w.metadata
|
||||||
|
w.executionArtifact = state.PromptExecutionArtifact{
|
||||||
SchemaVersion: state.PromptExecutionSchemaVersion,
|
SchemaVersion: state.PromptExecutionSchemaVersion,
|
||||||
ReportID: req.Resolved.Definition.ID, RunID: metadata.RunID,
|
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
|
||||||
PromptID: req.Inspection.PromptID, PromptVersion: req.Inspection.PromptVersion,
|
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
||||||
Provenance: ptr(state.PromptExecutionProvenanceFrom(*execution)), Validation: &execution.Validation,
|
Provenance: ptr(state.PromptExecutionProvenanceFrom(execution)), Validation: &execution.Validation,
|
||||||
Paths: state.PromptExecutionPaths{RawOutputPath: rawPath},
|
Paths: state.PromptExecutionPaths{RawOutputPath: rawPath},
|
||||||
StartedAt: execution.StartedAt, EndedAt: execution.EndedAt, Duration: execution.Duration,
|
StartedAt: execution.StartedAt, EndedAt: execution.EndedAt, Duration: execution.Duration,
|
||||||
}
|
}
|
||||||
if execution.Validation.Status == promptexec.ValidationPassed {
|
if execution.Validation.Status == promptexec.ValidationPassed {
|
||||||
executionArtifact.Status = state.PromptExecutionSucceeded
|
w.executionArtifact.Status = state.PromptExecutionSucceeded
|
||||||
} else {
|
} else {
|
||||||
executionArtifact.Status = state.PromptExecutionValidationRejected
|
w.executionArtifact.Status = state.PromptExecutionValidationRejected
|
||||||
}
|
}
|
||||||
executionPath, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact)
|
executionPath, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, w.executionArtifact)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return err
|
||||||
}
|
}
|
||||||
metadata.ExecutionPath = executionPath
|
w.result.ExecutionPath = executionPath
|
||||||
result.ExecutionPath = executionPath
|
w.metadata.ExecutionPath = executionPath
|
||||||
result.Metadata = metadata
|
w.result.Metadata = w.metadata
|
||||||
metadataPath, err := store.SaveMetadata(ctx, metadata)
|
return w.saveMetadata()
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
result.MetadataPath = metadataPath
|
|
||||||
if execution.Validation.Status == promptexec.ValidationFailed {
|
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
generatedText, normalized, err := handler.Validate(execution.RawOutput)
|
func (w *promptReportWorkflow) persistGeneratedContent(raw []byte) ([]byte, error) {
|
||||||
|
generatedText, normalized, err := w.handler.Validate(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "validate generated text", err)
|
return nil, w.reportError("validate generated text", err)
|
||||||
}
|
}
|
||||||
generatedTextPath, err := store.SaveGeneratedText(ctx, req.Resolved, normalized)
|
generatedTextPath, err := w.store.SaveGeneratedText(w.ctx, w.req.Resolved, normalized)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return nil, err
|
||||||
}
|
}
|
||||||
metadata.GeneratedTextPath = generatedTextPath
|
w.result.GeneratedTextPath = generatedTextPath
|
||||||
result.GeneratedTextPath = generatedTextPath
|
w.metadata.GeneratedTextPath = generatedTextPath
|
||||||
result.Metadata = metadata
|
w.result.Metadata = w.metadata
|
||||||
executionArtifact.Paths.GeneratedTextPath = generatedTextPath
|
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.GeneratedTextPath = generatedTextPath }); err != nil {
|
||||||
if _, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact); err != nil {
|
return nil, err
|
||||||
return result, err
|
|
||||||
}
|
}
|
||||||
metadataPath, err = store.SaveMetadata(ctx, metadata)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
result.MetadataPath = metadataPath
|
|
||||||
|
|
||||||
renderContext, err := handler.BuildRenderContext(briefingMetadata, moduleSnapshot, reportFacts.Collected, reportFacts.Derived, generatedText)
|
renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "build render context", err)
|
return nil, w.reportError("build render context", err)
|
||||||
}
|
}
|
||||||
renderContextPath, err := store.SaveRenderContext(ctx, req.Resolved, renderContext)
|
renderContextPath, err := w.store.SaveRenderContext(w.ctx, w.req.Resolved, renderContext)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return nil, err
|
||||||
}
|
}
|
||||||
metadata.RenderContextPath = renderContextPath
|
w.result.RenderContextPath = renderContextPath
|
||||||
result.RenderContextPath = renderContextPath
|
w.metadata.RenderContextPath = renderContextPath
|
||||||
result.Metadata = metadata
|
w.result.Metadata = w.metadata
|
||||||
executionArtifact.Paths.RenderContextPath = renderContextPath
|
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.RenderContextPath = renderContextPath }); err != nil {
|
||||||
if _, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact); err != nil {
|
return nil, err
|
||||||
return result, err
|
|
||||||
}
|
}
|
||||||
metadataPath, err = store.SaveMetadata(ctx, metadata)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
result.MetadataPath = metadataPath
|
|
||||||
|
|
||||||
rendered, err := handler.Render(renderContext)
|
rendered, err := w.handler.Render(renderContext)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, generatedReportError(req.Resolved, metadata.RunID, "render template", err)
|
return nil, w.reportError("render template", err)
|
||||||
}
|
}
|
||||||
reportPath, err := store.PrepareRenderedReport(ctx, req.Resolved)
|
return rendered, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *promptReportWorkflow) finalizeReport(rendered []byte) (*ReportResult, error) {
|
||||||
|
reportPath, err := w.store.PrepareRenderedReport(w.ctx, w.req.Resolved)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return w.result, err
|
||||||
}
|
}
|
||||||
if err := fileutil.WriteFileAtomic(reportPath, rendered); err != nil {
|
if err := fileutil.WriteFileAtomic(reportPath, rendered); err != nil {
|
||||||
return result, err
|
return w.result, err
|
||||||
}
|
}
|
||||||
result.ReportPath = reportPath
|
w.result.ReportPath = reportPath
|
||||||
metadata.RenderedReportPath = reportPath
|
w.metadata.RenderedReportPath = reportPath
|
||||||
result.Metadata = metadata
|
w.result.Metadata = w.metadata
|
||||||
executionArtifact.Paths.RenderedReportPath = reportPath
|
if err := w.persistReachedPath(func(paths *state.PromptExecutionPaths) { paths.RenderedReportPath = reportPath }); err != nil {
|
||||||
if _, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact); err != nil {
|
return w.result, err
|
||||||
return result, err
|
|
||||||
}
|
}
|
||||||
finalized, err := finalizeRenderedReport(ctx, finalizeRenderedReportRequest{
|
finalized, err := finalizeRenderedReport(w.ctx, finalizeRenderedReportRequest{
|
||||||
Config: req.Config, Store: store, Resolved: req.Resolved, Metadata: metadata, MetadataPath: result.MetadataPath,
|
Config: w.req.Config, Store: w.store, Resolved: w.req.Resolved, Metadata: w.metadata, MetadataPath: w.result.MetadataPath,
|
||||||
ExecutionArtifact: &executionArtifact, ManagedReportPath: reportPath, OutputPath: req.OutputPath,
|
ExecutionArtifact: &w.executionArtifact, ManagedReportPath: reportPath, OutputPath: w.req.OutputPath,
|
||||||
Notifier: req.Notifier, noNotify: req.noNotify,
|
Notifier: w.req.Notifier, noNotify: w.req.noNotify,
|
||||||
})
|
})
|
||||||
result.OutputPath, result.NotificationPath = finalized.OutputPath, finalized.NotificationPath
|
w.result.OutputPath, w.result.NotificationPath = finalized.OutputPath, finalized.NotificationPath
|
||||||
result.Metadata, result.MetadataPath, result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification
|
w.result.Metadata, w.result.MetadataPath, w.result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification
|
||||||
if err != nil {
|
return w.result, err
|
||||||
return result, err
|
|
||||||
}
|
}
|
||||||
return result, nil
|
|
||||||
|
func (w *promptReportWorkflow) persistReachedPath(update func(*state.PromptExecutionPaths)) error {
|
||||||
|
return persistReachedPromptPath(w.ctx, w.store, w.req.Resolved, &w.executionArtifact, update)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *promptReportWorkflow) persistReachedPathAndMetadata(update func(*state.PromptExecutionPaths)) error {
|
||||||
|
if err := w.persistReachedPath(update); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return w.saveMetadata()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *promptReportWorkflow) saveMetadata() error {
|
||||||
|
path, err := w.store.SaveMetadata(w.ctx, w.metadata)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
w.result.Metadata = w.metadata
|
||||||
|
w.result.MetadataPath = path
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *promptReportWorkflow) reportError(operation string, err error) error {
|
||||||
|
return generatedReportError(w.req.Resolved, w.metadata.RunID, operation, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func persistReachedPromptPath(
|
||||||
|
ctx context.Context,
|
||||||
|
store state.Store,
|
||||||
|
resolved report.Resolved,
|
||||||
|
artifact *state.PromptExecutionArtifact,
|
||||||
|
update func(*state.PromptExecutionPaths),
|
||||||
|
) error {
|
||||||
|
update(&artifact.Paths)
|
||||||
|
_, err := store.SavePromptExecution(ctx, resolved, *artifact)
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func failedPromptExecutionArtifact(resolved report.Resolved, metadata state.Metadata, inspection PromptInspectionResult, err error) state.PromptExecutionArtifact {
|
func failedPromptExecutionArtifact(resolved report.Resolved, metadata state.Metadata, inspection PromptInspectionResult, err error) state.PromptExecutionArtifact {
|
||||||
|
|||||||
Reference in New Issue
Block a user