package main import ( "context" "embed" "encoding/json" "fmt" "io" "net/http" "os" "strings" "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" ) //go:embed prompts/*.yaml schemas/*.json var assets embed.FS type scriptedTransport struct { proposalCalls int } func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error) { if err := req.Context().Err(); err != nil { return nil, err } var request struct { Model string `json:"model"` } if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return nil, err } switch request.Model { case "proposal-model": t.proposalCalls++ if t.proposalCalls == 1 { return httpResponse(http.StatusServiceUnavailable, `{"error":{}}`), nil } return modelResponse(`{"title":"Queue cleanup","risk":"low"}`, 10, 8), nil case "assessment-model": return modelResponse(`{"recommendation":"accept","confidence":0.9}`, 7, 5), nil default: return nil, fmt.Errorf("unexpected model in offline transport") } } func modelResponse(content string, promptTokens, completionTokens int) *http.Response { body, _ := json.Marshal(map[string]any{ "choices": []any{map[string]any{"message": map[string]string{"content": content}}}, "usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens}, }) return httpResponse(http.StatusOK, string(body)) } func httpResponse(status int, body string) *http.Response { return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))} } type eventLog struct{ events []promptkit.ExecutionEvent } func (l *eventLog) record(event promptkit.ExecutionEvent) { l.events = append(l.events, event) } func (l *eventLog) terminal() (promptkit.ExecutionEvent, error) { if len(l.events) == 0 || l.events[len(l.events)-1].Kind != promptkit.ExecutionFinished { return promptkit.ExecutionEvent{}, fmt.Errorf("execution did not finish") } return l.events[len(l.events)-1], nil } type summary struct { ProposalAttempts uint64 `json:"proposal_attempts"` ProposalTransientRetries uint64 `json:"proposal_transient_retries"` ProposalUsageComplete bool `json:"proposal_usage_complete"` AssessmentAttempts uint64 `json:"assessment_attempts"` AssessmentTransientRetries uint64 `json:"assessment_transient_retries"` AssessmentOutcome string `json:"assessment_outcome"` } func run() (summary, error) { transport := &scriptedTransport{} inputs, err := promptkit.FilesystemInputs("example prompts", assets, "prompts") if err != nil { return summary{}, err } backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model" inputs = append(inputs, configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{ backendID: promptkit.LocalBackend("https://example.invalid/v1", 1), })}, configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{ "proposal-profile": {Backend: &backendID, Model: &proposalModel}, "assessment-profile": {Backend: &backendID, Model: &assessmentModel}, })}, ) engine, err := promptkit.NewEngine(promptkit.Config{ Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}, HTTPClient: &http.Client{Transport: transport}, }, promptkit.WithSchemaFS(assets, "schemas")) if err != nil { return summary{}, err } var proposalLog eventLog proposal, err := engine.Run(context.Background(), promptkit.RunRequest{ PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, Observer: proposalLog.record, }) if err != nil { return summary{}, err } if !proposal.Validation.IsValid { return summary{}, fmt.Errorf("proposal failed structural validation") } proposalTerminal, err := proposalLog.terminal() if err != nil { return summary{}, err } var assessmentLog eventLog assessment, err := engine.Run(context.Background(), promptkit.RunRequest{ PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(proposal.RawOutput)}, Observer: assessmentLog.record, }) if err != nil { return summary{}, err } if !assessment.Validation.IsValid { return summary{}, fmt.Errorf("assessment failed structural validation") } assessmentTerminal, err := assessmentLog.terminal() if err != nil { return summary{}, err } var proposalValue struct { Risk string `json:"risk"` } var assessmentValue struct { Recommendation string `json:"recommendation"` } if err := json.Unmarshal([]byte(proposal.RawOutput), &proposalValue); err != nil { return summary{}, err } if err := json.Unmarshal([]byte(assessment.RawOutput), &assessmentValue); err != nil { return summary{}, err } decision := "rejected" if proposalValue.Risk == "low" && assessmentValue.Recommendation == "accept" { decision = "accepted" } return summary{ ProposalAttempts: proposalTerminal.Attempts, ProposalTransientRetries: proposalTerminal.TransientRetries, ProposalUsageComplete: proposalTerminal.Usage.PromptTokens.Complete, AssessmentAttempts: assessmentTerminal.Attempts, AssessmentTransientRetries: assessmentTerminal.TransientRetries, AssessmentOutcome: decision, }, nil } func main() { value, err := run() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } if err := json.NewEncoder(os.Stdout).Encode(value); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } }