diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go new file mode 100644 index 0000000..bd2ddb3 --- /dev/null +++ b/internal/usecase/runner.go @@ -0,0 +1,201 @@ +package usecase + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "gitea.maximumdirect.net/eric/scriptorium/internal/artifact" + "gitea.maximumdirect.net/eric/scriptorium/internal/domain" + "gitea.maximumdirect.net/eric/scriptorium/internal/llm" + "gitea.maximumdirect.net/eric/scriptorium/internal/profile" + "gitea.maximumdirect.net/eric/scriptorium/internal/prompt" + "gitea.maximumdirect.net/eric/scriptorium/internal/validate" +) + +var ( + ErrInvalidRequest = errors.New("invalid run request") + ErrProfileLoad = errors.New("failed to load profile") + ErrArtifactLoad = errors.New("failed to load artifact") + ErrPromptRender = errors.New("failed to render prompt") + ErrLLMGenerate = errors.New("failed to generate output") + ErrValidation = errors.New("failed to validate output") +) + +// Runner executes the Analyzer core use case. +type Runner struct { + profiles profile.Repository + artifacts artifact.Reader + renderer prompt.Renderer + llm llm.Client + validator validate.Validator +} + +func NewRunner( + profiles profile.Repository, + artifacts artifact.Reader, + renderer prompt.Renderer, + llmClient llm.Client, + validator validate.Validator, +) *Runner { + return &Runner{ + profiles: profiles, + artifacts: artifacts, + renderer: renderer, + llm: llmClient, + validator: validator, + } +} + +func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) { + if strings.TrimSpace(req.ProfileID) == "" { + return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest) + } + + start := time.Now().UTC() + + prof, err := r.profiles.GetProfile(ctx, req.ProfileID, req.ProfileVersion) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrProfileLoad, err) + } + + effectiveModel := mergeModelTarget(prof.ModelDefaults, req.Model) + effectiveContract := resolveOutputContract(prof, req.Validation) + + resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs)) + inputHashes := make(map[string]string, len(req.Inputs)) + for name, ref := range req.Inputs { + art, readErr := r.artifacts.Read(ctx, ref) + if readErr != nil { + return nil, fmt.Errorf("%w: input %q: %v", ErrArtifactLoad, name, readErr) + } + if art.Name == "" { + art.Name = name + } + resolvedInputs[name] = art + inputHashes[name] = art.Hash + } + + renderedPrompt, err := r.renderer.Render(ctx, prof, resolvedInputs, req.Vars) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrPromptRender, err) + } + + promptHash := hashRenderedPrompt(*renderedPrompt) + + genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{ + Prompt: *renderedPrompt, + Target: effectiveModel, + }) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrLLMGenerate, err) + } + + outputArtifact := buildOutputArtifact(genResp.Content, effectiveContract.Format) + + validationResult := domain.ValidationResult{ + Status: domain.ValidationSkipped, + Mode: effectiveContract.ValidationMode, + SchemaPath: effectiveContract.SchemaPath, + RepairAttempts: effectiveContract.RepairAttempts, + IsValid: true, + } + if r.validator != nil && effectiveContract.ValidationMode != domain.ValidationNone { + validationResult, err = r.validator.Validate(ctx, &outputArtifact, effectiveContract) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrValidation, err) + } + } + + end := time.Now().UTC() + + return &domain.RunResult{ + Artifact: outputArtifact, + RawOutput: genResp.Content, + Validation: validationResult, + ProfileID: prof.ID, + ProfileVersion: prof.Version, + ModelName: effectiveModel.Model, + Endpoint: effectiveModel.Endpoint, + InputHashes: inputHashes, + PromptHash: promptHash, + Usage: genResp.Usage, + StartTime: start, + EndTime: end, + }, nil +} + +func mergeModelTarget(base domain.ModelTarget, override *domain.ModelTarget) domain.ModelTarget { + if override == nil { + return base + } + + out := base + if override.Endpoint != "" { + out.Endpoint = override.Endpoint + } + if override.Model != "" { + out.Model = override.Model + } + if override.Temperature != 0 { + out.Temperature = override.Temperature + } + if override.MaxTokens != 0 { + out.MaxTokens = override.MaxTokens + } + if override.TopP != 0 { + out.TopP = override.TopP + } + return out +} + +func resolveOutputContract(prof *domain.PromptProfile, override *domain.OutputContract) domain.OutputContract { + contract := prof.Validation + if contract.Format == "" { + contract.Format = prof.OutputFormat + } + if override != nil { + contract = *override + } + if contract.Format == "" { + contract.Format = domain.FormatText + } + return contract +} + +func hashRenderedPrompt(p domain.RenderedPrompt) string { + var b strings.Builder + for _, msg := range p.Messages { + b.WriteString(msg.Role) + b.WriteByte('\n') + b.WriteString(msg.Content) + b.WriteString("\n---\n") + } + h := sha256.Sum256([]byte(b.String())) + return hex.EncodeToString(h[:]) +} + +func buildOutputArtifact(content string, format domain.OutputFormat) domain.Artifact { + body := []byte(content) + hash := sha256.Sum256(body) + + contentType := "text/plain" + switch format { + case domain.FormatMarkdown: + contentType = "text/markdown" + case domain.FormatJSON: + contentType = "application/json" + } + + return domain.Artifact{ + Name: "output", + ContentType: contentType, + Body: body, + Size: int64(len(body)), + Hash: hex.EncodeToString(hash[:]), + } +} diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go new file mode 100644 index 0000000..6c2a040 --- /dev/null +++ b/internal/usecase/runner_test.go @@ -0,0 +1,398 @@ +package usecase + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "testing" + + "gitea.maximumdirect.net/eric/scriptorium/internal/domain" + "gitea.maximumdirect.net/eric/scriptorium/internal/validate" +) + +type fakeProfileRepo struct { + profile *domain.PromptProfile + err error + lastID string + lastVersion string +} + +func (f *fakeProfileRepo) GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error) { + f.lastID = id + f.lastVersion = version + if f.err != nil { + return nil, f.err + } + return f.profile, nil +} + +type fakeArtifactReader struct { + artifactsByURI map[string]*domain.Artifact + errByURI map[string]error +} + +func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { + if err, ok := f.errByURI[ref.URI]; ok { + return nil, err + } + if art, ok := f.artifactsByURI[ref.URI]; ok { + copy := *art + return ©, nil + } + return nil, errors.New("artifact not found") +} + +type fakeRenderer struct { + rendered *domain.RenderedPrompt + err error +} + +func (f *fakeRenderer) Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) { + if f.err != nil { + return nil, f.err + } + return f.rendered, nil +} + +type fakeLLM struct { + resp *domain.GenerateResponse + err error + lastReq domain.GenerateRequest +} + +func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { + f.lastReq = req + if f.err != nil { + return nil, f.err + } + return f.resp, nil +} + +type fakeValidator struct { + result domain.ValidationResult + err error + called bool + lastContract domain.OutputContract +} + +func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { + f.called = true + f.lastContract = contract + if f.err != nil { + return domain.ValidationResult{}, f.err + } + return f.result, nil +} + +func TestRunnerRunSuccessful(t *testing.T) { + repo := &fakeProfileRepo{ + profile: &domain.PromptProfile{ + ID: "p1", + Version: "1.0.0", + OutputFormat: domain.FormatMarkdown, + ModelDefaults: domain.ModelTarget{ + Endpoint: "ep1", + Model: "model-default", + Temperature: 0.4, + MaxTokens: 200, + TopP: 0.9, + }, + Validation: domain.OutputContract{ + ValidationMode: domain.ValidationBasic, + }, + }, + } + + reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ + "a://t": {Body: []byte("transcript body"), Hash: hashString("transcript body")}, + "a://g": {Body: []byte("glossary body"), Hash: hashString("glossary body")}, + }} + + renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{ + {Role: "system", Content: "System context"}, + {Role: "user", Content: "Please summarize"}, + }}} + + llmClient := &fakeLLM{resp: &domain.GenerateResponse{ + Content: "# recap\n- item", + Usage: domain.TokenUsage{ + PromptTokens: 10, + CompletionTokens: 20, + TotalTokens: 30, + }, + }} + + runner := NewRunner(repo, reader, renderer, llmClient, nil) + + res, err := runner.Run(context.Background(), domain.RunRequest{ + ProfileID: "p1", + ProfileVersion: "1.0.0", + Inputs: map[string]domain.ArtifactRef{ + "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, + "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, + }, + Model: &domain.ModelTarget{ + Model: "model-override", + Temperature: 0, + MaxTokens: 0, + }, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" { + t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion) + } + if res.ModelName != "model-override" { + t.Fatalf("expected model override to apply, got %q", res.ModelName) + } + if res.Endpoint != "ep1" { + t.Fatalf("expected endpoint from profile default, got %q", res.Endpoint) + } + if res.Artifact.ContentType != "text/markdown" { + t.Fatalf("expected markdown content type, got %q", res.Artifact.ContentType) + } + if string(res.Artifact.Body) != "# recap\n- item" { + t.Fatalf("unexpected artifact body: %q", string(res.Artifact.Body)) + } + if res.RawOutput != "# recap\n- item" { + t.Fatalf("expected raw output preserved, got %q", res.RawOutput) + } + if res.Validation.Status != domain.ValidationSkipped { + t.Fatalf("expected skipped validation, got %q", res.Validation.Status) + } + if res.Validation.Mode != domain.ValidationBasic { + t.Fatalf("expected validation mode basic in skipped result, got %q", res.Validation.Mode) + } + if res.PromptHash == "" { + t.Fatal("expected non-empty prompt hash") + } + if res.Usage.TotalTokens != 30 { + t.Fatalf("expected usage to propagate, got %+v", res.Usage) + } + if res.StartTime.IsZero() || res.EndTime.IsZero() { + t.Fatal("expected start and end times") + } + if res.EndTime.Before(res.StartTime) { + t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime) + } + + if got := res.InputHashes["transcript"]; got != hashString("transcript body") { + t.Fatalf("unexpected transcript hash: %q", got) + } + if got := res.InputHashes["glossary"]; got != hashString("glossary body") { + t.Fatalf("unexpected glossary hash: %q", got) + } + + if llmClient.lastReq.Target.Temperature != 0.4 { + t.Fatalf("expected zero-valued request field not to override default temperature, got %v", llmClient.lastReq.Target.Temperature) + } +} + +func TestRunnerRunProfileLoadFailure(t *testing.T) { + runner := NewRunner( + &fakeProfileRepo{err: errors.New("boom")}, + &fakeArtifactReader{}, + &fakeRenderer{}, + &fakeLLM{}, + nil, + ) + + _, err := runner.Run(context.Background(), domain.RunRequest{ProfileID: "p"}) + if !errors.Is(err, ErrProfileLoad) { + t.Fatalf("expected ErrProfileLoad, got %v", err) + } +} + +func TestRunnerRunArtifactLoadFailure(t *testing.T) { + runner := NewRunner( + &fakeProfileRepo{profile: minimalProfile()}, + &fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}}, + &fakeRenderer{rendered: &domain.RenderedPrompt{}}, + &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, + nil, + ) + + _, err := runner.Run(context.Background(), domain.RunRequest{ + ProfileID: "p", + Inputs: map[string]domain.ArtifactRef{ + "transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}, + }, + }) + if !errors.Is(err, ErrArtifactLoad) { + t.Fatalf("expected ErrArtifactLoad, got %v", err) + } +} + +func TestRunnerRunPromptRenderFailure(t *testing.T) { + runner := NewRunner( + &fakeProfileRepo{profile: minimalProfile()}, + &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, + &fakeRenderer{err: errors.New("render failed")}, + &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, + nil, + ) + + _, err := runner.Run(context.Background(), domain.RunRequest{ + ProfileID: "p", + Inputs: map[string]domain.ArtifactRef{ + "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, + }, + }) + if !errors.Is(err, ErrPromptRender) { + t.Fatalf("expected ErrPromptRender, got %v", err) + } +} + +func TestRunnerRunLLMFailure(t *testing.T) { + runner := NewRunner( + &fakeProfileRepo{profile: minimalProfile()}, + &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, + &fakeRenderer{rendered: &domain.RenderedPrompt{}}, + &fakeLLM{err: errors.New("llm failed")}, + nil, + ) + + _, err := runner.Run(context.Background(), domain.RunRequest{ + ProfileID: "p", + Inputs: map[string]domain.ArtifactRef{ + "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, + }, + }) + if !errors.Is(err, ErrLLMGenerate) { + t.Fatalf("expected ErrLLMGenerate, got %v", err) + } +} + +func TestRunnerRunValidationFailureNonError(t *testing.T) { + validator := &fakeValidator{result: domain.ValidationResult{ + Status: domain.ValidationFailed, + Mode: domain.ValidationBasic, + Errors: []string{"bad output"}, + IsValid: false, + }} + + runner := NewRunner( + &fakeProfileRepo{profile: minimalProfile()}, + &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, + &fakeRenderer{rendered: &domain.RenderedPrompt{}}, + &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, + validator, + ) + + res, err := runner.Run(context.Background(), domain.RunRequest{ + ProfileID: "p", + Inputs: map[string]domain.ArtifactRef{ + "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, + }, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Validation.Status != domain.ValidationFailed { + t.Fatalf("expected validation failed result, got %q", res.Validation.Status) + } + if res.RawOutput != "raw output" { + t.Fatalf("expected raw output preserved, got %q", res.RawOutput) + } + if !validator.called { + t.Fatal("expected validator to be called") + } +} + +func TestRunnerRunValidationRuntimeError(t *testing.T) { + validator := &fakeValidator{err: errors.New("validator unavailable")} + + runner := NewRunner( + &fakeProfileRepo{profile: minimalProfile()}, + &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, + &fakeRenderer{rendered: &domain.RenderedPrompt{}}, + &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, + validator, + ) + + _, err := runner.Run(context.Background(), domain.RunRequest{ + ProfileID: "p", + Inputs: map[string]domain.ArtifactRef{ + "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, + }, + }) + if !errors.Is(err, ErrValidation) { + t.Fatalf("expected ErrValidation, got %v", err) + } +} + +func TestRunnerRunValidationFailureWithRealValidatorPreservesRawOutput(t *testing.T) { + tmp := t.TempDir() + if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["name"], + "properties": { + "name": {"type": "string"} + } +}`), 0644); err != nil { + t.Fatal(err) + } + + runner := NewRunner( + &fakeProfileRepo{profile: &domain.PromptProfile{ + ID: "p-json", + Version: "1", + OutputFormat: domain.FormatJSON, + ModelDefaults: domain.ModelTarget{ + Endpoint: "ep", + Model: "m", + }, + Validation: domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schema.json", + Format: domain.FormatJSON, + }, + }}, + &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, + &fakeRenderer{rendered: &domain.RenderedPrompt{}}, + &fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}}, + validate.NewStandardValidator(tmp), + ) + + res, err := runner.Run(context.Background(), domain.RunRequest{ + ProfileID: "p-json", + Inputs: map[string]domain.ArtifactRef{ + "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, + }, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Validation.Status != domain.ValidationFailed { + t.Fatalf("expected validation failed, got %q", res.Validation.Status) + } + if res.RawOutput != `{"count":1}` { + t.Fatalf("expected raw output preserved, got %q", res.RawOutput) + } +} + +func minimalProfile() *domain.PromptProfile { + return &domain.PromptProfile{ + ID: "p", + Version: "1", + OutputFormat: domain.FormatText, + ModelDefaults: domain.ModelTarget{ + Endpoint: "ep", + Model: "m", + }, + Validation: domain.OutputContract{ + ValidationMode: domain.ValidationBasic, + }, + } +} + +func hashString(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +}