package usecase import ( "context" "crypto/sha256" "encoding/hex" "errors" "os" "path/filepath" "testing" "gitea.maximumdirect.net/eric/scriptorium/internal/domain" "gitea.maximumdirect.net/eric/scriptorium/internal/validate" ) type fakeProfileRepo struct { profile *domain.PromptProfile err error lastID string lastVersion string } func (f *fakeProfileRepo) GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error) { f.lastID = id f.lastVersion = version if f.err != nil { return nil, f.err } return f.profile, nil } type fakeArtifactReader struct { artifactsByURI map[string]*domain.Artifact errByURI map[string]error } func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { if err, ok := f.errByURI[ref.URI]; ok { return nil, err } if art, ok := f.artifactsByURI[ref.URI]; ok { copy := *art return ©, nil } return nil, errors.New("artifact not found") } type fakeRenderer struct { rendered *domain.RenderedPrompt err error } func (f *fakeRenderer) Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) { if f.err != nil { return nil, f.err } return f.rendered, nil } type fakeLLM struct { resp *domain.GenerateResponse err error lastReq domain.GenerateRequest } func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { f.lastReq = req if f.err != nil { return nil, f.err } return f.resp, nil } type fakeValidator struct { result domain.ValidationResult err error called bool lastContract domain.OutputContract } func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { f.called = true f.lastContract = contract if f.err != nil { return domain.ValidationResult{}, f.err } return f.result, nil } type fakeRepairer struct { responses []*domain.GenerateResponse err error calls int lastReq RepairRequest } func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) { f.calls++ f.lastReq = req if f.err != nil { return nil, f.err } if len(f.responses) == 0 { return nil, errors.New("no repair response configured") } idx := f.calls - 1 if idx >= len(f.responses) { idx = len(f.responses) - 1 } return f.responses[idx], nil } func TestRunnerRunSuccessful(t *testing.T) { repo := &fakeProfileRepo{ profile: &domain.PromptProfile{ ID: "p1", Version: "1.0.0", OutputFormat: domain.FormatMarkdown, ModelDefaults: domain.ModelTarget{ Endpoint: "ep1", Model: "model-default", Temperature: 0.4, MaxTokens: 200, TopP: 0.9, }, Validation: domain.OutputContract{ ValidationMode: domain.ValidationBasic, }, }, } reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ "a://t": {Body: []byte("transcript body"), Hash: hashString("transcript body")}, "a://g": {Body: []byte("glossary body"), Hash: hashString("glossary body")}, }} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{ {Role: "system", Content: "System context"}, {Role: "user", Content: "Please summarize"}, }}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{ Content: "# recap\n- item", Usage: domain.TokenUsage{ PromptTokens: 10, CompletionTokens: 20, TotalTokens: 30, }, }} runner := NewRunner(repo, reader, renderer, llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p1", ProfileVersion: "1.0.0", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, }, Model: &domain.ModelTarget{ Model: "model-override", Temperature: 0, MaxTokens: 0, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" { t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion) } if res.ModelName != "model-override" { t.Fatalf("expected model override to apply, got %q", res.ModelName) } if res.Endpoint != "ep1" { t.Fatalf("expected endpoint from profile default, got %q", res.Endpoint) } if res.Artifact.ContentType != "text/markdown" { t.Fatalf("expected markdown content type, got %q", res.Artifact.ContentType) } if string(res.Artifact.Body) != "# recap\n- item" { t.Fatalf("unexpected artifact body: %q", string(res.Artifact.Body)) } if res.RawOutput != "# recap\n- item" { t.Fatalf("expected raw output preserved, got %q", res.RawOutput) } if res.Validation.Status != domain.ValidationSkipped { t.Fatalf("expected skipped validation, got %q", res.Validation.Status) } if res.Validation.Mode != domain.ValidationBasic { t.Fatalf("expected validation mode basic in skipped result, got %q", res.Validation.Mode) } if res.PromptHash == "" { t.Fatal("expected non-empty prompt hash") } if res.Usage.TotalTokens != 30 { t.Fatalf("expected usage to propagate, got %+v", res.Usage) } if res.StartTime.IsZero() || res.EndTime.IsZero() { t.Fatal("expected start and end times") } if res.EndTime.Before(res.StartTime) { t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime) } if got := res.InputHashes["transcript"]; got != hashString("transcript body") { t.Fatalf("unexpected transcript hash: %q", got) } if got := res.InputHashes["glossary"]; got != hashString("glossary body") { t.Fatalf("unexpected glossary hash: %q", got) } if llmClient.lastReq.Target.Temperature != 0.4 { t.Fatalf("expected zero-valued request field not to override default temperature, got %v", llmClient.lastReq.Target.Temperature) } } func TestRunnerRunProfileLoadFailure(t *testing.T) { runner := NewRunner( &fakeProfileRepo{err: errors.New("boom")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil, ) _, err := runner.Run(context.Background(), domain.RunRequest{ProfileID: "p"}) if !errors.Is(err, ErrProfileLoad) { t.Fatalf("expected ErrProfileLoad, got %v", err) } } func TestRunnerRunArtifactLoadFailure(t *testing.T) { runner := NewRunner( &fakeProfileRepo{profile: minimalProfile()}, &fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, ) _, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}, }, }) if !errors.Is(err, ErrArtifactLoad) { t.Fatalf("expected ErrArtifactLoad, got %v", err) } } func TestRunnerRunPromptRenderFailure(t *testing.T) { runner := NewRunner( &fakeProfileRepo{profile: minimalProfile()}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{err: errors.New("render failed")}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, ) _, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, }, }) if !errors.Is(err, ErrPromptRender) { t.Fatalf("expected ErrPromptRender, got %v", err) } } func TestRunnerRunLLMFailure(t *testing.T) { runner := NewRunner( &fakeProfileRepo{profile: minimalProfile()}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{err: errors.New("llm failed")}, nil, ) _, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, }, }) if !errors.Is(err, ErrLLMGenerate) { t.Fatalf("expected ErrLLMGenerate, got %v", err) } } func TestRunnerRunValidationFailureNonError(t *testing.T) { validator := &fakeValidator{result: domain.ValidationResult{ Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad output"}, IsValid: false, }} runner := NewRunner( &fakeProfileRepo{profile: minimalProfile()}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, validator, ) res, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.Validation.Status != domain.ValidationFailed { t.Fatalf("expected validation failed result, got %q", res.Validation.Status) } if res.RawOutput != "raw output" { t.Fatalf("expected raw output preserved, got %q", res.RawOutput) } if !validator.called { t.Fatal("expected validator to be called") } } func TestRunnerRunValidationRuntimeError(t *testing.T) { validator := &fakeValidator{err: errors.New("validator unavailable")} runner := NewRunner( &fakeProfileRepo{profile: minimalProfile()}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, validator, ) _, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, }, }) if !errors.Is(err, ErrValidation) { t.Fatalf("expected ErrValidation, got %v", err) } } func TestRunnerRunValidationFailureWithRealValidatorPreservesRawOutput(t *testing.T) { tmp := t.TempDir() if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "required": ["name"], "properties": { "name": {"type": "string"} } }`), 0644); err != nil { t.Fatal(err) } runner := NewRunner( &fakeProfileRepo{profile: &domain.PromptProfile{ ID: "p-json", Version: "1", OutputFormat: domain.FormatJSON, ModelDefaults: domain.ModelTarget{ Endpoint: "ep", Model: "m", }, Validation: domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json", Format: domain.FormatJSON, }, }}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}}, validate.NewStandardValidator(tmp), ) res, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p-json", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.Validation.Status != domain.ValidationFailed { t.Fatalf("expected validation failed, got %q", res.Validation.Status) } if res.RawOutput != `{"count":1}` { t.Fatalf("expected raw output preserved, got %q", res.RawOutput) } } func TestRunnerRunNoRepairWhenDisabled(t *testing.T) { repairer := &fakeRepairer{ responses: []*domain.GenerateResponse{{Content: `{"ok":true}`}}, } runner := NewRunnerWithRepairer( &fakeProfileRepo{profile: &domain.PromptProfile{ ID: "p-json", Version: "1", OutputFormat: domain.FormatJSON, ModelDefaults: domain.ModelTarget{ Endpoint: "ep", Model: "m", }, Validation: domain.OutputContract{ ValidationMode: domain.ValidationJSON, Format: domain.FormatJSON, RepairAttempts: 0, }, }}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`}}, validate.NewStandardValidator(t.TempDir()), repairer, ) res, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p-json", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if repairer.calls != 0 { t.Fatalf("expected no repair calls, got %d", repairer.calls) } if res.Validation.Status != domain.ValidationFailed { t.Fatalf("expected failed validation, got %q", res.Validation.Status) } if res.RawOutput != `{"broken":` { t.Fatalf("expected original output preserved, got %q", res.RawOutput) } if res.Validation.RepairAttempts != 0 { t.Fatalf("expected repair attempts 0, got %d", res.Validation.RepairAttempts) } } func TestRunnerRunSuccessfulRepairAfterInvalidJSON(t *testing.T) { repairer := &fakeRepairer{ responses: []*domain.GenerateResponse{{Content: `{"ok":true}`, Usage: domain.TokenUsage{TotalTokens: 5}}}, } runner := NewRunnerWithRepairer( &fakeProfileRepo{profile: &domain.PromptProfile{ ID: "p-json", Version: "1", OutputFormat: domain.FormatJSON, ModelDefaults: domain.ModelTarget{ Endpoint: "ep", Model: "m", }, Validation: domain.OutputContract{ ValidationMode: domain.ValidationJSON, Format: domain.FormatJSON, RepairAttempts: 1, }, }}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`, Usage: domain.TokenUsage{TotalTokens: 3}}}, validate.NewStandardValidator(t.TempDir()), repairer, ) res, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p-json", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if repairer.calls != 1 { t.Fatalf("expected one repair call, got %d", repairer.calls) } if res.Validation.Status != domain.ValidationPassed { t.Fatalf("expected passed validation, got %q", res.Validation.Status) } if res.Validation.RepairAttempts != 1 { t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts) } if res.RawOutput != `{"ok":true}` { t.Fatalf("expected repaired output, got %q", res.RawOutput) } } func TestRunnerRunSuccessfulRepairAfterSchemaFailure(t *testing.T) { tmp := t.TempDir() if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "required": ["name"], "properties": { "name": {"type": "string"} } }`), 0644); err != nil { t.Fatal(err) } repairer := &fakeRepairer{ responses: []*domain.GenerateResponse{{Content: `{"name":"eris"}`}}, } runner := NewRunnerWithRepairer( &fakeProfileRepo{profile: &domain.PromptProfile{ ID: "p-json", Version: "1", OutputFormat: domain.FormatJSON, ModelDefaults: domain.ModelTarget{ Endpoint: "ep", Model: "m", }, Validation: domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json", Format: domain.FormatJSON, RepairAttempts: 1, }, }}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}}, validate.NewStandardValidator(tmp), repairer, ) res, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p-json", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.Validation.Status != domain.ValidationPassed { t.Fatalf("expected passed validation, got %q", res.Validation.Status) } if res.Validation.RepairAttempts != 1 { t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts) } if res.RawOutput != `{"name":"eris"}` { t.Fatalf("expected repaired output, got %q", res.RawOutput) } } func TestRunnerRunFailedRepairPreservesRawOutputAndErrors(t *testing.T) { repairer := &fakeRepairer{ responses: []*domain.GenerateResponse{ {Content: `{"repair1":`}, {Content: `{"repair2":`}, }, } runner := NewRunnerWithRepairer( &fakeProfileRepo{profile: &domain.PromptProfile{ ID: "p-json", Version: "1", OutputFormat: domain.FormatJSON, ModelDefaults: domain.ModelTarget{ Endpoint: "ep", Model: "m", }, Validation: domain.OutputContract{ ValidationMode: domain.ValidationJSON, Format: domain.FormatJSON, RepairAttempts: 2, }, }}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}}, validate.NewStandardValidator(t.TempDir()), repairer, ) res, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p-json", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if res.Validation.Status != domain.ValidationFailed { t.Fatalf("expected failed validation, got %q", res.Validation.Status) } if len(res.Validation.Errors) == 0 { t.Fatal("expected validation errors after failed repair") } if res.Validation.RepairAttempts != 2 { t.Fatalf("expected repair attempts 2, got %d", res.Validation.RepairAttempts) } if res.RawOutput != `{"repair2":` { t.Fatalf("expected final repaired output preserved, got %q", res.RawOutput) } } func TestRunnerRunRepairAttemptsBounded(t *testing.T) { repairer := &fakeRepairer{ responses: []*domain.GenerateResponse{ {Content: `{"repair1":`}, {Content: `{"repair2":`}, {Content: `{"repair3":`}, }, } runner := NewRunnerWithRepairer( &fakeProfileRepo{profile: &domain.PromptProfile{ ID: "p-json", Version: "1", OutputFormat: domain.FormatJSON, ModelDefaults: domain.ModelTarget{ Endpoint: "ep", Model: "m", }, Validation: domain.OutputContract{ ValidationMode: domain.ValidationJSON, Format: domain.FormatJSON, RepairAttempts: 1, }, }}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}}, validate.NewStandardValidator(t.TempDir()), repairer, ) res, err := runner.Run(context.Background(), domain.RunRequest{ ProfileID: "p-json", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, }, }) if err != nil { t.Fatalf("expected no error, got %v", err) } if repairer.calls != 1 { t.Fatalf("expected repair calls bounded to 1, got %d", repairer.calls) } if res.Validation.RepairAttempts != 1 { t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts) } } func minimalProfile() *domain.PromptProfile { return &domain.PromptProfile{ ID: "p", Version: "1", OutputFormat: domain.FormatText, ModelDefaults: domain.ModelTarget{ Endpoint: "ep", Model: "m", }, Validation: domain.OutputContract{ ValidationMode: domain.ValidationBasic, }, } } func hashString(s string) string { sum := sha256.Sum256([]byte(s)) return hex.EncodeToString(sum[:]) }