From 5a00ca81a2fa813bff3b8ddfe0ee8d789615e68e Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 28 Jul 2026 14:07:59 +0000 Subject: [PATCH] Remove the duplicated prompt framework --- artifact_reader.go | 39 - artifact_reader_internal_test.go | 36 - convert.go | 406 --- engine.go | 343 --- engine_test.go | 2559 ----------------- errors.go | 60 - examples/go-library/prepare/main.go | 50 - formatting.go | 51 - go.mod | 6 +- internal/adapter/cli/run.go | 2 +- internal/adapter/cli/run_test.go | 4 +- internal/artifact/reader.go | 122 - internal/artifact/reader_test.go | 105 - internal/defaults/defaults.go | 30 +- internal/domain/domain.go | 288 -- internal/domain/prepared_run_test.go | 141 - internal/filecatalog/catalog.go | 142 - internal/filecatalog/catalog_test.go | 270 -- internal/llm/client.go | 11 - internal/llm/openai_compatible_client.go | 385 --- internal/llm/openai_compatible_client_test.go | 1069 ------- .../builtin/assets/aion-labs/aion-2.yml | 9 - .../assets/anthropic/claude-fable-latest.yml | 7 - .../assets/anthropic/claude-haiku-latest.yml | 7 - .../assets/anthropic/claude-opus-latest.yml | 7 - .../assets/anthropic/claude-sonnet-latest.yml | 7 - .../builtin/assets/deepseek/deepseek-3-2.yml | 7 - .../assets/deepseek/deepseek-4-flash.yml | 7 - .../assets/deepseek/deepseek-4-pro.yml | 7 - .../assets/google/gemini-2-flash-lite.yml | 9 - .../builtin/assets/google/gemini-2-flash.yml | 9 - .../builtin/assets/google/gemini-2-pro.yml | 9 - .../assets/google/gemini-3-flash-lite.yml | 9 - .../assets/google/gemini-flash-latest.yml | 9 - .../assets/google/gemini-pro-latest.yml | 9 - .../builtin/assets/google/gemma-4-31b.yml | 9 - .../builtin/assets/minimax/minimax-m2.yml | 9 - .../builtin/assets/minimax/minimax-m3.yml | 9 - .../assets/mistral/mistral-large-2512.yml | 7 - .../assets/mistral/mistral-medium-3-5.yml | 8 - .../assets/mistral/mistral-small-3.yml | 7 - .../assets/mistral/mistral-small-4.yml | 8 - .../assets/nvidia/nemotron-3-ultra.yml | 7 - .../builtin/assets/openai/gpt-5-mini.yml | 7 - .../builtin/assets/openai/gpt-5-nano.yml | 7 - internal/profile/builtin/repository.go | 31 - internal/profile/builtin/repository_test.go | 127 - internal/profile/filesystem_repository.go | 213 -- internal/profile/repository.go | 12 - internal/profile/repository_test.go | 479 --- internal/profile/testdata/invalid_yaml.yaml | 3 - .../profile/testdata/missing_endpoint.yaml | 2 - internal/profile/testdata/missing_id.yaml | 2 - internal/profile/testdata/missing_model.yaml | 2 - internal/profile/testdata/raw_api_key.yaml | 4 - internal/profile/testdata/unknown_field.yaml | 4 - .../profile/testdata/valid_local_profile.yaml | 7 - .../testdata/valid_with_api_key_env.yaml | 8 - internal/prompt/go_renderer.go | 125 - internal/prompt/renderer.go | 11 - internal/prompt/renderer_test.go | 345 --- internal/promptdef/filesystem_repository.go | 484 ---- internal/promptdef/repository.go | 12 - internal/promptdef/repository_test.go | 526 ---- .../both_content_and_content_file.yaml | 10 - .../testdata/duplicate_input_names.yaml | 14 - .../testdata/empty_cache_control_type.yaml | 10 - .../testdata/invalid_validation_mode.yaml | 9 - internal/promptdef/testdata/invalid_yaml.yaml | 9 - .../json_schema_without_schema_path.yaml | 9 - .../testdata/messages/user_prompt.tmpl | 2 - .../testdata/missing_content_file.yaml | 9 - internal/promptdef/testdata/missing_id.yaml | 8 - .../neither_content_nor_content_file.yaml | 8 - internal/promptdef/testdata/no_messages.yaml | 6 - .../testdata/unknown_cache_control_field.yaml | 12 - .../testdata/unknown_input_field.yaml | 13 - .../unsupported_cache_control_ttl.yaml | 12 - .../unsupported_cache_control_type.yaml | 11 - .../testdata/valid_cache_control_ttl.yaml | 14 - .../valid_cache_control_without_ttl.yaml | 13 - .../promptdef/testdata/valid_file_backed.yaml | 14 - internal/promptdef/testdata/valid_inline.yaml | 18 - .../promptdef/testdata/valid_session_id.yaml | 10 - .../testdata/with_default_profile.yaml | 13 - internal/usecase/repairer.go | 76 - internal/usecase/runner.go | 576 ---- internal/usecase/runner_test.go | 1810 ------------ internal/validate/standard_validator.go | 292 -- internal/validate/standard_validator_test.go | 383 --- internal/validate/validator.go | 16 - json_copy.go | 218 -- llm_adapter.go | 23 - profiles.go | 124 - testdata/framework/fixtures/glossary.yml | 2 - testdata/framework/fixtures/transcript.md | 2 - .../framework/profiles/contract-fast.yaml | 7 - .../framework/profiles/contract-quality.yaml | 7 - .../contract.markdown_summary.system.md | 1 - .../prompts/contract.markdown_summary.user.md | 7 - .../prompts/contract.markdown_summary.yaml | 20 - .../contract.structured_events.system.md | 1 - .../contract.structured_events.user.md | 7 - .../prompts/contract.structured_events.yaml | 21 - .../schemas/structured_events.schema.json | 19 - types.go | 306 -- 106 files changed, 9 insertions(+), 12839 deletions(-) delete mode 100644 artifact_reader.go delete mode 100644 artifact_reader_internal_test.go delete mode 100644 convert.go delete mode 100644 engine.go delete mode 100644 engine_test.go delete mode 100644 errors.go delete mode 100644 examples/go-library/prepare/main.go delete mode 100644 formatting.go delete mode 100644 internal/artifact/reader.go delete mode 100644 internal/artifact/reader_test.go delete mode 100644 internal/domain/domain.go delete mode 100644 internal/domain/prepared_run_test.go delete mode 100644 internal/filecatalog/catalog.go delete mode 100644 internal/filecatalog/catalog_test.go delete mode 100644 internal/llm/client.go delete mode 100644 internal/llm/openai_compatible_client.go delete mode 100644 internal/llm/openai_compatible_client_test.go delete mode 100644 internal/profile/builtin/assets/aion-labs/aion-2.yml delete mode 100644 internal/profile/builtin/assets/anthropic/claude-fable-latest.yml delete mode 100644 internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml delete mode 100644 internal/profile/builtin/assets/anthropic/claude-opus-latest.yml delete mode 100644 internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml delete mode 100644 internal/profile/builtin/assets/deepseek/deepseek-3-2.yml delete mode 100644 internal/profile/builtin/assets/deepseek/deepseek-4-flash.yml delete mode 100644 internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml delete mode 100644 internal/profile/builtin/assets/google/gemini-2-flash-lite.yml delete mode 100644 internal/profile/builtin/assets/google/gemini-2-flash.yml delete mode 100644 internal/profile/builtin/assets/google/gemini-2-pro.yml delete mode 100644 internal/profile/builtin/assets/google/gemini-3-flash-lite.yml delete mode 100644 internal/profile/builtin/assets/google/gemini-flash-latest.yml delete mode 100644 internal/profile/builtin/assets/google/gemini-pro-latest.yml delete mode 100644 internal/profile/builtin/assets/google/gemma-4-31b.yml delete mode 100644 internal/profile/builtin/assets/minimax/minimax-m2.yml delete mode 100644 internal/profile/builtin/assets/minimax/minimax-m3.yml delete mode 100644 internal/profile/builtin/assets/mistral/mistral-large-2512.yml delete mode 100644 internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml delete mode 100644 internal/profile/builtin/assets/mistral/mistral-small-3.yml delete mode 100644 internal/profile/builtin/assets/mistral/mistral-small-4.yml delete mode 100644 internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml delete mode 100644 internal/profile/builtin/assets/openai/gpt-5-mini.yml delete mode 100644 internal/profile/builtin/assets/openai/gpt-5-nano.yml delete mode 100644 internal/profile/builtin/repository.go delete mode 100644 internal/profile/builtin/repository_test.go delete mode 100644 internal/profile/filesystem_repository.go delete mode 100644 internal/profile/repository.go delete mode 100644 internal/profile/repository_test.go delete mode 100644 internal/profile/testdata/invalid_yaml.yaml delete mode 100644 internal/profile/testdata/missing_endpoint.yaml delete mode 100644 internal/profile/testdata/missing_id.yaml delete mode 100644 internal/profile/testdata/missing_model.yaml delete mode 100644 internal/profile/testdata/raw_api_key.yaml delete mode 100644 internal/profile/testdata/unknown_field.yaml delete mode 100644 internal/profile/testdata/valid_local_profile.yaml delete mode 100644 internal/profile/testdata/valid_with_api_key_env.yaml delete mode 100644 internal/prompt/go_renderer.go delete mode 100644 internal/prompt/renderer.go delete mode 100644 internal/prompt/renderer_test.go delete mode 100644 internal/promptdef/filesystem_repository.go delete mode 100644 internal/promptdef/repository.go delete mode 100644 internal/promptdef/repository_test.go delete mode 100644 internal/promptdef/testdata/both_content_and_content_file.yaml delete mode 100644 internal/promptdef/testdata/duplicate_input_names.yaml delete mode 100644 internal/promptdef/testdata/empty_cache_control_type.yaml delete mode 100644 internal/promptdef/testdata/invalid_validation_mode.yaml delete mode 100644 internal/promptdef/testdata/invalid_yaml.yaml delete mode 100644 internal/promptdef/testdata/json_schema_without_schema_path.yaml delete mode 100644 internal/promptdef/testdata/messages/user_prompt.tmpl delete mode 100644 internal/promptdef/testdata/missing_content_file.yaml delete mode 100644 internal/promptdef/testdata/missing_id.yaml delete mode 100644 internal/promptdef/testdata/neither_content_nor_content_file.yaml delete mode 100644 internal/promptdef/testdata/no_messages.yaml delete mode 100644 internal/promptdef/testdata/unknown_cache_control_field.yaml delete mode 100644 internal/promptdef/testdata/unknown_input_field.yaml delete mode 100644 internal/promptdef/testdata/unsupported_cache_control_ttl.yaml delete mode 100644 internal/promptdef/testdata/unsupported_cache_control_type.yaml delete mode 100644 internal/promptdef/testdata/valid_cache_control_ttl.yaml delete mode 100644 internal/promptdef/testdata/valid_cache_control_without_ttl.yaml delete mode 100644 internal/promptdef/testdata/valid_file_backed.yaml delete mode 100644 internal/promptdef/testdata/valid_inline.yaml delete mode 100644 internal/promptdef/testdata/valid_session_id.yaml delete mode 100644 internal/promptdef/testdata/with_default_profile.yaml delete mode 100644 internal/usecase/repairer.go delete mode 100644 internal/usecase/runner.go delete mode 100644 internal/usecase/runner_test.go delete mode 100644 internal/validate/standard_validator.go delete mode 100644 internal/validate/standard_validator_test.go delete mode 100644 internal/validate/validator.go delete mode 100644 json_copy.go delete mode 100644 llm_adapter.go delete mode 100644 profiles.go delete mode 100644 testdata/framework/fixtures/glossary.yml delete mode 100644 testdata/framework/fixtures/transcript.md delete mode 100644 testdata/framework/profiles/contract-fast.yaml delete mode 100644 testdata/framework/profiles/contract-quality.yaml delete mode 100644 testdata/framework/prompts/contract.markdown_summary.system.md delete mode 100644 testdata/framework/prompts/contract.markdown_summary.user.md delete mode 100644 testdata/framework/prompts/contract.markdown_summary.yaml delete mode 100644 testdata/framework/prompts/contract.structured_events.system.md delete mode 100644 testdata/framework/prompts/contract.structured_events.user.md delete mode 100644 testdata/framework/prompts/contract.structured_events.yaml delete mode 100644 testdata/framework/schemas/structured_events.schema.json delete mode 100644 types.go diff --git a/artifact_reader.go b/artifact_reader.go deleted file mode 100644 index 74c9e9f..0000000 --- a/artifact_reader.go +++ /dev/null @@ -1,39 +0,0 @@ -package scriptorium - -import ( - "context" - "errors" - - artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact" - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -var errNilArtifactReaderResponse = errors.New("artifact reader returned nil artifact without error") - -type publicArtifactReaderAdapter struct { - reader ArtifactReader -} - -var _ artifactadapter.Reader = publicArtifactReaderAdapter{} - -func (a publicArtifactReaderAdapter) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { - artifact, err := a.reader.Read(ctx, ArtifactRef{ - Type: ArtifactRefType(ref.Type), - URI: ref.URI, - Body: ref.Body, - }) - if err != nil { - return nil, err - } - if artifact == nil { - return nil, errNilArtifactReaderResponse - } - return &domain.Artifact{ - Name: artifact.Name, - ContentType: artifact.ContentType, - Body: copyBytes(artifact.Body), - URI: artifact.URI, - Size: artifact.Size, - Hash: artifact.Hash, - }, nil -} diff --git a/artifact_reader_internal_test.go b/artifact_reader_internal_test.go deleted file mode 100644 index 1709e87..0000000 --- a/artifact_reader_internal_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package scriptorium - -import ( - "context" - "testing" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -func TestPublicArtifactReaderAdapterCopiesBody(t *testing.T) { - reader := internalArtifactReaderFake{ - artifact: &Artifact{Body: []byte("original")}, - } - adapter := publicArtifactReaderAdapter{reader: &reader} - - artifact, err := adapter.Read(context.Background(), domain.ArtifactRef{ - Type: domain.ArtifactRefInline, - URI: "memory://input", - Body: "input", - }) - if err != nil { - t.Fatalf("read artifact: %v", err) - } - artifact.Body[0] = 'X' - if got := string(reader.artifact.Body); got != "original" { - t.Fatalf("reader artifact body was mutated: %q", got) - } -} - -type internalArtifactReaderFake struct { - artifact *Artifact -} - -func (r *internalArtifactReaderFake) Read(context.Context, ArtifactRef) (*Artifact, error) { - return r.artifact, nil -} diff --git a/convert.go b/convert.go deleted file mode 100644 index 13d3264..0000000 --- a/convert.go +++ /dev/null @@ -1,406 +0,0 @@ -package scriptorium - -import ( - "reflect" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) { - execution, err := toDomainExecutionTargetOverride(req.Execution) - if err != nil { - return domain.RunRequest{}, err - } - return domain.RunRequest{ - PromptID: req.PromptID, - PromptVersion: req.PromptVersion, - ProfileID: req.ProfileID, - APIKey: req.APIKey, - Inputs: toDomainArtifactRefMap(req.Inputs), - Vars: copyStringMap(req.Vars), - Execution: execution, - Validation: toDomainOutputContractPtr(req.Validation), - Metadata: copyStringMap(req.Metadata), - }, nil -} - -func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun { - if prepared == nil { - return nil - } - return &PreparedRun{ - PromptID: prepared.PromptID, - PromptVersion: prepared.PromptVersion, - PromptHash: prepared.PromptHash, - SelectedProfileID: prepared.SelectedProfileID, - EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams), - OutputContract: fromDomainOutputContract(prepared.OutputContract), - StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput), - InputHashes: copyStringMap(prepared.InputHashes), - SessionID: prepared.SessionID, - RenderedPromptHash: prepared.RenderedPromptHash, - Messages: fromDomainRenderedMessages(prepared.Messages), - StartTime: prepared.StartTime, - EndTime: prepared.EndTime, - DurationMS: prepared.DurationMS, - } -} - -func fromDomainRunResult(result *domain.RunResult) *RunResult { - if result == nil { - return nil - } - return &RunResult{ - RunID: result.RunID, - Artifact: fromDomainArtifact(result.Artifact), - RawOutput: result.RawOutput, - Validation: fromDomainValidationResult(result.Validation), - PromptID: result.PromptID, - PromptVersion: result.PromptVersion, - PromptHash: result.PromptHash, - RenderedPromptHash: result.RenderedPromptHash, - SelectedProfileID: result.SelectedProfileID, - ModelName: result.ModelName, - Endpoint: result.Endpoint, - EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams), - InputHashes: copyStringMap(result.InputHashes), - Usage: fromDomainTokenUsage(result.Usage), - StartTime: result.StartTime, - EndTime: result.EndTime, - Duration: result.Duration, - } -} - -func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest { - return GenerateRequest{ - Prompt: fromDomainRenderedPrompt(req.Prompt), - Target: fromDomainExecutionTarget(req.Target), - TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence), - StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput), - APIKey: req.Target.APIKey, - } -} - -func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse { - if resp == nil { - return nil - } - return &domain.GenerateResponse{ - Content: resp.Content, - Usage: toDomainTokenUsage(resp.Usage), - } -} - -func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt { - return RenderedPrompt{ - SessionID: prompt.SessionID, - Messages: fromDomainRenderedMessages(prompt.Messages), - } -} - -func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef { - if src == nil { - return nil - } - out := make(map[string]domain.ArtifactRef, len(src)) - for k, v := range src { - out[k] = toDomainArtifactRef(v) - } - return out -} - -func toDomainArtifactRef(ref ArtifactRef) domain.ArtifactRef { - return domain.ArtifactRef{ - Type: domain.ArtifactRefType(ref.Type), - URI: ref.URI, - Body: ref.Body, - } -} - -func fromDomainArtifact(artifact domain.Artifact) Artifact { - return Artifact{ - Name: artifact.Name, - ContentType: artifact.ContentType, - Body: copyBytes(artifact.Body), - URI: artifact.URI, - Size: artifact.Size, - Hash: artifact.Hash, - } -} - -func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain.ExecutionTargetOverride, error) { - if override == nil { - return nil, nil - } - extraParams, err := copyPublicJSONMap(override.ExtraParams) - if err != nil { - return nil, err - } - return &domain.ExecutionTargetOverride{ - Endpoint: override.Endpoint, - Model: override.Model, - Temperature: copyFloat64Ptr(override.Temperature), - MaxTokens: copyIntPtr(override.MaxTokens), - TopP: copyFloat64Ptr(override.TopP), - TimeoutSeconds: copyIntPtr(override.TimeoutSeconds), - ServiceTier: override.ServiceTier, - ReasoningEffort: override.ReasoningEffort, - APIKeyEnv: override.APIKeyEnv, - ExtraParams: extraParams, - }, nil -} - -func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget { - return ExecutionTarget{ - Endpoint: target.Endpoint, - Model: target.Model, - Temperature: target.Temperature, - MaxTokens: target.MaxTokens, - TopP: target.TopP, - TimeoutSeconds: target.TimeoutSeconds, - ServiceTier: target.ServiceTier, - ReasoningEffort: target.ReasoningEffort, - APIKeyEnv: target.APIKeyEnv, - ExtraParams: copyAnyMap(target.ExtraParams), - } -} - -func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence { - return ExecutionTargetPresence{ - Temperature: presence.Temperature, - MaxTokens: presence.MaxTokens, - TopP: presence.TopP, - TimeoutSeconds: presence.TimeoutSeconds, - } -} - -func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract { - if contract == nil { - return nil - } - out := toDomainOutputContract(*contract) - return &out -} - -func toDomainOutputContract(contract OutputContract) domain.OutputContract { - return domain.OutputContract{ - Format: domain.OutputFormat(contract.Format), - ValidationMode: domain.ValidationMode(contract.ValidationMode), - SchemaPath: contract.SchemaPath, - RepairAttempts: contract.RepairAttempts, - } -} - -func fromDomainOutputContract(contract domain.OutputContract) OutputContract { - return OutputContract{ - Format: OutputFormat(contract.Format), - ValidationMode: ValidationMode(contract.ValidationMode), - SchemaPath: contract.SchemaPath, - RepairAttempts: contract.RepairAttempts, - } -} - -func fromDomainValidationResult(result domain.ValidationResult) ValidationResult { - return ValidationResult{ - Status: ValidationStatus(result.Status), - Mode: ValidationMode(result.Mode), - Errors: copyStringSlice(result.Errors), - SchemaPath: result.SchemaPath, - RepairAttempts: result.RepairAttempts, - IsValid: result.IsValid, - } -} - -func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage { - return TokenUsage{ - PromptTokens: usage.PromptTokens, - CompletionTokens: usage.CompletionTokens, - TotalTokens: usage.TotalTokens, - CachedTokens: usage.CachedTokens, - CacheWriteTokens: usage.CacheWriteTokens, - } -} - -func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage { - return domain.TokenUsage{ - PromptTokens: usage.PromptTokens, - CompletionTokens: usage.CompletionTokens, - TotalTokens: usage.TotalTokens, - CachedTokens: usage.CachedTokens, - CacheWriteTokens: usage.CacheWriteTokens, - } -} - -func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage { - if messages == nil { - return nil - } - out := make([]RenderedMessage, len(messages)) - for i, msg := range messages { - out[i] = RenderedMessage{ - Role: msg.Role, - Content: msg.Content, - CacheControl: fromDomainCacheControl(msg.CacheControl), - } - } - return out -} - -func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl { - if cacheControl == nil { - return nil - } - return &CacheControl{ - Type: CacheControlType(cacheControl.Type), - TTL: cacheControl.TTL, - } -} - -func fromDomainStructuredOutputSpec(spec *domain.StructuredOutputSpec) *StructuredOutputSpec { - if spec == nil { - return nil - } - out := &StructuredOutputSpec{ - Type: StructuredOutputType(spec.Type), - } - if spec.JSONSchema != nil { - out.JSONSchema = &StructuredOutputJSONSpec{ - Name: spec.JSONSchema.Name, - Strict: spec.JSONSchema.Strict, - Schema: copyAny(spec.JSONSchema.Schema), - } - } - return out -} - -func copyStringMap(src map[string]string) map[string]string { - if src == nil { - return nil - } - out := make(map[string]string, len(src)) - for k, v := range src { - out[k] = v - } - return out -} - -func copyAnyMap(src map[string]any) map[string]any { - if src == nil { - return nil - } - out := make(map[string]any, len(src)) - for k, v := range src { - out[k] = copyAny(v) - } - return out -} - -func copyAny(value any) any { - if value == nil { - return nil - } - switch v := value.(type) { - case map[string]any: - return copyAnyMap(v) - case []any: - out := make([]any, len(v)) - for i, item := range v { - out[i] = copyAny(item) - } - return out - case []string: - return copyStringSlice(v) - case []byte: - return copyBytes(v) - default: - return copyReflectValue(reflect.ValueOf(value)).Interface() - } -} - -func copyReflectValue(value reflect.Value) reflect.Value { - if !value.IsValid() { - return value - } - - switch value.Kind() { - case reflect.Interface: - if value.IsNil() { - return reflect.Zero(value.Type()) - } - copied := copyReflectValue(value.Elem()) - if copied.IsValid() && copied.Type().AssignableTo(value.Type()) { - return copied - } - out := reflect.New(value.Type()).Elem() - out.Set(copied) - return out - case reflect.Pointer: - if value.IsNil() { - return reflect.Zero(value.Type()) - } - out := reflect.New(value.Type().Elem()) - out.Elem().Set(copyReflectValue(value.Elem())) - return out - case reflect.Map: - if value.IsNil() { - return reflect.Zero(value.Type()) - } - out := reflect.MakeMapWithSize(value.Type(), value.Len()) - iter := value.MapRange() - for iter.Next() { - out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value())) - } - return out - case reflect.Slice: - if value.IsNil() { - return reflect.Zero(value.Type()) - } - out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap()) - for i := 0; i < value.Len(); i++ { - out.Index(i).Set(copyReflectValue(value.Index(i))) - } - return out - case reflect.Array: - out := reflect.New(value.Type()).Elem() - for i := 0; i < value.Len(); i++ { - out.Index(i).Set(copyReflectValue(value.Index(i))) - } - return out - default: - return value - } -} - -func copyStringSlice(src []string) []string { - if src == nil { - return nil - } - out := make([]string, len(src)) - copy(out, src) - return out -} - -func copyBytes(src []byte) []byte { - if src == nil { - return nil - } - out := make([]byte, len(src)) - copy(out, src) - return out -} - -func copyFloat64Ptr(src *float64) *float64 { - if src == nil { - return nil - } - v := *src - return &v -} - -func copyIntPtr(src *int) *int { - if src == nil { - return nil - } - v := *src - return &v -} diff --git a/engine.go b/engine.go deleted file mode 100644 index 4fcc5c7..0000000 --- a/engine.go +++ /dev/null @@ -1,343 +0,0 @@ -package scriptorium - -import ( - "context" - "errors" - "fmt" - "io/fs" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact" - "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" - "gitea.maximumdirect.net/eric/scriptorium/internal/llm" - "gitea.maximumdirect.net/eric/scriptorium/internal/profile" - "gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin" - "gitea.maximumdirect.net/eric/scriptorium/internal/prompt" - "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" - "gitea.maximumdirect.net/eric/scriptorium/internal/usecase" - "gitea.maximumdirect.net/eric/scriptorium/internal/validate" -) - -// ErrInvalidConfig indicates invalid public engine configuration. -var ErrInvalidConfig = errors.New("invalid engine configuration") - -var ( - ErrInvalidRequest = errors.New("invalid run request") - ErrPromptNotFound = errors.New("prompt not found") - ErrProfileNotFound = errors.New("profile not found") - ErrProfileRequired = errors.New("profile selection is required") - ErrPromptLoad = errors.New("failed to load prompt definition") - ErrProfileLoad = errors.New("failed to load execution profile") - ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable") - ErrArtifactLoad = errors.New("failed to load artifact") - ErrPromptRender = errors.New("failed to render prompt") - ErrLLMGenerate = errors.New("failed to generate output") - ErrValidation = errors.New("failed to validate output") -) - -// Engine prepares and runs Scriptorium prompt requests. -type Engine struct { - runner *usecase.Runner -} - -// Config configures a public Scriptorium engine. -type Config struct { - PromptDir string - ProfileDir string - SchemaDir string - // Timeout is the transport-wide safety cap for the built-in LLM client - // when HTTPClient is absent or has a non-positive timeout. - Timeout time.Duration - // HTTPClient is cloned for the built-in LLM client. Its positive Timeout - // takes precedence over Config.Timeout as the transport-wide safety cap. - HTTPClient *http.Client -} - -// Option customizes engine construction. -type Option interface { - apply(*engineOptions) error -} - -type optionFunc func(*engineOptions) error - -func (f optionFunc) apply(options *engineOptions) error { - return f(options) -} - -type engineOptions struct { - llmClient llm.Client - artifactReader artifactadapter.Reader - promptDefs promptdef.Repository - profiles profile.Repository - memoryProfiles profile.Repository - validator validate.Validator - promptSource bool - profileSource bool - memorySource bool - validatorSource bool - artifactSource bool -} - -// WithLLMClient injects a custom LLM client for execution. -func WithLLMClient(client LLMClient) Option { - return optionFunc(func(options *engineOptions) error { - if client == nil { - return ErrInvalidConfig - } - options.llmClient = publicLLMClientAdapter{client: client} - return nil - }) -} - -// WithArtifactReader injects a reader for every input artifact reference. -func WithArtifactReader(reader ArtifactReader) Option { - return optionFunc(func(options *engineOptions) error { - if reader == nil { - return ErrInvalidConfig - } - options.artifactReader = publicArtifactReaderAdapter{reader: reader} - options.artifactSource = true - return nil - }) -} - -// WithPromptFS loads prompt definitions from fsys under root. -// -// The source uses the same strict prompt YAML rules as configured prompt -// directories, and prompt content_file paths resolve within this source. -func WithPromptFS(fsys fs.FS, root string) Option { - return optionFunc(func(options *engineOptions) error { - if fsys == nil { - return ErrInvalidConfig - } - if strings.TrimSpace(root) == "" { - return ErrInvalidConfig - } - options.promptDefs = promptdef.NewFSRepository(fsys, root) - options.promptSource = true - return nil - }) -} - -// WithPromptFile loads prompt definitions from the single prompt file at path. -// -// Relative prompt content_file paths resolve from the file's directory. -func WithPromptFile(path string) Option { - return optionFunc(func(options *engineOptions) error { - fsys, root, err := fileSource(path) - if err != nil { - return err - } - options.promptDefs = promptdef.NewFSRepository(fsys, root) - options.promptSource = true - return nil - }) -} - -// WithProfileFS loads execution profiles from fsys under root. -// -// Profiles from this source overlay built-in profiles. Profile YAML must use -// api_key_env for environment-based credentials; raw API keys are rejected. -func WithProfileFS(fsys fs.FS, root string) Option { - return optionFunc(func(options *engineOptions) error { - if fsys == nil { - return ErrInvalidConfig - } - if strings.TrimSpace(root) == "" { - return ErrInvalidConfig - } - options.profiles = profile.NewFSRepository(fsys, root) - options.profileSource = true - return nil - }) -} - -// WithProfileFile loads execution profiles from the single profile file at path. -// -// The profile overlays built-in profiles. Profile YAML must use api_key_env for -// environment-based credentials; raw API keys are rejected. -func WithProfileFile(path string) Option { - return optionFunc(func(options *engineOptions) error { - fsys, root, err := fileSource(path) - if err != nil { - return err - } - options.profiles = profile.NewFSRepository(fsys, root) - options.profileSource = true - return nil - }) -} - -// WithProfiles configures in-memory profiles that take precedence over -// configured profile files and built-in profiles. -func WithProfiles(profiles ...Profile) Option { - return optionFunc(func(options *engineOptions) error { - repo, err := newMemoryProfileRepository(profiles) - if err != nil { - return err - } - options.memoryProfiles = repo - options.memorySource = true - return nil - }) -} - -// WithSchemaFS loads JSON Schema documents from fsys under root. -// -// Prompt schema_path values resolve within this source when schema validation -// or structured output is requested. -func WithSchemaFS(fsys fs.FS, root string) Option { - return optionFunc(func(options *engineOptions) error { - if fsys == nil { - return ErrInvalidConfig - } - if strings.TrimSpace(root) == "" { - return ErrInvalidConfig - } - options.validator = validate.NewFSValidator(fsys, root) - options.validatorSource = true - return nil - }) -} - -// WithSchemaFile loads JSON Schema documents from the single schema file at path. -// -// Prompt schema_path values refer to the file's base name. -func WithSchemaFile(path string) Option { - return optionFunc(func(options *engineOptions) error { - fsys, root, err := fileSource(path) - if err != nil { - return err - } - options.validator = validate.NewFSValidator(fsys, root) - options.validatorSource = true - return nil - }) -} - -// NewEngine constructs an Engine using the same default internal components as -// the CLI and HTTP adapters. -func NewEngine(cfg Config, opts ...Option) (*Engine, error) { - var options engineOptions - for _, opt := range opts { - if opt == nil { - continue - } - if err := opt.apply(&options); err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) - } - } - - promptDefs := options.promptDefs - if !options.promptSource { - if strings.TrimSpace(cfg.PromptDir) == "" { - return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig) - } - promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir) - } - - profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir) - if options.profileSource { - profiles = builtin.NewRepositoryWithPrimary(options.profiles) - } - if options.memorySource { - profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles) - } - - validator := options.validator - if !options.validatorSource { - schemaDir := cfg.SchemaDir - if strings.TrimSpace(schemaDir) == "" { - schemaDir = defaults.SchemaDirDefault - } - validator = validate.NewStandardValidator(schemaDir) - } - - llmClient := options.llmClient - if llmClient == nil { - var err error - llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ - Timeout: cfg.Timeout, - HTTPClient: cfg.HTTPClient, - }) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) - } - } - - artifacts := options.artifactReader - if !options.artifactSource { - artifacts = artifactadapter.NewCompositeReader() - } - - return &Engine{ - runner: usecase.NewRunner( - promptDefs, - profiles, - artifacts, - prompt.NewGoRenderer(), - llmClient, - validator, - ), - }, nil -} - -func fileSource(name string) (fs.FS, string, error) { - cleanName := strings.TrimSpace(name) - if cleanName == "" { - return nil, "", ErrInvalidConfig - } - dir := filepath.Dir(cleanName) - base := filepath.Base(cleanName) - if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" { - return nil, "", ErrInvalidConfig - } - info, err := os.Stat(cleanName) - if err != nil { - return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err) - } - if info.IsDir() { - return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName) - } - return os.DirFS(dir), filepath.ToSlash(base), nil -} - -// Prepare resolves a prompt request without calling an LLM. -func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) { - if e == nil || e.runner == nil { - return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) - } - - domainReq, err := toDomainRunRequest(req) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) - } - - prepared, err := e.runner.Prepare(ctx, domainReq) - if err != nil { - return nil, mapPublicError(err) - } - return fromDomainPreparedRun(prepared), nil -} - -// Run executes a prompt request and returns the generated artifact and metadata. -func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { - if e == nil || e.runner == nil { - return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) - } - - domainReq, err := toDomainRunRequest(req) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) - } - - result, err := e.runner.Run(ctx, domainReq) - if err != nil { - return nil, mapPublicError(err) - } - return fromDomainRunResult(result), nil -} diff --git a/engine_test.go b/engine_test.go deleted file mode 100644 index f2719ec..0000000 --- a/engine_test.go +++ /dev/null @@ -1,2559 +0,0 @@ -package scriptorium_test - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "math" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "reflect" - "strings" - "testing" - "testing/fstest" - "time" - - "gitea.maximumdirect.net/eric/scriptorium" -) - -const ( - frameworkContractRoot = "./testdata/framework" - frameworkPromptDir = frameworkContractRoot + "/prompts" - frameworkProfileDir = frameworkContractRoot + "/profiles" - frameworkSchemaDir = frameworkContractRoot + "/schemas" - - frameworkMarkdownSummaryPromptID = "contract.markdown_summary" - frameworkStructuredEventsPromptID = "contract.structured_events" - frameworkFastProfileID = "contract-fast" - frameworkQualityProfileID = "contract-quality" - - frameworkTranscriptPath = frameworkContractRoot + "/fixtures/transcript.md" - frameworkGlossaryPath = frameworkContractRoot + "/fixtures/glossary.yml" -) - -func TestNewEngineRejectsMissingPromptDir(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: frameworkProfileDir}) - if !errors.Is(err, scriptorium.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestNewEngineAcceptsMissingProfileDir(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}) - if err != nil { - t.Fatalf("expected missing profile dir to use built-ins, got %v", err) - } -} - -func TestPrepareWorksWithFrameworkContractCorpus(t *testing.T) { - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("construct engine from framework contract corpus: %v", err) - } - - tests := []struct { - name string - promptID string - profileID string - model string - structured bool - }{ - { - name: "markdown summary", - promptID: frameworkMarkdownSummaryPromptID, - profileID: frameworkFastProfileID, - model: "contract-fast-model", - }, - { - name: "structured events", - promptID: frameworkStructuredEventsPromptID, - profileID: frameworkQualityProfileID, - model: "contract-quality-model", - structured: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: tt.promptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.File(frameworkTranscriptPath), - "glossary": scriptorium.File(frameworkGlossaryPath), - }, - }) - if err != nil { - t.Fatalf("prepare framework contract prompt: %v", err) - } - if prepared.PromptID != tt.promptID { - t.Fatalf("unexpected prompt id: got %q, want %q", prepared.PromptID, tt.promptID) - } - if prepared.SelectedProfileID != tt.profileID { - t.Fatalf("unexpected selected profile: got %q, want %q", prepared.SelectedProfileID, tt.profileID) - } - if prepared.EffectiveModelParams.Model != tt.model { - t.Fatalf("unexpected effective model: got %q, want %q", prepared.EffectiveModelParams.Model, tt.model) - } - if len(prepared.Messages) != 2 { - t.Fatalf("expected rendered messages, got %d", len(prepared.Messages)) - } - if !strings.Contains(prepared.Messages[1].Content, "Nia labels the archive.") { - t.Fatalf("expected relative prompt content to render the transcript, got %q", prepared.Messages[1].Content) - } - if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" { - t.Fatalf("expected input hashes, got %#v", prepared.InputHashes) - } - - if !tt.structured { - if prepared.StructuredOutput != nil { - t.Fatalf("expected no structured output specification, got %#v", prepared.StructuredOutput) - } - return - } - - if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil { - t.Fatalf("expected loaded JSON Schema structured output, got %#v", prepared.StructuredOutput) - } - schema, ok := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any) - if !ok || schema["type"] != "object" { - t.Fatalf("expected loaded object JSON Schema, got %#v", prepared.StructuredOutput.JSONSchema.Schema) - } - properties, ok := schema["properties"].(map[string]any) - if !ok || properties["events"] == nil { - t.Fatalf("expected loaded events schema property, got %#v", schema) - } - }) - } -} - -func TestPrepareWorksWithInlineInputs(t *testing.T) { - engine := newContractEngine(t) - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin scouts the tower.\nKara lights a lantern."), - "glossary": scriptorium.InlineWithURI("memory://glossary.yml", "party:\n - Rin\n - Kara\n"), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if len(prepared.Messages) != 2 { - t.Fatalf("expected rendered messages, got %d", len(prepared.Messages)) - } - rendered := prepared.Messages[1].Content - if !strings.Contains(rendered, "Rin scouts the tower.") || !strings.Contains(rendered, "party:") { - t.Fatalf("expected inline inputs in rendered prompt, got %q", rendered) - } -} - -func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) { - const envName = "SCRIPTORIUM_API_KEY" - const secret = "public-api-test-secret" - t.Setenv(envName, secret) - - profileDir := t.TempDir() - writePublicProfileFileWithAPIKeyEnv(t, profileDir, "prepared-secret", "http://localhost:8000/v1", "prepared-secret-model", envName) - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkStructuredEventsPromptID, - ProfileID: "prepared-secret", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.File(frameworkTranscriptPath), - "glossary": scriptorium.File(frameworkGlossaryPath), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - - payload, err := json.Marshal(prepared) - if err != nil { - t.Fatalf("expected prepared run to marshal, got %v", err) - } - out := string(payload) - if strings.Contains(out, secret) { - t.Fatalf("prepared run JSON leaked raw API key value: %s", out) - } - if !strings.Contains(out, envName) { - t.Fatalf("prepared run JSON should retain api_key_env name, got %s", out) - } - for _, forbidden := range []string{"TargetPresence", "target_presence"} { - if strings.Contains(out, forbidden) { - t.Fatalf("prepared run JSON exposed internal target presence metadata %q: %s", forbidden, out) - } - } -} - -func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) { - const secret = "run-request-secret" - req := scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: frameworkFastProfileID, - APIKey: secret, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - }, - } - - for _, formatted := range []string{ - fmt.Sprint(req), - fmt.Sprintf("%+v", req), - fmt.Sprintf("%#v", req), - } { - if strings.Contains(formatted, secret) { - t.Fatalf("formatted RunRequest leaked API key: %s", formatted) - } - if !strings.Contains(formatted, "APIKeySet:true") { - t.Fatalf("formatted RunRequest should indicate an API key is set, got %s", formatted) - } - } - - payload, err := json.Marshal(req) - if err != nil { - t.Fatalf("expected RunRequest to marshal, got %v", err) - } - if strings.Contains(string(payload), secret) { - t.Fatalf("RunRequest JSON leaked API key: %s", payload) - } -} - -func TestGenerateRequestFormattingRedactsDirectAPIKey(t *testing.T) { - const secret = "generate-request-secret" - req := scriptorium.GenerateRequest{ - Prompt: scriptorium.RenderedPrompt{Messages: []scriptorium.RenderedMessage{ - {Role: "user", Content: "secret prompt content"}, - }}, - Target: scriptorium.ExecutionTarget{ - Model: "test-model", - ExtraParams: map[string]any{ - "provider_option": "on", - }, - }, - APIKey: secret, - } - - for _, formatted := range []string{ - fmt.Sprint(req), - fmt.Sprintf("%+v", req), - fmt.Sprintf("%#v", req), - } { - if strings.Contains(formatted, secret) { - t.Fatalf("formatted GenerateRequest leaked API key: %s", formatted) - } - if strings.Contains(formatted, "secret prompt content") { - t.Fatalf("formatted GenerateRequest leaked prompt content: %s", formatted) - } - if !strings.Contains(formatted, "APIKeySet:true") { - t.Fatalf("formatted GenerateRequest should indicate an API key is set, got %s", formatted) - } - } - - payload, err := json.Marshal(req) - if err != nil { - t.Fatalf("expected GenerateRequest to marshal, got %v", err) - } - if strings.Contains(string(payload), secret) { - t.Fatalf("GenerateRequest JSON leaked API key: %s", payload) - } -} - -func TestEngineExecutionSettingPrecedence(t *testing.T) { - floatPointer := func(value float64) *float64 { - return &value - } - intPointer := func(value int) *int { - return &value - } - - defaultsProfile := executionProfileFixture{ - id: "settings-defaults", - endpoint: "http://profile-defaults.test/v1", - model: "profile-defaults-model", - serviceTier: "profile-defaults-tier", - reasoningEffort: "profile-defaults-reasoning", - apiKeyEnv: "SCRIPTORIUM_PRECEDENCE_DEFAULTS", - extraParamSource: "profile-defaults", - } - profileSettings := executionProfileFixture{ - id: "settings-profile", - endpoint: "http://profile-settings.test/v1", - model: "profile-settings-model", - temperature: 0.31, - maxTokens: 311, - topP: 0.61, - timeoutSeconds: 71, - serviceTier: "profile-settings-tier", - reasoningEffort: "profile-settings-reasoning", - apiKeyEnv: "SCRIPTORIUM_PRECEDENCE_PROFILE", - extraParamSource: "profile-settings", - } - requestProfile := executionProfileFixture{ - id: "settings-request", - endpoint: "http://profile-request.test/v1", - model: "profile-request-model", - temperature: 0.29, - maxTokens: 299, - topP: 0.59, - timeoutSeconds: 79, - serviceTier: "profile-request-tier", - reasoningEffort: "profile-request-reasoning", - apiKeyEnv: "SCRIPTORIUM_PRECEDENCE_REQUEST_PROFILE", - extraParamSource: "profile-request", - } - zeroOverrideProfile := executionProfileFixture{ - id: "settings-zero", - endpoint: "http://profile-zero.test/v1", - model: "profile-zero-model", - temperature: 0.43, - maxTokens: 433, - topP: 0.73, - timeoutSeconds: 83, - serviceTier: "profile-zero-tier", - reasoningEffort: "profile-zero-reasoning", - apiKeyEnv: "SCRIPTORIUM_PRECEDENCE_ZERO", - extraParamSource: "profile-zero", - } - - requestTarget := scriptorium.ExecutionTarget{ - Endpoint: "http://request-settings.test/v1", - Model: "request-settings-model", - Temperature: 0.87, - MaxTokens: 877, - TopP: 0.97, - TimeoutSeconds: 177, - ServiceTier: "request-settings-tier", - ReasoningEffort: "request-settings-reasoning", - APIKeyEnv: "SCRIPTORIUM_PRECEDENCE_REQUEST", - ExtraParams: map[string]any{"source": "request-settings"}, - } - zeroOverrideTarget := executionTargetFromProfileFixture(zeroOverrideProfile) - zeroOverrideTarget.Temperature = 0 - zeroOverrideTarget.MaxTokens = 0 - zeroOverrideTarget.TopP = 0 - zeroOverrideTarget.TimeoutSeconds = 0 - - tests := []struct { - name string - profile executionProfileFixture - override *scriptorium.ExecutionTargetOverride - want scriptorium.ExecutionTarget - wantPresence scriptorium.ExecutionTargetPresence - }{ - { - name: "framework defaults fill zero-valued profile settings", - profile: defaultsProfile, - want: scriptorium.ExecutionTarget{ - Endpoint: defaultsProfile.endpoint, - Model: defaultsProfile.model, - Temperature: 0, - MaxTokens: 0, - TopP: 1, - TimeoutSeconds: 600, - ServiceTier: defaultsProfile.serviceTier, - ReasoningEffort: defaultsProfile.reasoningEffort, - APIKeyEnv: defaultsProfile.apiKeyEnv, - ExtraParams: map[string]any{"source": defaultsProfile.extraParamSource}, - }, - }, - { - name: "profile settings replace framework defaults", - profile: profileSettings, - want: executionTargetFromProfileFixture(profileSettings), - }, - { - name: "request settings replace profile settings", - profile: requestProfile, - override: &scriptorium.ExecutionTargetOverride{ - Endpoint: requestTarget.Endpoint, - Model: requestTarget.Model, - Temperature: floatPointer(requestTarget.Temperature), - MaxTokens: intPointer(requestTarget.MaxTokens), - TopP: floatPointer(requestTarget.TopP), - TimeoutSeconds: intPointer(requestTarget.TimeoutSeconds), - ServiceTier: requestTarget.ServiceTier, - ReasoningEffort: requestTarget.ReasoningEffort, - APIKeyEnv: requestTarget.APIKeyEnv, - ExtraParams: requestTarget.ExtraParams, - }, - want: requestTarget, - wantPresence: scriptorium.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}, - }, - { - name: "explicit request zero replaces profile settings", - profile: zeroOverrideProfile, - override: &scriptorium.ExecutionTargetOverride{ - Temperature: floatPointer(0), - MaxTokens: intPointer(0), - TopP: floatPointer(0), - TimeoutSeconds: intPointer(0), - }, - want: zeroOverrideTarget, - wantPresence: scriptorium.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Setenv(tt.profile.apiKeyEnv, "set") - if tt.override != nil && tt.override.APIKeyEnv != "" { - t.Setenv(tt.override.APIKeyEnv, "set") - } - - profileDir := t.TempDir() - writeExecutionProfileFixture(t, profileDir, tt.profile) - fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithLLMClient(fake)) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - _, err = engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: tt.profile.id, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Nia labels the archive."), - "glossary": scriptorium.Inline("archive: A catalogued collection."), - }, - Execution: tt.override, - }) - if err != nil { - t.Fatalf("run engine: %v", err) - } - if len(fake.requests) != 1 { - t.Fatalf("expected one generation request, got %d", len(fake.requests)) - } - got := fake.requests[0] - if !reflect.DeepEqual(got.Target, tt.want) { - t.Fatalf("unexpected effective target:\ngot=%#v\nwant=%#v", got.Target, tt.want) - } - if got.TargetPresence != tt.wantPresence { - t.Fatalf("unexpected target presence: got=%+v want=%+v", got.TargetPresence, tt.wantPresence) - } - }) - } -} - -func TestRunSucceedsWithInjectedLLMClient(t *testing.T) { - const envName = "SCRIPTORIUM_API_KEY" - const secret = "run-secret-value" - t.Setenv(envName, secret) - - fake := &fakeLLMClient{ - response: &scriptorium.GenerateResponse{ - Content: "# Summary\n\nDone.", - Usage: scriptorium.TokenUsage{ - PromptTokens: 10, - CompletionTokens: 5, - TotalTokens: 15, - CachedTokens: 3, - CacheWriteTokens: 2, - }, - }, - } - engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) - - result, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - Execution: &scriptorium.ExecutionTargetOverride{ - APIKeyEnv: envName, - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if result.RunID == "" { - t.Fatalf("expected run id") - } - if result.RawOutput != fake.response.Content { - t.Fatalf("unexpected raw output: %q", result.RawOutput) - } - if string(result.Artifact.Body) != fake.response.Content { - t.Fatalf("unexpected artifact body: %q", string(result.Artifact.Body)) - } - if result.Artifact.ContentType != "text/markdown" { - t.Fatalf("unexpected artifact content type: %q", result.Artifact.ContentType) - } - if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid { - t.Fatalf("expected passed validation, got %+v", result.Validation) - } - if result.PromptID != frameworkMarkdownSummaryPromptID || result.SelectedProfileID != frameworkFastProfileID || result.ModelName != "contract-fast-model" { - t.Fatalf("unexpected run metadata: %+v", result) - } - if result.Usage.TotalTokens != 15 || result.Usage.CachedTokens != 3 || result.Usage.CacheWriteTokens != 2 { - t.Fatalf("unexpected usage: %+v", result.Usage) - } - - payload, err := json.Marshal(result) - if err != nil { - t.Fatalf("expected run result to marshal, got %v", err) - } - if strings.Contains(string(payload), secret) { - t.Fatalf("run result JSON leaked raw API key value: %s", payload) - } -} - -func TestEngineRunWithDirectorySourcesAndFileInputs(t *testing.T) { - fake := &fakeLLMClient{ - response: &scriptorium.GenerateResponse{ - Content: `{"events":[{"title":"Archive labelled"}]}`, - Usage: scriptorium.TokenUsage{ - PromptTokens: 42, - CompletionTokens: 36, - TotalTokens: 78, - }, - }, - } - engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) - - result, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkStructuredEventsPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.File(frameworkTranscriptPath), - "glossary": scriptorium.File(frameworkGlossaryPath), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if result.PromptID != frameworkStructuredEventsPromptID || result.SelectedProfileID != frameworkQualityProfileID { - t.Fatalf("unexpected run metadata: %+v", result) - } - if result.RunID == "" || result.PromptHash == "" || result.RenderedPromptHash == "" { - t.Fatalf("expected run and prompt hashes, got %+v", result) - } - if result.InputHashes["transcript"] == "" || result.InputHashes["glossary"] == "" { - t.Fatalf("expected both input hashes, got %#v", result.InputHashes) - } - if len(fake.requests) != 1 || fake.requests[0].StructuredOutput == nil || - fake.requests[0].StructuredOutput.Type != scriptorium.StructuredOutputJSONSchema || - fake.requests[0].StructuredOutput.JSONSchema == nil || - fake.requests[0].StructuredOutput.JSONSchema.Schema == nil { - t.Fatalf("expected provider JSON Schema structured output, got %+v", fake.requests) - } - if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid || result.Validation.Mode != scriptorium.ValidationJSONSchema { - t.Fatalf("expected passed JSON Schema validation, got %+v", result.Validation) - } - if result.Artifact.ContentType != "application/json" { - t.Fatalf("expected JSON artifact, got %q", result.Artifact.ContentType) - } - if result.RawOutput != fake.response.Content || result.Usage != fake.response.Usage { - t.Fatalf("expected preserved output and usage, got output=%q usage=%+v", result.RawOutput, result.Usage) - } - if result.StartTime.IsZero() || result.EndTime.IsZero() || result.EndTime.Before(result.StartTime) || result.Duration < 0 { - t.Fatalf("expected ordered non-zero timestamps and non-negative duration, got start=%v end=%v duration=%v", result.StartTime, result.EndTime, result.Duration) - } -} - -func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) { - const directKey = "direct-injected-key" - fake := &fakeLLMClient{ - response: &scriptorium.GenerateResponse{Content: `{"events":[{"title":"Archive labelled"}]}`}, - } - engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) - - _, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkStructuredEventsPromptID, - APIKey: directKey, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if len(fake.requests) != 1 { - t.Fatalf("expected one generate request, got %d", len(fake.requests)) - } - req := fake.requests[0] - if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") { - t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt) - } - if req.StructuredOutput == nil || req.StructuredOutput.Type != scriptorium.StructuredOutputJSONSchema || req.StructuredOutput.JSONSchema == nil { - t.Fatalf("expected structured output handoff, got %+v", req.StructuredOutput) - } - if req.APIKey != directKey { - t.Fatalf("expected direct key on injected generate request") - } - payload, err := json.Marshal(req) - if err != nil { - t.Fatalf("expected generate request to marshal, got %v", err) - } - if strings.Contains(string(payload), directKey) { - t.Fatalf("generate request JSON leaked direct API key: %s", payload) - } -} - -func TestEngineRunPropagatesCallerCancellation(t *testing.T) { - const synchronizationTimeout = 5 * time.Second - - started := make(chan struct{}) - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - close(started) - <-req.Context().Done() - return nil, req.Context().Err() - }) - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - HTTPClient: &http.Client{Transport: transport}, - }) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - result := make(chan error, 1) - go func() { - _, err := engine.Run(ctx, scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Nia labels the archive."), - "glossary": scriptorium.Inline("archive: A catalogued collection."), - }, - }) - result <- err - }() - - watchdog := time.NewTimer(synchronizationTimeout) - defer watchdog.Stop() - select { - case <-started: - case err := <-result: - t.Fatalf("Engine.Run returned before the transport started: %v", err) - case <-watchdog.C: - t.Fatal("timed out waiting for the transport to start") - } - - cancel() - select { - case err := <-result: - if !errors.Is(err, scriptorium.ErrLLMGenerate) { - t.Fatalf("expected ErrLLMGenerate after caller cancellation, got %v", err) - } - case <-watchdog.C: - t.Fatal("timed out waiting for Engine.Run to return after cancellation") - } -} - -func TestRunRejectsReservedExtraParamsBeforeProviderCall(t *testing.T) { - called := false - transport := roundTripFunc(func(*http.Request) (*http.Response, error) { - called = true - return nil, errors.New("provider should not be called") - }) - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - HTTPClient: &http.Client{Transport: transport}, - }) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - _, err = engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Nia labels the archive."), - "glossary": scriptorium.Inline("archive: A catalogued collection."), - }, - Execution: &scriptorium.ExecutionTargetOverride{ - ExtraParams: map[string]any{"model": "collision"}, - }, - }) - if !errors.Is(err, scriptorium.ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if called { - t.Fatal("expected reserved provider parameter to fail before the provider call") - } -} - -func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) { - const directKey = "direct-public-key" - const missingEnv = "SCRIPTORIUM_PUBLIC_DIRECT_MISSING" - t.Setenv(missingEnv, "") - - var gotAuth string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotAuth = r.Header.Get("Authorization") - if r.URL.Path != "/v1/chat/completions" { - t.Errorf("unexpected path: %s", r.URL.Path) - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}], - "usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7} -}`)) - })) - defer server.Close() - - profileDir := t.TempDir() - writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-auth", server.URL+"/v1", "test-model", missingEnv) - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - result, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "direct-auth", - APIKey: directKey, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run with direct API key to succeed, got %v", err) - } - if gotAuth != "Bearer "+directKey { - t.Fatalf("unexpected Authorization header: %q", gotAuth) - } - if result.Usage.TotalTokens != 7 { - t.Fatalf("unexpected usage: %+v", result.Usage) - } - - payload, err := json.Marshal(result) - if err != nil { - t.Fatalf("expected run result to marshal, got %v", err) - } - if strings.Contains(string(payload), directKey) { - t.Fatalf("run result JSON leaked direct API key: %s", payload) - } -} - -func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) { - const missingEnv = "SCRIPTORIUM_PUBLIC_PREPARE_MISSING" - const firstKey = "first-direct-key" - const secondKey = "second-direct-key" - t.Setenv(missingEnv, "") - - profileDir := t.TempDir() - writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-prepare", "http://localhost:8000/v1", "test-model", missingEnv) - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - baseReq := scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "direct-prepare", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - } - firstReq := baseReq - firstReq.APIKey = firstKey - firstPrepared, err := engine.Prepare(context.Background(), firstReq) - if err != nil { - t.Fatalf("expected prepare with direct API key to succeed, got %v", err) - } - secondReq := baseReq - secondReq.APIKey = secondKey - secondPrepared, err := engine.Prepare(context.Background(), secondReq) - if err != nil { - t.Fatalf("expected prepare with alternate direct API key to succeed, got %v", err) - } - - if firstPrepared.PromptHash != secondPrepared.PromptHash { - t.Fatalf("direct API keys changed prompt hash: %q vs %q", firstPrepared.PromptHash, secondPrepared.PromptHash) - } - if firstPrepared.RenderedPromptHash != secondPrepared.RenderedPromptHash { - t.Fatalf("direct API keys changed rendered prompt hash: %q vs %q", firstPrepared.RenderedPromptHash, secondPrepared.RenderedPromptHash) - } - - payload, err := json.Marshal(firstPrepared) - if err != nil { - t.Fatalf("expected prepared run to marshal, got %v", err) - } - if strings.Contains(string(payload), firstKey) { - t.Fatalf("prepared run JSON leaked direct API key: %s", payload) - } -} - -func TestMissingCredentialsFailClearlyWhenProfileRequiresAuth(t *testing.T) { - const missingEnv = "SCRIPTORIUM_PUBLIC_AUTH_MISSING" - t.Setenv(missingEnv, "") - - profileDir := t.TempDir() - writePublicProfileFileWithAPIKeyEnv(t, profileDir, "requires-auth", "http://localhost:8000/v1", "test-model", missingEnv) - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "requires-auth", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, scriptorium.ErrInvalidRequest) { - t.Fatalf("expected invalid request for missing credentials, got %v", err) - } - if !errors.Is(err, scriptorium.ErrAPIKeyEnvMissing) { - t.Fatalf("expected missing credential environment error, got %v", err) - } - if err == nil || !strings.Contains(err.Error(), missingEnv) { - t.Fatalf("expected missing env name in error, got %v", err) - } -} - -func TestWithArtifactReaderRejectsNilReader(t *testing.T) { - _, err := scriptorium.NewEngine(contractConfig(frameworkSchemaDir), scriptorium.WithArtifactReader(nil)) - if !errors.Is(err, scriptorium.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestArtifactReaderReceivesPublicReferenceAndPreparesArtifact(t *testing.T) { - reader := &recordingArtifactReader{ - artifact: &scriptorium.Artifact{ - ContentType: "text/plain", - Body: []byte("Reader-supplied transcript."), - URI: "reader://transcript", - Size: int64(len("Reader-supplied transcript.")), - Hash: "reader-transcript-hash", - }, - } - engine := newArtifactReaderEngine(t, reader) - - ref := scriptorium.ArtifactRef{ - Type: scriptorium.ArtifactRefInline, - URI: "reader://transcript", - Body: "request body", - } - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "artifact-reader", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": ref, - }, - }) - if err != nil { - t.Fatalf("prepare with artifact reader: %v", err) - } - if len(reader.refs) != 1 || !reflect.DeepEqual(reader.refs[0], ref) { - t.Fatalf("reader received %#v, want %#v", reader.refs, ref) - } - if prepared.InputHashes["transcript"] != "reader-transcript-hash" { - t.Fatalf("unexpected input hash: %#v", prepared.InputHashes) - } - if len(prepared.Messages) != 1 || !strings.Contains(prepared.Messages[0].Content, "Reader-supplied transcript.") { - t.Fatalf("prepared prompt omitted reader artifact: %#v", prepared.Messages) - } -} - -func TestArtifactReaderFailuresPreserveArtifactLoadErrors(t *testing.T) { - readerErr := errors.New("artifact reader failed") - - tests := []struct { - name string - ctx context.Context - reader *recordingArtifactReader - wantNested error - }{ - { - name: "reader error", - ctx: context.Background(), - reader: &recordingArtifactReader{err: readerErr}, - wantNested: readerErr, - }, - { - name: "nil artifact", - ctx: context.Background(), - reader: &recordingArtifactReader{}, - }, - { - name: "reader cancellation", - ctx: context.Background(), - reader: &recordingArtifactReader{read: func(context.Context, scriptorium.ArtifactRef) (*scriptorium.Artifact, error) { - return nil, context.Canceled - }}, - wantNested: context.Canceled, - }, - { - name: "public reader error", - ctx: context.Background(), - reader: &recordingArtifactReader{err: scriptorium.ErrInvalidRequest}, - wantNested: scriptorium.ErrInvalidRequest, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - engine := newArtifactReaderEngine(t, tc.reader) - _, err := engine.Prepare(tc.ctx, scriptorium.RunRequest{ - PromptID: "artifact-reader", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("input"), - }, - }) - if !errors.Is(err, scriptorium.ErrArtifactLoad) { - t.Fatalf("expected ErrArtifactLoad, got %v", err) - } - if tc.wantNested != nil && !errors.Is(err, tc.wantNested) { - t.Fatalf("expected nested %v, got %v", tc.wantNested, err) - } - }) - } -} - -func TestRunAddsLLMGenerateToCollaboratorPublicError(t *testing.T) { - engine := newContractEngineWithOptions(t, frameworkSchemaDir, - scriptorium.WithLLMClient(&fakeLLMClient{err: scriptorium.ErrArtifactLoad}), - ) - - _, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, scriptorium.ErrLLMGenerate) { - t.Fatalf("expected ErrLLMGenerate, got %v", err) - } - if !errors.Is(err, scriptorium.ErrArtifactLoad) { - t.Fatalf("expected preserved ErrArtifactLoad, got %v", err) - } -} - -func TestPrepareWithoutProfileMatchesSpecificPublicError(t *testing.T) { - promptDir := t.TempDir() - writePublicPromptFile(t, promptDir, "profile-required", "") - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: promptDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "profile-required", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("input"), - }, - }) - if !errors.Is(err, scriptorium.ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if !errors.Is(err, scriptorium.ErrProfileRequired) { - t.Fatalf("expected ErrProfileRequired, got %v", err) - } -} - -func TestRunValidationFailureReturnsResult(t *testing.T) { - fake := &fakeLLMClient{ - response: &scriptorium.GenerateResponse{Content: ""}, - } - engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) - - result, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected validation failure as successful result, got %v", err) - } - if result.Validation.Status != scriptorium.ValidationFailed || result.Validation.IsValid { - t.Fatalf("expected failed validation result, got %+v", result.Validation) - } - if len(result.Validation.Errors) == 0 { - t.Fatalf("expected validation errors") - } -} - -func TestPublicErrorsSupportErrorsIs(t *testing.T) { - llmErr := errors.New("llm failed") - - tests := []struct { - name string - req scriptorium.RunRequest - client scriptorium.LLMClient - schemaDir string - want error - }{ - { - name: "invalid request", - req: scriptorium.RunRequest{}, - client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, - want: scriptorium.ErrInvalidRequest, - }, - { - name: "prompt not found", - req: scriptorium.RunRequest{PromptID: "missing.prompt"}, - client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, - want: scriptorium.ErrPromptNotFound, - }, - { - name: "profile not found", - req: scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "missing-profile", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - }, - }, - client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, - want: scriptorium.ErrProfileNotFound, - }, - { - name: "artifact load", - req: scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.File(filepath.Join(t.TempDir(), "does-not-exist.md")), - }, - }, - client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, - want: scriptorium.ErrArtifactLoad, - }, - { - name: "prompt render", - req: scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - }, - client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, - want: scriptorium.ErrPromptRender, - }, - { - name: "llm failure", - req: scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }, - client: &fakeLLMClient{err: llmErr}, - want: scriptorium.ErrLLMGenerate, - }, - { - name: "nil llm response", - req: scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }, - client: &fakeLLMClient{}, - want: scriptorium.ErrLLMGenerate, - }, - { - name: "validation runtime failure", - req: scriptorium.RunRequest{ - PromptID: frameworkStructuredEventsPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - }, - }, - client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}}, - schemaDir: t.TempDir(), - want: scriptorium.ErrValidation, - }, - } - - t.Setenv("SCRIPTORIUM_API_KEY", "test-secret") - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - schemaDir := tc.schemaDir - if schemaDir == "" { - schemaDir = frameworkSchemaDir - } - engine := newContractEngineWithOptions(t, schemaDir, scriptorium.WithLLMClient(tc.client)) - _, err := engine.Run(context.Background(), tc.req) - if !errors.Is(err, tc.want) { - t.Fatalf("expected errors.Is(%v), got %v", tc.want, err) - } - }) - } -} - -func TestSelectedProfileRawAPIKeyMapsToProfileLoad(t *testing.T) { - profileDir := t.TempDir() - if err := os.WriteFile(filepath.Join(profileDir, "raw.yaml"), []byte(` -id: raw-profile -endpoint: http://localhost:8000/v1 -model: model -api_key: secret -`), 0644); err != nil { - t.Fatal(err) - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "raw-profile", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, scriptorium.ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } - if errors.Is(err, scriptorium.ErrPromptLoad) { - t.Fatalf("did not expect ErrPromptLoad, got %v", err) - } -} - -func TestSelectedProfileInvalidYAMLMapsToProfileLoad(t *testing.T) { - profileDir := t.TempDir() - if err := os.WriteFile(filepath.Join(profileDir, "broken.yaml"), []byte(` -id: broken-profile -unknown_field: true -`), 0644); err != nil { - t.Fatal(err) - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "broken-profile", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, scriptorium.ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } - if errors.Is(err, scriptorium.ErrPromptLoad) { - t.Fatalf("did not expect ErrPromptLoad, got %v", err) - } -} - -func TestPromptRepositoryReadFailureMapsToPromptLoad(t *testing.T) { - missingPromptDir := filepath.Join(t.TempDir(), "missing-prompts") - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: missingPromptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, scriptorium.ErrPromptLoad) { - t.Fatalf("expected ErrPromptLoad, got %v", err) - } - if errors.Is(err, scriptorium.ErrProfileLoad) { - t.Fatalf("did not expect ErrProfileLoad, got %v", err) - } -} - -func TestSelectedProfileRepositoryReadFailureMapsToProfileLoad(t *testing.T) { - missingProfileDir := filepath.Join(t.TempDir(), "missing-profiles") - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: missingProfileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: frameworkFastProfileID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, scriptorium.ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } - if errors.Is(err, scriptorium.ErrPromptLoad) { - t.Fatalf("did not expect ErrPromptLoad, got %v", err) - } -} - -func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) { - t.Setenv("OPENROUTER_API_KEY", "test-key") - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected built-in profile prepare to succeed, got %v", err) - } - if prepared.SelectedProfileID != "mistral-small-3" { - t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) - } - if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" { - t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model) - } -} - -func TestPromptDefaultProfileCanUseBuiltInProfile(t *testing.T) { - t.Setenv("OPENROUTER_API_KEY", "test-key") - promptDir := t.TempDir() - writePublicPromptFile(t, promptDir, "prompt.builtin.default", "mistral-small-3") - - engine, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: promptDir}) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "prompt.builtin.default", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - }, - }) - if err != nil { - t.Fatalf("expected built-in default profile prepare to succeed, got %v", err) - } - if prepared.SelectedProfileID != "mistral-small-3" { - t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) - } -} - -func TestCustomProfileOverridesBuiltInProfile(t *testing.T) { - t.Setenv("OPENROUTER_API_KEY", "test-key") - profileDir := t.TempDir() - writePublicProfileFile(t, profileDir, "mistral-small-3", "http://localhost:8000/v1", "custom-model") - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected custom profile prepare to succeed, got %v", err) - } - if prepared.EffectiveModelParams.Model != "custom-model" { - t.Fatalf("expected custom profile to override built-in, got %q", prepared.EffectiveModelParams.Model) - } -} - -func TestMalformedCustomProfileDoesNotFallbackToBuiltIn(t *testing.T) { - t.Setenv("OPENROUTER_API_KEY", "test-key") - profileDir := t.TempDir() - if err := os.WriteFile(filepath.Join(profileDir, "mistral-small-3.yml"), []byte(` -id: mistral-small-3 -endpoint: http://localhost:8000/v1 -model: custom-model -unexpected: true -`), 0o644); err != nil { - t.Fatal(err) - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, scriptorium.ErrProfileLoad) { - t.Fatalf("expected custom profile load error, got %v", err) - } -} - -func TestPrepareWorksWithPromptFSAndRelativeContentFile(t *testing.T) { - promptFS := fstest.MapFS{ - "assets/prompts/fs-summary.yaml": &fstest.MapFile{Data: []byte(` -id: fs.summary -version: "1.0.0" -default_profile: contract-fast -inputs: - - name: transcript - required: true -messages: - - role: user - content_file: ./messages/summary.tmpl -output: - format: text - validation_mode: none - repair_attempts: 0 -`)}, - "assets/prompts/messages/summary.tmpl": &fstest.MapFile{Data: []byte(`Summarize {{input "transcript"}} from prompt fs.`)}, - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: t.TempDir(), - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithPromptFS(promptFS, "assets/prompts")) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "fs.summary", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if len(prepared.Messages) != 1 || !strings.Contains(prepared.Messages[0].Content, "prompt fs") { - t.Fatalf("expected content_file body from prompt fs, got %+v", prepared.Messages) - } -} - -func TestPrepareWithPromptFSRejectsEscapedContentFile(t *testing.T) { - promptFS := fstest.MapFS{ - "assets/prompts/fs-escape.yaml": &fstest.MapFile{Data: []byte(` -id: fs.escape -version: "1.0.0" -default_profile: contract-fast -messages: - - role: user - content_file: ../outside.tmpl -output: - format: text - validation_mode: none - repair_attempts: 0 -`)}, - "assets/outside.tmpl": &fstest.MapFile{Data: []byte(`Outside root.`)}, - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: t.TempDir(), - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithPromptFS(promptFS, "assets/prompts")) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{PromptID: "fs.escape"}) - if !errors.Is(err, scriptorium.ErrPromptLoad) { - t.Fatalf("expected ErrPromptLoad, got %v", err) - } -} - -func TestPrepareWorksWithPromptFile(t *testing.T) { - promptDir := t.TempDir() - promptPath := filepath.Join(promptDir, "single.yaml") - if err := os.WriteFile(promptPath, []byte(` -id: single.file.prompt -version: "1.0.0" -default_profile: contract-fast -inputs: - - name: transcript - required: true -messages: - - role: user - content_file: ./single.tmpl -output: - format: text - validation_mode: none - repair_attempts: 0 -`), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(promptDir, "single.tmpl"), []byte(`Summarize {{input "transcript"}} from file.`), 0o644); err != nil { - t.Fatal(err) - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithPromptFile(promptPath)) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "single.file.prompt", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if prepared.PromptID != "single.file.prompt" { - t.Fatalf("unexpected prompt id: %q", prepared.PromptID) - } - if len(prepared.Messages) != 1 || !strings.Contains(prepared.Messages[0].Content, "from file") { - t.Fatalf("expected content_file body from prompt file, got %+v", prepared.Messages) - } -} - -func TestPrepareWorksWithProfileFSOverBuiltIns(t *testing.T) { - profileFS := fstest.MapFS{ - "profiles/mistral-small-3.yaml": &fstest.MapFile{Data: []byte(` -id: mistral-small-3 -endpoint: http://profile-fs/v1 -model: profile-fs-model -`)}, - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithProfileFS(profileFS, "profiles")) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if prepared.EffectiveModelParams.Model != "profile-fs-model" { - t.Fatalf("expected profile fs to override built-in, got %q", prepared.EffectiveModelParams.Model) - } -} - -func TestPrepareWorksWithProfileFileOverBuiltIns(t *testing.T) { - profileDir := t.TempDir() - profilePath := filepath.Join(profileDir, "mistral-small-3.yaml") - if err := os.WriteFile(profilePath, []byte(` -id: mistral-small-3 -endpoint: http://profile-file/v1 -model: profile-file-model -`), 0o644); err != nil { - t.Fatal(err) - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithProfileFile(profilePath)) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if prepared.EffectiveModelParams.Model != "profile-file-model" { - t.Fatalf("expected profile file to override built-in, got %q", prepared.EffectiveModelParams.Model) - } -} - -func TestPrepareWorksWithInMemoryProfilesWithoutProfileFiles(t *testing.T) { - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithProfiles(scriptorium.Profile{ - ID: "memory-profile", - Endpoint: "http://memory-profile/v1", - Model: "memory-model", - })) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "memory-profile", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if prepared.EffectiveModelParams.Model != "memory-model" { - t.Fatalf("expected in-memory profile model, got %q", prepared.EffectiveModelParams.Model) - } -} - -func TestInMemoryProfilesOverrideBuiltInsAndProfileSources(t *testing.T) { - profileFS := fstest.MapFS{ - "profiles/mistral-small-3.yaml": &fstest.MapFile{Data: []byte(` -id: mistral-small-3 -endpoint: http://profile-fs/v1 -model: profile-fs-model -`)}, - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, - scriptorium.WithProfileFS(profileFS, "profiles"), - scriptorium.WithProfiles(scriptorium.Profile{ - ID: "mistral-small-3", - Endpoint: "http://memory-profile/v1", - Model: "memory-profile-model", - }), - ) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if prepared.EffectiveModelParams.Model != "memory-profile-model" { - t.Fatalf("expected in-memory profile to have highest precedence, got %q", prepared.EffectiveModelParams.Model) - } -} - -func TestWithProfilesRejectsDuplicateIDs(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}, - scriptorium.WithProfiles( - scriptorium.Profile{ID: "duplicate", Endpoint: "http://one/v1", Model: "one"}, - scriptorium.Profile{ID: "duplicate", Endpoint: "http://two/v1", Model: "two"}, - ), - ) - if !errors.Is(err, scriptorium.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) { - fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - prof := scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ - ID: "template-profile", - Endpoint: "http://template/v1", - Model: "template-model", - APIKeyRequired: true, - ExtraParams: map[string]any{ - "provider": "template", - }, - }) - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithProfiles(prof), scriptorium.WithLLMClient(fake)) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "template-profile", - APIKey: "template-key", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if len(fake.requests) != 1 { - t.Fatalf("expected one request, got %d", len(fake.requests)) - } - if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" { - t.Fatalf("unexpected generated request: %+v", fake.requests[0]) - } - if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) { - t.Fatalf("unexpected extra params: %#v", fake.requests[0].Target.ExtraParams) - } -} - -func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) { - intPointer := func(value int) *int { - return &value - } - - tests := []struct { - name string - configTimeout time.Duration - suppliedClientTimeout time.Duration - profileTimeoutSeconds int - requestTimeoutSeconds *int - callerTimeout time.Duration - wantRemainingAtRequest time.Duration - }{ - { - name: "positive supplied client cap takes precedence over config", - configTimeout: 2 * time.Second, - suppliedClientTimeout: 6 * time.Second, - wantRemainingAtRequest: 6 * time.Second, - }, - { - name: "zero supplied client timeout inherits config cap", - configTimeout: 5 * time.Second, - wantRemainingAtRequest: 5 * time.Second, - }, - { - name: "profile deadline is shorter than transport cap", - suppliedClientTimeout: 6 * time.Second, - profileTimeoutSeconds: 4, - wantRemainingAtRequest: 4 * time.Second, - }, - { - name: "request deadline is shorter than profile and transport limits", - suppliedClientTimeout: 6 * time.Second, - profileTimeoutSeconds: 4, - requestTimeoutSeconds: intPointer(2), - wantRemainingAtRequest: 2 * time.Second, - }, - { - name: "explicit zero removes generation deadline but retains transport cap", - suppliedClientTimeout: 5 * time.Second, - profileTimeoutSeconds: 2, - requestTimeoutSeconds: intPointer(0), - wantRemainingAtRequest: 5 * time.Second, - }, - { - name: "framework default remains layered with shorter transport cap", - configTimeout: 7 * time.Second, - suppliedClientTimeout: 3 * time.Second, - wantRemainingAtRequest: 3 * time.Second, - }, - { - name: "caller deadline remains layered with other limits", - suppliedClientTimeout: 6 * time.Second, - profileTimeoutSeconds: 4, - callerTimeout: 2 * time.Second, - wantRemainingAtRequest: 2 * time.Second, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var ( - sawDeadline bool - remaining time.Duration - ) - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - deadline, ok := req.Context().Deadline() - sawDeadline = ok - if ok { - remaining = time.Until(deadline) - } - return &http.Response{ - StatusCode: http.StatusOK, - Status: "200 OK", - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"choices":[{"message":{"content":"ok"}}]}`, - )), - Request: req, - }, nil - }) - httpClient := &http.Client{ - Timeout: tc.suppliedClientTimeout, - Transport: transport, - } - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - Timeout: tc.configTimeout, - HTTPClient: httpClient, - }, scriptorium.WithProfiles(scriptorium.Profile{ - ID: "layered-timeout", - Endpoint: "http://timeout.test/v1", - Model: "timeout-model", - TimeoutSeconds: tc.profileTimeoutSeconds, - })) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - ctx := context.Background() - cancel := func() {} - if tc.callerTimeout > 0 { - ctx, cancel = context.WithTimeout(ctx, tc.callerTimeout) - } - defer cancel() - - _, err = engine.Run(ctx, scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "layered-timeout", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - Execution: &scriptorium.ExecutionTargetOverride{ - TimeoutSeconds: tc.requestTimeoutSeconds, - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if !sawDeadline { - t.Fatal("expected outbound request context to have a deadline") - } - - const deadlineTolerance = 750 * time.Millisecond - if remaining < tc.wantRemainingAtRequest-deadlineTolerance || - remaining > tc.wantRemainingAtRequest+50*time.Millisecond { - t.Fatalf( - "unexpected request deadline: remaining=%v want approximately %v", - remaining, - tc.wantRemainingAtRequest, - ) - } - }) - } -} - -func TestOpenAICompatibleProfileDefersExtraParamsValidation(t *testing.T) { - cyclic := map[string]any{} - cyclic["self"] = cyclic - - prof := scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ - ID: "cyclic-template-profile", - Endpoint: "http://cyclic-template/v1", - Model: "cyclic-template-model", - ExtraParams: cyclic, - }) - - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}, - scriptorium.WithProfiles(prof), - ) - if !errors.Is(err, scriptorium.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestOpenAICompatibleProfileNestedExtraParamsRunThroughWithProfiles(t *testing.T) { - fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - nested := map[string]any{ - "labels": map[string]string{"route": "primary"}, - "ids": []int{1, 2, 3}, - } - extraParams := map[string]any{ - "nested": nested, - } - prof := scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ - ID: "nested-template-profile", - Endpoint: "http://nested-template/v1", - Model: "nested-template-model", - ExtraParams: extraParams, - }) - extraParams["added"] = "mutated-after-construction" - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithProfiles(prof), scriptorium.WithLLMClient(fake)) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - nested["added"] = "mutated-after-construction" - - _, err = engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "nested-template-profile", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - want := map[string]any{ - "nested": map[string]any{ - "labels": map[string]string{"route": "primary"}, - "ids": []int{1, 2, 3}, - }, - } - if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, want) { - t.Fatalf("unexpected extra params:\ngot=%#v\nwant=%#v", fake.requests[0].Target.ExtraParams, want) - } -} - -func TestInMemoryProfileAPIKeyRequiredBehavior(t *testing.T) { - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithProfiles(scriptorium.Profile{ - ID: "requires-key", - Endpoint: "http://requires-key/v1", - Model: "requires-key-model", - APIKeyRequired: true, - })) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - req := scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "requires-key", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - } - _, err = engine.Prepare(context.Background(), req) - if !errors.Is(err, scriptorium.ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest without API key, got %v", err) - } - req.APIKey = "direct-required-key" - if _, err := engine.Prepare(context.Background(), req); err != nil { - t.Fatalf("expected direct API key to satisfy APIKeyRequired, got %v", err) - } -} - -func TestInMemoryProfileWithoutAPIKeyRequiredWorksWithoutKey(t *testing.T) { - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithProfiles(scriptorium.Profile{ - ID: "no-key-required", - Endpoint: "http://no-key/v1", - Model: "no-key-model", - })) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "no-key-required", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected prepare without API key to succeed, got %v", err) - } -} - -func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) { - fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - labels := map[string]string{"route": "primary"} - ids := []int{1, 2, 3} - extraParams := map[string]any{ - "labels": labels, - "ids": ids, - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, - scriptorium.WithProfiles(scriptorium.Profile{ - ID: "copy-profile", - Endpoint: "http://copy/v1", - Model: "copy-model", - ExtraParams: extraParams, - }), - scriptorium.WithLLMClient(fake), - ) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - labels["route"] = "mutated-before-run" - ids[0] = 99 - extraParams["added"] = "mutated" - - _, err = engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "copy-profile", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - want := map[string]any{ - "labels": map[string]string{"route": "primary"}, - "ids": []int{1, 2, 3}, - } - if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, want) { - t.Fatalf("captured extra params changed after mutation:\ngot=%#v\nwant=%#v", fake.requests[0].Target.ExtraParams, want) - } -} - -func TestWithProfilesRejectsInvalidExtraParams(t *testing.T) { - tests := []struct { - name string - extraParams map[string]any - }{ - {name: "function", extraParams: map[string]any{"bad": func() {}}}, - {name: "channel", extraParams: map[string]any{"bad": make(chan struct{})}}, - {name: "struct", extraParams: map[string]any{"bad": struct{ Name string }{Name: "bad"}}}, - {name: "non string map key", extraParams: map[string]any{"bad": map[int]string{1: "one"}}}, - {name: "nan", extraParams: map[string]any{"bad": math.NaN()}}, - {name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}}, - {name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}, - scriptorium.WithProfiles(scriptorium.Profile{ - ID: "invalid-extra-params", - Endpoint: "http://invalid/v1", - Model: "invalid-model", - ExtraParams: tc.extraParams, - }), - ) - if !errors.Is(err, scriptorium.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) - } -} - -func TestWithProfilesRejectsCyclicExtraParams(t *testing.T) { - cyclicMap := map[string]any{} - cyclicMap["self"] = cyclicMap - cyclicSlice := []any{nil} - cyclicSlice[0] = cyclicSlice - - tests := []struct { - name string - extraParams map[string]any - }{ - {name: "map", extraParams: cyclicMap}, - {name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}, - scriptorium.WithProfiles(scriptorium.Profile{ - ID: "cyclic-extra-params", - Endpoint: "http://cyclic/v1", - Model: "cyclic-model", - ExtraParams: tc.extraParams, - }), - ) - if !errors.Is(err, scriptorium.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) - } -} - -func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) { - fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}} - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: t.TempDir(), - ProfileDir: frameworkProfileDir, - SchemaDir: t.TempDir(), - }, - scriptorium.WithPromptFS(publicStructuredPromptFS("schema.fs.prompt", "events.schema.json"), "prompts"), - scriptorium.WithSchemaFS(publicSchemaFS(), "schemas"), - scriptorium.WithLLMClient(fake), - ) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - result, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "schema.fs.prompt", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid { - t.Fatalf("expected schema validation to pass, got %+v", result.Validation) - } - if len(fake.requests) != 1 || fake.requests[0].StructuredOutput == nil { - t.Fatalf("expected structured output request, got %+v", fake.requests) - } -} - -func TestRunStructuredOutputWorksWithSchemaFile(t *testing.T) { - schemaDir := t.TempDir() - schemaPath := filepath.Join(schemaDir, "events.schema.json") - if err := os.WriteFile(schemaPath, []byte(publicSchemaJSON()), 0o644); err != nil { - t.Fatal(err) - } - - fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}} - engine, err := scriptorium.NewEngine(scriptorium.Config{ - ProfileDir: frameworkProfileDir, - }, - scriptorium.WithPromptFS(publicStructuredPromptFS("schema.file.prompt", "events.schema.json"), "prompts"), - scriptorium.WithSchemaFile(schemaPath), - scriptorium.WithLLMClient(fake), - ) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - result, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "schema.file.prompt", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid { - t.Fatalf("expected schema validation to pass, got %+v", result.Validation) - } -} - -func TestSourceOptionsRejectInvalidInputs(t *testing.T) { - missingFile := filepath.Join(t.TempDir(), "missing.yaml") - directoryPath := t.TempDir() - - tests := []struct { - name string - opt scriptorium.Option - }{ - {name: "prompt fs nil", opt: scriptorium.WithPromptFS(nil, "prompts")}, - {name: "prompt fs empty root", opt: scriptorium.WithPromptFS(fstest.MapFS{}, "")}, - {name: "prompt file empty", opt: scriptorium.WithPromptFile("")}, - {name: "prompt file missing", opt: scriptorium.WithPromptFile(missingFile)}, - {name: "prompt file directory", opt: scriptorium.WithPromptFile(directoryPath)}, - {name: "profile fs nil", opt: scriptorium.WithProfileFS(nil, "profiles")}, - {name: "profile fs empty root", opt: scriptorium.WithProfileFS(fstest.MapFS{}, "")}, - {name: "profile file empty", opt: scriptorium.WithProfileFile("")}, - {name: "schema fs nil", opt: scriptorium.WithSchemaFS(nil, "schemas")}, - {name: "schema fs empty root", opt: scriptorium.WithSchemaFS(fstest.MapFS{}, "")}, - {name: "schema file empty", opt: scriptorium.WithSchemaFile("")}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}, tc.opt) - if !errors.Is(err, scriptorium.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) - } -} - -func TestPackageOptionsComposeFromSlice(t *testing.T) { - fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - options := []scriptorium.Option{ - nil, - scriptorium.WithProfiles(scriptorium.Profile{ - ID: "slice-profile", - Endpoint: "http://slice/v1", - Model: "slice-model", - }), - scriptorium.WithLLMClient(fake), - } - - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, options...) - if err != nil { - t.Fatalf("expected package-provided options to compose, got %v", err) - } - - _, err = engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "slice-profile", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run with composed options to succeed, got %v", err) - } - if len(fake.requests) != 1 { - t.Fatalf("expected one generate request, got %d", len(fake.requests)) - } - if fake.requests[0].Target.Model != "slice-model" { - t.Fatalf("expected profile from composed options, got %q", fake.requests[0].Target.Model) - } -} - -func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T) { - fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) - - labels := map[string]string{"route": "primary"} - counts := map[string]int{"retry_budget": 2} - weights := []float64{0.25, 0.75} - ids := []int{1, 2, 3} - nested := map[string]any{ - "labels": labels, - "counts": counts, - "weights": weights, - "ids": ids, - } - extraParams := map[string]any{ - "labels": labels, - "counts": counts, - "nested": nested, - } - - _, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - Execution: &scriptorium.ExecutionTargetOverride{ExtraParams: extraParams}, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if len(fake.requests) != 1 { - t.Fatalf("expected one generate request, got %d", len(fake.requests)) - } - - captured := fake.requests[0].Target.ExtraParams - labels["route"] = "mutated" - counts["retry_budget"] = 99 - weights[0] = 9.9 - ids[0] = 99 - nested["added"] = "mutated" - extraParams["new_top_level"] = "mutated" - - want := map[string]any{ - "labels": map[string]string{"route": "primary"}, - "counts": map[string]int{"retry_budget": 2}, - "nested": map[string]any{ - "labels": map[string]string{"route": "primary"}, - "counts": map[string]int{"retry_budget": 2}, - "weights": []float64{0.25, 0.75}, - "ids": []int{1, 2, 3}, - }, - } - if !reflect.DeepEqual(captured, want) { - t.Fatalf("captured extra_params changed after mutating source:\ngot=%#v\nwant=%#v", captured, want) - } -} - -func TestRunRejectsInvalidExtraParams(t *testing.T) { - tests := []struct { - name string - extraParams map[string]any - }{ - {name: "function", extraParams: map[string]any{"bad": func() {}}}, - {name: "channel", extraParams: map[string]any{"bad": make(chan struct{})}}, - {name: "struct", extraParams: map[string]any{"bad": struct{ Name string }{Name: "bad"}}}, - {name: "non string map key", extraParams: map[string]any{"bad": map[int]string{1: "one"}}}, - {name: "nan", extraParams: map[string]any{"bad": math.NaN()}}, - {name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}}, - {name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) - - _, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - Execution: &scriptorium.ExecutionTargetOverride{ExtraParams: tc.extraParams}, - }) - if !errors.Is(err, scriptorium.ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if len(fake.requests) != 0 { - t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests)) - } - }) - } -} - -func TestRunRejectsCyclicExtraParams(t *testing.T) { - cyclicMap := map[string]any{} - cyclicMap["self"] = cyclicMap - cyclicSlice := []any{nil} - cyclicSlice[0] = cyclicSlice - - tests := []struct { - name string - extraParams map[string]any - }{ - {name: "map", extraParams: cyclicMap}, - {name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) - - _, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.Inline("Rin opens the gate."), - "glossary": scriptorium.Inline("gate: A guarded passage."), - }, - Execution: &scriptorium.ExecutionTargetOverride{ExtraParams: tc.extraParams}, - }) - if !errors.Is(err, scriptorium.ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if len(fake.requests) != 0 { - t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests)) - } - }) - } -} - -func TestWithLLMClientRejectsNilClient(t *testing.T) { - _, err := scriptorium.NewEngine(contractConfig(frameworkSchemaDir), scriptorium.WithLLMClient(nil)) - if !errors.Is(err, scriptorium.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestNewEngineConstructsDefaultLLMClientWithoutCredentials(t *testing.T) { - if _, err := scriptorium.NewEngine(contractConfig(frameworkSchemaDir)); err != nil { - t.Fatalf("expected default engine construction without credentials to succeed, got %v", err) - } -} - -func newContractEngine(t *testing.T) *scriptorium.Engine { - t.Helper() - - for _, path := range []string{ - frameworkPromptDir, - frameworkProfileDir, - frameworkSchemaDir, - } { - if _, err := os.Stat(path); err != nil { - t.Fatalf("expected framework contract path %s to exist: %v", path, err) - } - } - - engine, err := scriptorium.NewEngine(contractConfig(frameworkSchemaDir)) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - return engine -} - -func newContractEngineWithOptions(t *testing.T, schemaDir string, opts ...scriptorium.Option) *scriptorium.Engine { - t.Helper() - - engine, err := scriptorium.NewEngine(contractConfig(schemaDir), opts...) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - return engine -} - -func newArtifactReaderEngine(t *testing.T, reader scriptorium.ArtifactReader) *scriptorium.Engine { - t.Helper() - - promptDir := t.TempDir() - writePublicPromptFile(t, promptDir, "artifact-reader", frameworkFastProfileID) - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: promptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - }, scriptorium.WithArtifactReader(reader)) - if err != nil { - t.Fatalf("construct engine with artifact reader: %v", err) - } - return engine -} - -func contractConfig(schemaDir string) scriptorium.Config { - return scriptorium.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: schemaDir, - } -} - -func writePublicPromptFile(t *testing.T, dir, id, defaultProfile string) { - t.Helper() - data := `id: ` + id + ` -version: "1.0.0" -default_profile: ` + defaultProfile + ` -inputs: - - name: transcript - required: true -messages: - - role: user - content: "Summarize: {{input \"transcript\"}}" -output: - format: text - validation_mode: none - repair_attempts: 0 -` - if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { - t.Fatalf("failed to write prompt fixture: %v", err) - } -} - -func writePublicProfileFile(t *testing.T, dir, id, endpoint, model string) { - t.Helper() - data := `id: ` + id + ` -endpoint: ` + endpoint + ` -model: ` + model + ` -` - if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { - t.Fatalf("failed to write profile fixture: %v", err) - } -} - -func writePublicProfileFileWithAPIKeyEnv(t *testing.T, dir, id, endpoint, model, apiKeyEnv string) { - t.Helper() - data := `id: ` + id + ` -endpoint: ` + endpoint + ` -model: ` + model + ` -api_key_env: ` + apiKeyEnv + ` -` - if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { - t.Fatalf("failed to write profile fixture: %v", err) - } -} - -func publicStructuredPromptFS(id string, schemaPath string) fstest.MapFS { - return fstest.MapFS{ - "prompts/prompt.yaml": &fstest.MapFile{Data: []byte(`id: ` + id + ` -version: "1.0.0" -default_profile: contract-fast -inputs: - - name: transcript - required: true -messages: - - role: user - content: "Extract events from {{input \"transcript\"}}." -output: - format: json - validation_mode: json_schema - schema_path: ` + schemaPath + ` - repair_attempts: 0 -`)}, - } -} - -func publicSchemaFS() fstest.MapFS { - return fstest.MapFS{ - "schemas/events.schema.json": &fstest.MapFile{Data: []byte(publicSchemaJSON())}, - } -} - -func publicSchemaJSON() string { - return `{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": ["events"], - "properties": { - "events": {"type": "array"} - } -}` -} - -type executionProfileFixture struct { - id string - endpoint string - model string - temperature float64 - maxTokens int - topP float64 - timeoutSeconds int - serviceTier string - reasoningEffort string - apiKeyEnv string - extraParamSource string -} - -func executionTargetFromProfileFixture(profile executionProfileFixture) scriptorium.ExecutionTarget { - return scriptorium.ExecutionTarget{ - Endpoint: profile.endpoint, - Model: profile.model, - Temperature: profile.temperature, - MaxTokens: profile.maxTokens, - TopP: profile.topP, - TimeoutSeconds: profile.timeoutSeconds, - ServiceTier: profile.serviceTier, - ReasoningEffort: profile.reasoningEffort, - APIKeyEnv: profile.apiKeyEnv, - ExtraParams: map[string]any{"source": profile.extraParamSource}, - } -} - -func writeExecutionProfileFixture(t *testing.T, dir string, profile executionProfileFixture) { - t.Helper() - data := fmt.Sprintf(`id: %s -endpoint: %s -model: %s -temperature: %g -max_tokens: %d -top_p: %g -timeout_seconds: %d -service_tier: %s -reasoning_effort: %s -api_key_env: %s -extra_params: - source: %q -`, - profile.id, - profile.endpoint, - profile.model, - profile.temperature, - profile.maxTokens, - profile.topP, - profile.timeoutSeconds, - profile.serviceTier, - profile.reasoningEffort, - profile.apiKeyEnv, - profile.extraParamSource, - ) - if err := os.WriteFile(filepath.Join(dir, profile.id+".yaml"), []byte(data), 0o644); err != nil { - t.Fatalf("write execution profile fixture: %v", err) - } -} - -type fakeLLMClient struct { - response *scriptorium.GenerateResponse - err error - requests []scriptorium.GenerateRequest -} - -type recordingArtifactReader struct { - artifact *scriptorium.Artifact - err error - refs []scriptorium.ArtifactRef - read func(context.Context, scriptorium.ArtifactRef) (*scriptorium.Artifact, error) -} - -func (r *recordingArtifactReader) Read(ctx context.Context, ref scriptorium.ArtifactRef) (*scriptorium.Artifact, error) { - r.refs = append(r.refs, ref) - if r.read != nil { - return r.read(ctx, ref) - } - if r.err != nil { - return nil, r.err - } - return r.artifact, nil -} - -type roundTripFunc func(*http.Request) (*http.Response, error) - -func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { - return f(req) -} - -func (f *fakeLLMClient) Generate(_ context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) { - f.requests = append(f.requests, req) - if f.err != nil { - return nil, f.err - } - return f.response, nil -} diff --git a/errors.go b/errors.go deleted file mode 100644 index 6099b45..0000000 --- a/errors.go +++ /dev/null @@ -1,60 +0,0 @@ -package scriptorium - -import ( - "errors" - "fmt" - - "gitea.maximumdirect.net/eric/scriptorium/internal/profile" - "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" - "gitea.maximumdirect.net/eric/scriptorium/internal/usecase" -) - -func mapPublicError(err error) error { - if err == nil { - return nil - } - publicErr := publicErrorFor(err) - if publicErr == nil { - return err - } - return fmt.Errorf("%w: %w", publicErr, err) -} - -func publicErrorFor(err error) error { - switch { - case errors.Is(err, promptdef.ErrPromptDefinitionNotFound): - return ErrPromptNotFound - case errors.Is(err, profile.ErrProfileNotFound): - return ErrProfileNotFound - case errors.Is(err, usecase.ErrProfileRequired): - return errors.Join(ErrInvalidRequest, ErrProfileRequired) - case errors.Is(err, usecase.ErrPromptLoad): - return ErrPromptLoad - case errors.Is(err, usecase.ErrProfileLoad): - return ErrProfileLoad - case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition): - return ErrPromptLoad - case isProfileLoadCause(err): - return ErrProfileLoad - case errors.Is(err, usecase.ErrAPIKeyEnvMissing): - return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing) - case errors.Is(err, usecase.ErrArtifactLoad): - return ErrArtifactLoad - case errors.Is(err, usecase.ErrPromptRender): - return ErrPromptRender - case errors.Is(err, usecase.ErrLLMGenerate): - return ErrLLMGenerate - case errors.Is(err, usecase.ErrValidation): - return ErrValidation - case errors.Is(err, usecase.ErrInvalidRequest): - return ErrInvalidRequest - default: - return nil - } -} - -func isProfileLoadCause(err error) bool { - return errors.Is(err, profile.ErrInvalidYAML) || - errors.Is(err, profile.ErrInvalidProfile) || - errors.Is(err, profile.ErrRawAPIKeyNotAllowed) -} diff --git a/examples/go-library/prepare/main.go b/examples/go-library/prepare/main.go deleted file mode 100644 index 4b3397d..0000000 --- a/examples/go-library/prepare/main.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "log" - "os" - - "gitea.maximumdirect.net/eric/scriptorium" -) - -func main() { - engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - ProfileDir: "./examples/profiles", - SchemaDir: "./examples/schemas", - }) - if err != nil { - log.Fatal(err) - } - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.File("./examples/fixtures/transcript.md"), - "glossary": scriptorium.File("./examples/fixtures/glossary.yml"), - }, - }) - if err != nil { - log.Fatal(err) - } - - summary := struct { - PromptID string `json:"prompt_id"` - SelectedProfileID string `json:"selected_profile_id"` - Model string `json:"model"` - MessageCount int `json:"message_count"` - InputHashes map[string]string `json:"input_hashes"` - }{ - PromptID: prepared.PromptID, - SelectedProfileID: prepared.SelectedProfileID, - Model: prepared.EffectiveModelParams.Model, - MessageCount: len(prepared.Messages), - InputHashes: prepared.InputHashes, - } - - if err := json.NewEncoder(os.Stdout).Encode(summary); err != nil { - log.Fatal(err) - } -} diff --git a/formatting.go b/formatting.go deleted file mode 100644 index 910f094..0000000 --- a/formatting.go +++ /dev/null @@ -1,51 +0,0 @@ -package scriptorium - -import "fmt" - -// String returns a concise request summary without exposing direct API keys. -func (r RunRequest) String() string { - return r.redactedString() -} - -// GoString returns a concise request summary without exposing direct API keys. -func (r RunRequest) GoString() string { - return r.redactedString() -} - -func (r RunRequest) redactedString() string { - return fmt.Sprintf( - "scriptorium.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t Metadata:%d}", - r.PromptID, - r.PromptVersion, - r.ProfileID, - r.APIKey != "", - len(r.Inputs), - len(r.Vars), - r.Execution != nil, - r.Validation != nil, - len(r.Metadata), - ) -} - -// String returns a concise request summary without exposing direct API keys or -// rendered prompt content. -func (r GenerateRequest) String() string { - return r.redactedString() -} - -// GoString returns a concise request summary without exposing direct API keys or -// rendered prompt content. -func (r GenerateRequest) GoString() string { - return r.redactedString() -} - -func (r GenerateRequest) redactedString() string { - return fmt.Sprintf( - "scriptorium.GenerateRequest{Messages:%d Model:%q APIKeySet:%t StructuredOutputSet:%t ExtraParams:%d}", - len(r.Prompt.Messages), - r.Target.Model, - r.APIKey != "", - r.StructuredOutput != nil, - len(r.Target.ExtraParams), - ) -} diff --git a/go.mod b/go.mod index ce5844d..8a7fa2f 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,10 @@ go 1.25.5 require ( gitea.maximumdirect.net/eric/promptkit v0.1.0 - github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 gopkg.in/yaml.v3 v3.0.1 ) -require golang.org/x/text v0.14.0 // indirect +require ( + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + golang.org/x/text v0.14.0 // indirect +) diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index f6b5242..84236e8 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -359,7 +359,7 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) { fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override") fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override") fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override") - fs.DurationVar(&cfg.timeout, "timeout", defaults.LLMRequestTimeoutDefault, "LLM request timeout") + fs.DurationVar(&cfg.timeout, "timeout", 0, "LLM request timeout") fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt") fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile") } diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index 458af60..7c23ab5 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -222,8 +222,8 @@ func TestParseRunArgsTimeout(t *testing.T) { if err != nil { t.Fatalf("expected valid run args, got %v", err) } - if cfg.timeout != defaults.LLMRequestTimeoutDefault { - t.Fatalf("expected default timeout %s, got %s", defaults.LLMRequestTimeoutDefault, cfg.timeout) + if cfg.timeout != 0 { + t.Fatalf("expected omitted timeout to remain unset, got %s", cfg.timeout) } cfg, err = parseRunArgs([]string{ diff --git a/internal/artifact/reader.go b/internal/artifact/reader.go deleted file mode 100644 index eb881fb..0000000 --- a/internal/artifact/reader.go +++ /dev/null @@ -1,122 +0,0 @@ -package artifact - -import ( - "context" - "crypto/sha256" - "errors" - "fmt" - "io" - "mime" - "os" - "path/filepath" - - "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -var ( - ErrUnsupportedRefType = errors.New("unsupported artifact reference type") - ErrMissingInlineBody = errors.New("missing body for inline artifact") - ErrMissingFilePath = errors.New("missing file path for file artifact") -) - -// Reader resolves artifact references into actual artifacts. -type Reader interface { - Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) -} - -// CompositeReader routes artifact resolution based on the reference type. -type CompositeReader struct { - inlineReader *inlineReader - fileReader Reader -} - -func NewCompositeReader() Reader { - return &CompositeReader{ - inlineReader: &inlineReader{}, - fileReader: &fileReader{}, - } -} - -func (c *CompositeReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - switch ref.Type { - case domain.ArtifactRefInline: - return c.inlineReader.Read(ctx, ref) - case domain.ArtifactRefFile: - return c.fileReader.Read(ctx, ref) - default: - return nil, fmt.Errorf("%w: %s", ErrUnsupportedRefType, ref.Type) - } -} - -type inlineReader struct{} - -func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - if ref.Body == "" { - return nil, ErrMissingInlineBody - } - - body := []byte(ref.Body) - return &domain.Artifact{ - ContentType: defaults.ContentTypeTextPlain, - Body: body, - Size: int64(len(body)), - Hash: fmt.Sprintf("%x", sha256.Sum256(body)), - URI: ref.URI, - }, nil -} - -type fileReader struct{} - -func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - if ref.URI == "" { - return nil, ErrMissingFilePath - } - - return readFileArtifact(ref.URI) -} - -func readFileArtifact(path string) (*domain.Artifact, error) { - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to read file %s: %w", path, err) - } - defer file.Close() - - data, err := io.ReadAll(file) - if err != nil { - return nil, fmt.Errorf("failed to read file %s: %w", path, err) - } - - contentType := mime.TypeByExtension(filepath.Ext(path)) - if contentType == "" { - contentType = defaults.ContentTypeTextPlain - } - - return &domain.Artifact{ - Name: filepath.Base(path), - ContentType: contentType, - Body: data, - URI: path, - Size: int64(len(data)), - Hash: fmt.Sprintf("%x", sha256.Sum256(data)), - }, nil -} diff --git a/internal/artifact/reader_test.go b/internal/artifact/reader_test.go deleted file mode 100644 index 442e811..0000000 --- a/internal/artifact/reader_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package artifact - -import ( - "context" - "errors" - "os" - "testing" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -func TestCompositeReader_Read(t *testing.T) { - reader := NewCompositeReader() - ctx := context.Background() - - t.Run("inline artifact", func(t *testing.T) { - ref := domain.ArtifactRef{ - Type: domain.ArtifactRefInline, - Body: "hello world", - } - art, err := reader.Read(ctx, ref) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(art.Body) != "hello world" { - t.Errorf("expected 'hello world', got %s", string(art.Body)) - } - if art.ContentType != "text/plain" { - t.Errorf("expected text/plain content type, got %q", art.ContentType) - } - if art.Hash != "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" { - t.Errorf("unexpected hash: %s", art.Hash) - } - }) - - t.Run("inline artifact missing body", func(t *testing.T) { - ref := domain.ArtifactRef{ - Type: domain.ArtifactRefInline, - Body: "", - } - _, err := reader.Read(ctx, ref) - if !errors.Is(err, ErrMissingInlineBody) { - t.Errorf("expected ErrMissingInlineBody, got %v", err) - } - }) - - t.Run("unsupported ref type", func(t *testing.T) { - ref := domain.ArtifactRef{ - Type: domain.ArtifactRefType("unsupported"), - URI: "unsupported://bucket/key", - } - _, err := reader.Read(ctx, ref) - if !errors.Is(err, ErrUnsupportedRefType) { - t.Error("expected error for unsupported type") - } - }) -} - -func TestFileReader_Read(t *testing.T) { - content := []byte("test file content") - tmpFile, err := os.CreateTemp("", "artifact_test_*.txt") - if err != nil { - t.Fatal(err) - } - defer os.Remove(tmpFile.Name()) - - if _, err := tmpFile.Write(content); err != nil { - t.Fatal(err) - } - tmpFile.Close() - - reader := NewCompositeReader() - ctx := context.Background() - - t.Run("file artifact loading", func(t *testing.T) { - ref := domain.ArtifactRef{ - Type: domain.ArtifactRefFile, - URI: tmpFile.Name(), - } - art, err := reader.Read(ctx, ref) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(art.Body) != string(content) { - t.Errorf("expected %s, got %s", string(content), string(art.Body)) - } - if art.Name == "" { - t.Error("expected name to be inferred from filename") - } - if art.Hash != "60f5237ed4049f0382661ef009d2bc42e48c3ceb3edb6600f7024e7ab3b838f3" { - t.Errorf("unexpected hash: %s", art.Hash) - } - }) - - t.Run("missing file path", func(t *testing.T) { - ref := domain.ArtifactRef{ - Type: domain.ArtifactRefFile, - URI: "", - } - _, err := reader.Read(ctx, ref) - if !errors.Is(err, ErrMissingFilePath) { - t.Errorf("expected ErrMissingFilePath, got %v", err) - } - }) -} diff --git a/internal/defaults/defaults.go b/internal/defaults/defaults.go index 892b224..0536c76 100644 --- a/internal/defaults/defaults.go +++ b/internal/defaults/defaults.go @@ -1,39 +1,13 @@ package defaults -import ( - "time" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) +import "time" const ( HTTPAddrDefault = ":8080" SchemaDirDefault = "." - OutputArtifactName = "output" - ContentTypeTextPlain = "text/plain" - ContentTypeTextMarkdown = "text/markdown" - ContentTypeApplicationJSON = "application/json" - OpenAIChatCompletionsPath = "/chat/completions" HTTPMaxRequestBytesDefault = 16 * 1024 * 1024 HTTPMaxArtifactBytesDefault = 16 * 1024 * 1024 HTTPMaxResponseBytesDefault = 16 * 1024 * 1024 - - ExecutionDefaultTemperature = 0.0 - ExecutionDefaultMaxTokens = 0 - ExecutionDefaultTopP = 1.0 - ExecutionDefaultTimeoutSeconds = 600 ) -var ( - LLMRequestTimeoutDefault = 10 * time.Minute - HTTPReadHeaderTimeoutDefault = 10 * time.Second -) - -func ExecutionTargetDefault() domain.ExecutionTarget { - return domain.ExecutionTarget{ - Temperature: ExecutionDefaultTemperature, - MaxTokens: ExecutionDefaultMaxTokens, - TopP: ExecutionDefaultTopP, - TimeoutSeconds: ExecutionDefaultTimeoutSeconds, - } -} +var HTTPReadHeaderTimeoutDefault = 10 * time.Second diff --git a/internal/domain/domain.go b/internal/domain/domain.go deleted file mode 100644 index 0a1e03b..0000000 --- a/internal/domain/domain.go +++ /dev/null @@ -1,288 +0,0 @@ -package domain - -import ( - "time" -) - -// ArtifactRefType defines how an artifact is referenced. -type ArtifactRefType string - -const ( - ArtifactRefInline ArtifactRefType = "inline" - ArtifactRefFile ArtifactRefType = "file" -) - -// OutputFormat defines the desired format of the generated artifact. -type OutputFormat string - -const ( - FormatText OutputFormat = "text" - FormatMarkdown OutputFormat = "markdown" - FormatJSON OutputFormat = "json" -) - -// ValidationMode defines how the output should be validated. -type ValidationMode string - -const ( - ValidationNone ValidationMode = "none" - ValidationBasic ValidationMode = "basic" - ValidationJSON ValidationMode = "json" - ValidationJSONSchema ValidationMode = "json_schema" -) - -// ValidationStatus defines the result of a validation check. -type ValidationStatus string - -const ( - ValidationPassed ValidationStatus = "passed" - ValidationFailed ValidationStatus = "failed" - ValidationSkipped ValidationStatus = "skipped" -) - -// CacheControlType defines provider cache behavior for prompt content. -type CacheControlType string - -const ( - CacheControlEphemeral CacheControlType = "ephemeral" -) - -const ( - // SessionIDMaxLength is OpenRouter's documented maximum session_id length. - SessionIDMaxLength = 256 -) - -// CacheControl describes provider cache metadata attached to prompt content. -type CacheControl struct { - Type CacheControlType `yaml:"type" json:"type"` - TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty"` -} - -// RunRequest represents a request to generate a single artifact. -type RunRequest struct { - PromptID string - PromptVersion string - ProfileID string - APIKey string `json:"-" yaml:"-"` - Inputs map[string]ArtifactRef - Vars map[string]string - Execution *ExecutionTargetOverride - Validation *OutputContract - Metadata map[string]string -} - -// RunResult represents the complete result of a prompt execution run. -type RunResult struct { - RunID string - Artifact Artifact - RawOutput string - Validation ValidationResult - PromptID string - PromptVersion string - PromptHash string - RenderedPromptHash string - SelectedProfileID string - ModelName string - Endpoint string - EffectiveModelParams ExecutionTarget - InputHashes map[string]string - Usage TokenUsage - StartTime time.Time - EndTime time.Time - Duration time.Duration -} - -// PreparedRun contains pre-LLM execution state from the prepare/render phase. -// It must never include resolved API key values, model output, or validation data. -type PreparedRun struct { - PromptID string `json:"prompt_id"` - PromptVersion string `json:"prompt_version,omitempty"` - PromptHash string `json:"prompt_hash,omitempty"` - SelectedProfileID string `json:"selected_profile_id"` - EffectiveModelParams ExecutionTarget `json:"effective_model_params"` - TargetPresence ExecutionTargetPresence `json:"-"` - OutputContract OutputContract `json:"output_contract"` - StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` - InputHashes map[string]string `json:"input_hashes,omitempty"` - SessionID string `json:"session_id,omitempty"` - RenderedPromptHash string `json:"rendered_prompt_hash"` - Messages []RenderedMessage `json:"messages"` - StartTime time.Time `json:"start_time,omitempty"` - EndTime time.Time `json:"end_time,omitempty"` - DurationMS int64 `json:"duration_ms,omitempty"` -} - -// ArtifactRef represents a reference to an input artifact. -type ArtifactRef struct { - Type ArtifactRefType - URI string - Body string // Used for inline -} - -// Artifact represents the actual loaded content of a reference. -type Artifact struct { - Name string - ContentType string - Body []byte - URI string - Size int64 - Hash string -} - -// PromptDefinition represents a configured prompt execution definition. -type PromptDefinition struct { - ID string `yaml:"id"` - Version string `yaml:"version"` - DefaultProfile string `yaml:"default_profile"` - Description string `yaml:"description"` - SessionID string `yaml:"session_id" json:"session_id,omitempty"` - Inputs []PromptInput `yaml:"inputs"` - Templates []PromptMessageTemplate `yaml:"templates"` - OutputFormat OutputFormat `yaml:"output_format"` - Validation OutputContract `yaml:"validation"` -} - -// PromptInput describes one named input expected by a prompt definition. -type PromptInput struct { - Name string `yaml:"name"` - Required bool `yaml:"required"` - ContentType string `yaml:"content_type"` - Description string `yaml:"description"` -} - -// PromptMessageTemplate defines a template for a chat message. -type PromptMessageTemplate struct { - Role string `yaml:"role"` - Content string `yaml:"content"` - ContentFile string `yaml:"content_file"` - CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"` -} - -// ExecutionProfile describes how and where to execute a model. -type ExecutionProfile struct { - ID string `yaml:"id"` - Endpoint string `yaml:"endpoint"` - Model string `yaml:"model"` - Temperature float64 `yaml:"temperature"` - MaxTokens int `yaml:"max_tokens"` - TopP float64 `yaml:"top_p"` - TimeoutSeconds int `yaml:"timeout_seconds"` - ServiceTier string `yaml:"service_tier"` - ReasoningEffort string `yaml:"reasoning_effort"` - APIKeyEnv string `yaml:"api_key_env"` - APIKeyRequired bool `yaml:"-" json:"-"` - ExtraParams map[string]any `yaml:"extra_params"` -} - -// ExecutionTargetOverride represents per-request runtime setting overrides. -type ExecutionTargetOverride struct { - Endpoint string `json:"endpoint,omitempty"` - Model string `json:"model,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - MaxTokens *int `json:"max_tokens,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - TimeoutSeconds *int `json:"timeout_seconds,omitempty"` - ServiceTier string `json:"service_tier,omitempty"` - ReasoningEffort string `json:"reasoning_effort,omitempty"` - APIKeyEnv string `json:"api_key_env,omitempty"` - ExtraParams map[string]any `json:"extra_params,omitempty"` -} - -// ExecutionTargetPresence tracks which effective runtime fields came from an -// explicit request override even when the resolved value is a zero value. -type ExecutionTargetPresence struct { - Temperature bool - MaxTokens bool - TopP bool - TimeoutSeconds bool -} - -// ExecutionTarget represents effective model runtime settings for a run. -type ExecutionTarget struct { - Endpoint string `yaml:"endpoint" json:"endpoint"` - Model string `yaml:"model" json:"model"` - Temperature float64 `yaml:"temperature" json:"temperature"` - MaxTokens int `yaml:"max_tokens" json:"max_tokens"` - TopP float64 `yaml:"top_p" json:"top_p"` - TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"` - ServiceTier string `yaml:"service_tier" json:"service_tier"` - ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"` - APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"` - APIKey string `yaml:"-" json:"-"` - APIKeyRequired bool `yaml:"-" json:"-"` - ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"` -} - -// OutputContract defines the requirements for the output artifact. -type OutputContract struct { - Format OutputFormat `yaml:"format"` - ValidationMode ValidationMode `yaml:"validation_mode"` - SchemaPath string `yaml:"schema_path"` - RepairAttempts int `yaml:"repair_attempts"` -} - -// RenderedPrompt represents the prompt after template application. -type RenderedPrompt struct { - SessionID string `json:"session_id,omitempty"` - Messages []RenderedMessage `json:"messages"` -} - -// RenderedMessage is a single message in a rendered prompt. -type RenderedMessage struct { - Role string `json:"role"` - Content string `json:"content"` - CacheControl *CacheControl `json:"cache_control,omitempty"` -} - -// GenerateRequest is the internal request passed to the LLM client. -type GenerateRequest struct { - Prompt RenderedPrompt - Target ExecutionTarget - TargetPresence ExecutionTargetPresence - StructuredOutput *StructuredOutputSpec -} - -// StructuredOutputType indicates which provider-level output mode is requested. -type StructuredOutputType string - -const ( - StructuredOutputJSONSchema StructuredOutputType = "json_schema" -) - -// StructuredOutputSpec describes provider-level structured output requirements. -type StructuredOutputSpec struct { - Type StructuredOutputType `json:"type"` - JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"` -} - -// StructuredOutputJSONSpec contains json_schema output constraints. -type StructuredOutputJSONSpec struct { - Name string `json:"name"` - Strict bool `json:"strict"` - Schema any `json:"schema"` -} - -// GenerateResponse is the response received from the LLM client. -type GenerateResponse struct { - Content string - Usage TokenUsage -} - -// TokenUsage tracks token consumption. -type TokenUsage struct { - PromptTokens int - CompletionTokens int - TotalTokens int - CachedTokens int - CacheWriteTokens int -} - -// ValidationResult represents the outcome of an output validation. -type ValidationResult struct { - Status ValidationStatus - Mode ValidationMode - Errors []string - SchemaPath string - RepairAttempts int - IsValid bool -} diff --git a/internal/domain/prepared_run_test.go b/internal/domain/prepared_run_test.go deleted file mode 100644 index cf84c1d..0000000 --- a/internal/domain/prepared_run_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package domain - -import ( - "encoding/json" - "strings" - "testing" -) - -func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) { - const envName = "SCRIPTORIUM_TEST_API_KEY" - const secret = "super-secret-value" - t.Setenv(envName, secret) - - prepared := PreparedRun{ - PromptID: "prompt.id", - PromptVersion: "v1", - PromptHash: "prompt-hash", - SelectedProfileID: "local-fast", - EffectiveModelParams: ExecutionTarget{ - Endpoint: "http://llm/v1", - Model: "gpt-test", - APIKeyEnv: envName, - APIKey: secret, - }, - InputHashes: map[string]string{"transcript": "hash-1"}, - RenderedPromptHash: "rendered-hash", - Messages: []RenderedMessage{ - {Role: "system", Content: "You are helpful."}, - {Role: "user", Content: "Summarize this."}, - }, - } - - b, err := json.Marshal(prepared) - if err != nil { - t.Fatalf("marshal failed: %v", err) - } - - out := string(b) - if strings.Contains(out, secret) { - t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out) - } - if !strings.Contains(out, `"api_key_env":"`+envName+`"`) { - t.Fatalf("prepared run JSON should include api_key_env name: %s", out) - } - - var top map[string]any - if err := json.Unmarshal(b, &top); err != nil { - t.Fatalf("unmarshal failed: %v", err) - } - - for _, forbidden := range []string{"raw_output", "validation", "artifact"} { - if _, ok := top[forbidden]; ok { - t.Fatalf("prepared run JSON should not include %q", forbidden) - } - } -} - -func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) { - prepared := PreparedRun{ - PromptID: "prompt.id", - SelectedProfileID: "local-fast", - EffectiveModelParams: ExecutionTarget{ - Endpoint: "http://llm/v1", - Model: "gpt-test", - }, - RenderedPromptHash: "rendered-hash", - Messages: []RenderedMessage{ - { - Role: "system", - Content: "You are helpful.", - CacheControl: &CacheControl{ - Type: CacheControlEphemeral, - TTL: "1h", - }, - }, - {Role: "user", Content: "Summarize this."}, - }, - } - - b, err := json.Marshal(prepared) - if err != nil { - t.Fatalf("marshal failed: %v", err) - } - - var decoded struct { - Messages []map[string]any `json:"messages"` - } - if err := json.Unmarshal(b, &decoded); err != nil { - t.Fatalf("unmarshal failed: %v", err) - } - if len(decoded.Messages) != 2 { - t.Fatalf("expected 2 messages, got %d", len(decoded.Messages)) - } - - cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any) - if !ok { - t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0]) - } - if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" { - t.Fatalf("unexpected cache_control payload: %#v", cacheControl) - } - if _, ok := decoded.Messages[1]["cache_control"]; ok { - t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1]) - } -} - -func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) { - prepared := PreparedRun{ - PromptID: "prompt.id", - SelectedProfileID: "local-fast", - EffectiveModelParams: ExecutionTarget{ - Endpoint: "http://llm/v1", - Model: "gpt-test", - }, - SessionID: "session-123", - RenderedPromptHash: "rendered-hash", - Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}}, - } - - b, err := json.Marshal(prepared) - if err != nil { - t.Fatalf("marshal failed: %v", err) - } - - var decoded map[string]any - if err := json.Unmarshal(b, &decoded); err != nil { - t.Fatalf("unmarshal failed: %v", err) - } - if decoded["session_id"] != "session-123" { - t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"]) - } - - prepared.SessionID = "" - b, err = json.Marshal(prepared) - if err != nil { - t.Fatalf("marshal failed: %v", err) - } - if strings.Contains(string(b), "session_id") { - t.Fatalf("expected empty session_id to be omitted, got %s", b) - } -} diff --git a/internal/filecatalog/catalog.go b/internal/filecatalog/catalog.go deleted file mode 100644 index e0c8369..0000000 --- a/internal/filecatalog/catalog.go +++ /dev/null @@ -1,142 +0,0 @@ -package filecatalog - -import ( - "context" - "fmt" - "io/fs" - "os" - "path" - "path/filepath" - "sort" - "strings" -) - -// FindYAMLFiles returns sorted full paths for .yaml and .yml files under root. -func FindYAMLFiles(ctx context.Context, root string) ([]string, error) { - var files []string - err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if d.IsDir() { - return nil - } - if !IsYAMLFile(d.Name()) { - return nil - } - files = append(files, path) - return nil - }) - sort.Strings(files) - return files, err -} - -// FindFSYAMLFiles returns sorted paths for .yaml and .yml files under root in fsys. -func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) { - cleanRoot := CleanFSRoot(root) - var files []string - err := fs.WalkDir(fsys, cleanRoot, func(name string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if d.IsDir() { - return nil - } - if !IsYAMLFile(d.Name()) { - return nil - } - files = append(files, name) - return nil - }) - sort.Strings(files) - return files, err -} - -// RelativePath computes a clean relative path from root to path. -func RelativePath(root string, filePath string) string { - rel, err := filepath.Rel(root, filePath) - if err != nil { - return filepath.Clean(filePath) - } - return filepath.Clean(rel) -} - -// CleanFSRoot normalizes a root path for use with fs.FS. -func CleanFSRoot(root string) string { - root = strings.TrimSpace(root) - if root == "" || root == "." { - return "." - } - return path.Clean(root) -} - -// DisplayPath returns name relative to root for messages about fs.FS paths. -func DisplayPath(root string, name string) string { - cleanRoot := CleanFSRoot(root) - cleanName := path.Clean(name) - if cleanRoot == "." { - return cleanName - } - prefix := strings.TrimSuffix(cleanRoot, "/") + "/" - if strings.HasPrefix(cleanName, prefix) { - return strings.TrimPrefix(cleanName, prefix) - } - return cleanName -} - -// ResolveFSPath resolves userPath from baseDir and keeps it inside root. -func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) { - cleanRoot := CleanFSRoot(root) - cleanBase := path.Clean(strings.TrimSpace(baseDir)) - if cleanBase == "" { - cleanBase = cleanRoot - } - if !containsFSPath(cleanRoot, cleanBase) { - return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot) - } - - cleanUserPath := strings.TrimSpace(userPath) - if cleanUserPath == "" { - return "", "", fmt.Errorf("path is required") - } - cleanUserPath = path.Clean(cleanUserPath) - if path.IsAbs(cleanUserPath) { - return "", "", fmt.Errorf("path %q must be relative", userPath) - } - - resolved := path.Clean(path.Join(cleanBase, cleanUserPath)) - if !containsFSPath(cleanRoot, resolved) { - return "", "", fmt.Errorf("path %q escapes source root %q", userPath, cleanRoot) - } - return resolved, DisplayPath(cleanRoot, resolved), nil -} - -func containsFSPath(root string, name string) bool { - root = CleanFSRoot(root) - name = path.Clean(name) - if root == "." { - return name == "." || (name != ".." && !strings.HasPrefix(name, "../")) - } - return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/") -} - -// Stem strips .yaml or .yml from a file name. -func Stem(name string) string { - name = strings.TrimSuffix(name, ".yaml") - name = strings.TrimSuffix(name, ".yml") - return name -} - -func IsYAMLFile(name string) bool { - return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml") -} diff --git a/internal/filecatalog/catalog_test.go b/internal/filecatalog/catalog_test.go deleted file mode 100644 index 6a2a2cc..0000000 --- a/internal/filecatalog/catalog_test.go +++ /dev/null @@ -1,270 +0,0 @@ -package filecatalog - -import ( - "context" - "errors" - "os" - "path/filepath" - "reflect" - "strings" - "testing" - "testing/fstest" -) - -func TestFindYAMLFilesNestedSortedAndFiltered(t *testing.T) { - root := t.TempDir() - mustWriteFile(t, filepath.Join(root, "z", "prompt.yml"), "id: z") - mustWriteFile(t, filepath.Join(root, "a", "profile.yaml"), "id: a") - mustWriteFile(t, filepath.Join(root, "a", "ignore.txt"), "not yaml") - mustWriteFile(t, filepath.Join(root, "b", "ignore.yaml.bak"), "not yaml") - - got, err := FindYAMLFiles(context.Background(), root) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - want := []string{ - filepath.Join(root, "a", "profile.yaml"), - filepath.Join(root, "z", "prompt.yml"), - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("expected sorted YAML files %v, got %v", want, got) - } -} - -func TestFindYAMLFilesHonorsContextCancellation(t *testing.T) { - root := t.TempDir() - mustWriteFile(t, filepath.Join(root, "one.yaml"), "id: one") - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - _, err := FindYAMLFiles(ctx, root) - if !errors.Is(err, context.Canceled) { - t.Fatalf("expected context.Canceled, got %v", err) - } -} - -func TestFindFSYAMLFilesNestedSortedAndFiltered(t *testing.T) { - fsys := fstest.MapFS{ - "prompts/z/prompt.yml": &fstest.MapFile{Data: []byte("id: z")}, - "prompts/a/profile.yaml": &fstest.MapFile{Data: []byte("id: a")}, - "prompts/a/ignore.txt": &fstest.MapFile{Data: []byte("not yaml")}, - "prompts/b/ignore.yaml.bak": &fstest.MapFile{Data: []byte("not yaml")}, - "other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")}, - } - - got, err := FindFSYAMLFiles(context.Background(), fsys, " prompts ") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - want := []string{ - "prompts/a/profile.yaml", - "prompts/z/prompt.yml", - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("expected sorted YAML files %v, got %v", want, got) - } -} - -func TestFindFSYAMLFilesHonorsContextCancellation(t *testing.T) { - fsys := fstest.MapFS{ - "one.yaml": &fstest.MapFile{Data: []byte("id: one")}, - } - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - _, err := FindFSYAMLFiles(ctx, fsys, ".") - if !errors.Is(err, context.Canceled) { - t.Fatalf("expected context.Canceled, got %v", err) - } -} - -func TestRelativePathNested(t *testing.T) { - root := t.TempDir() - path := filepath.Join(root, "nested", "profiles", "local.yaml") - got := RelativePath(root, path) - want := filepath.Join("nested", "profiles", "local.yaml") - if got != want { - t.Fatalf("expected relative path %q, got %q", want, got) - } -} - -func TestCleanFSRoot(t *testing.T) { - tests := []struct { - name string - root string - want string - }{ - {name: "empty", root: "", want: "."}, - {name: "dot", root: ".", want: "."}, - {name: "trimmed", root: " prompts/../profiles ", want: "profiles"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := CleanFSRoot(tc.root); got != tc.want { - t.Fatalf("expected %q, got %q", tc.want, got) - } - }) - } -} - -func TestDisplayPath(t *testing.T) { - tests := []struct { - name string - root string - path string - want string - }{ - {name: "root dot", root: ".", path: "profiles/local.yaml", want: "profiles/local.yaml"}, - {name: "nested root", root: "profiles", path: "profiles/local.yaml", want: "local.yaml"}, - {name: "outside root", root: "profiles", path: "other/local.yaml", want: "other/local.yaml"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := DisplayPath(tc.root, tc.path); got != tc.want { - t.Fatalf("expected %q, got %q", tc.want, got) - } - }) - } -} - -func TestResolveFSPath(t *testing.T) { - tests := []struct { - name string - root string - baseDir string - userPath string - wantPath string - wantDisplay string - wantErr string - }{ - { - name: "sibling inside root", - root: "prompts", - baseDir: "prompts/nested", - userPath: "./messages/user.tmpl", - wantPath: "prompts/nested/messages/user.tmpl", - wantDisplay: "nested/messages/user.tmpl", - }, - { - name: "parent inside root", - root: "prompts", - baseDir: "prompts/nested", - userPath: "../shared/user.tmpl", - wantPath: "prompts/shared/user.tmpl", - wantDisplay: "shared/user.tmpl", - }, - { - name: "escape rejected", - root: "prompts", - baseDir: "prompts/nested", - userPath: "../../outside.tmpl", - wantErr: "escapes source root", - }, - { - name: "absolute path rejected", - root: "prompts", - baseDir: "prompts/nested", - userPath: "/outside.tmpl", - wantErr: "must be relative", - }, - { - name: "empty path rejected", - root: "prompts", - baseDir: "prompts/nested", - userPath: " ", - wantErr: "path is required", - }, - { - name: "dot root allows normal relative path", - root: ".", - baseDir: ".", - userPath: "schemas/events.schema.json", - wantPath: "schemas/events.schema.json", - wantDisplay: "schemas/events.schema.json", - }, - { - name: "dot root rejects parent escape", - root: ".", - baseDir: ".", - userPath: "../outside.tmpl", - wantErr: "escapes source root", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - gotPath, gotDisplay, err := ResolveFSPath(tc.root, tc.baseDir, tc.userPath) - if tc.wantErr != "" { - if err == nil { - t.Fatalf("expected error containing %q", tc.wantErr) - } - if !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err) - } - return - } - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if gotPath != tc.wantPath || gotDisplay != tc.wantDisplay { - t.Fatalf("expected path/display %q/%q, got %q/%q", tc.wantPath, tc.wantDisplay, gotPath, gotDisplay) - } - }) - } -} - -func TestStemStripsYAMLExtensions(t *testing.T) { - tests := []struct { - name string - in string - want string - }{ - {name: "yaml", in: "prompt.yaml", want: "prompt"}, - {name: "yml", in: "profile.yml", want: "profile"}, - {name: "other", in: "file.txt", want: "file.txt"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := Stem(tc.in); got != tc.want { - t.Fatalf("expected %q, got %q", tc.want, got) - } - }) - } -} - -func TestIsYAMLFile(t *testing.T) { - tests := []struct { - name string - in string - want bool - }{ - {name: "yaml", in: "prompt.yaml", want: true}, - {name: "yml", in: "profile.yml", want: true}, - {name: "backup", in: "profile.yaml.bak", want: false}, - {name: "uppercase", in: "profile.YAML", want: false}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := IsYAMLFile(tc.in); got != tc.want { - t.Fatalf("expected %v, got %v", tc.want, got) - } - }) - } -} - -func mustWriteFile(t *testing.T, path string, content string) { - t.Helper() - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - t.Fatalf("failed to create directory: %v", err) - } - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatalf("failed to write file %q: %v", path, err) - } -} diff --git a/internal/llm/client.go b/internal/llm/client.go deleted file mode 100644 index a1f799f..0000000 --- a/internal/llm/client.go +++ /dev/null @@ -1,11 +0,0 @@ -package llm - -import ( - "context" - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -// Client executes a rendered prompt against an LLM endpoint. -type Client interface { - Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) -} diff --git a/internal/llm/openai_compatible_client.go b/internal/llm/openai_compatible_client.go deleted file mode 100644 index af96d99..0000000 --- a/internal/llm/openai_compatible_client.go +++ /dev/null @@ -1,385 +0,0 @@ -package llm - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os" - "strings" - "time" - "unicode/utf8" - - "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -var ( - ErrInvalidConfig = errors.New("invalid llm client configuration") - ErrInvalidRequest = errors.New("invalid generate request") - ErrRequestFailed = errors.New("llm request failed") - ErrUnexpectedStatus = errors.New("llm returned non-success status") - ErrMalformedResponse = errors.New("malformed llm response") -) - -type OpenAICompatibleConfig struct { - BaseURL string - Model string - Timeout time.Duration - HTTPClient *http.Client -} - -type OpenAICompatibleClient struct { - baseURL string - defaultModel string - httpClient *http.Client -} - -func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) { - baseURL := strings.TrimSpace(cfg.BaseURL) - if baseURL != "" { - if _, err := url.ParseRequestURI(baseURL); err != nil { - return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err) - } - } - - timeout := cfg.Timeout - if timeout <= 0 { - timeout = defaults.LLMRequestTimeoutDefault - } - - var client *http.Client - if cfg.HTTPClient != nil { - cloned := *cfg.HTTPClient - if cloned.Timeout <= 0 { - cloned.Timeout = timeout - } - client = &cloned - } else { - client = &http.Client{Timeout: timeout} - } - - return &OpenAICompatibleClient{ - baseURL: strings.TrimRight(baseURL, "/"), - defaultModel: cfg.Model, - httpClient: client, - }, nil -} - -func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { - if req.Target.TimeoutSeconds < 0 { - return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest) - } - - endpoint := strings.TrimSpace(req.Target.Endpoint) - if endpoint == "" { - endpoint = c.baseURL - } - if endpoint == "" { - return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest) - } - endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath - - wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) - } - - wirePayload, err := openAIChatRequestPayload(wireReq) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) - } - - payload, err := json.Marshal(wirePayload) - if err != nil { - return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err) - } - - requestContext := ctx - if req.Target.TimeoutSeconds > 0 { - var cancel context.CancelFunc - requestContext, cancel = context.WithTimeout( - ctx, - time.Duration(req.Target.TimeoutSeconds)*time.Second, - ) - defer cancel() - } - - httpReq, err := http.NewRequestWithContext(requestContext, http.MethodPost, endpoint, bytes.NewReader(payload)) - if err != nil { - return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err) - } - httpReq.Header.Set("Content-Type", "application/json") - if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" { - httpReq.Header.Set("Authorization", "Bearer "+apiKey) - } else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" { - apiKey := strings.TrimSpace(os.Getenv(envName)) - if apiKey == "" { - return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName) - } - httpReq.Header.Set("Authorization", "Bearer "+apiKey) - } - - httpClient := c.httpClient - if httpClient == nil { - httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault} - } - - httpResp, err := httpClient.Do(httpReq) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrRequestFailed, err) - } - defer httpResp.Body.Close() - - if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { - _, _ = io.Copy(io.Discard, io.LimitReader(httpResp.Body, 4096)) - return nil, fmt.Errorf("%w: status=%d", ErrUnexpectedStatus, httpResp.StatusCode) - } - - var wireResp openAIChatResponse - if err := json.NewDecoder(httpResp.Body).Decode(&wireResp); err != nil { - return nil, fmt.Errorf("%w: failed to decode response: %v", ErrMalformedResponse, err) - } - - if len(wireResp.Choices) == 0 { - return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse) - } - content := wireResp.Choices[0].Message.Content - if content == "" { - return nil, fmt.Errorf("%w: first choice has empty message content", ErrMalformedResponse) - } - - return &domain.GenerateResponse{ - Content: content, - Usage: domain.TokenUsage{ - PromptTokens: wireResp.Usage.PromptTokens, - CompletionTokens: wireResp.Usage.CompletionTokens, - TotalTokens: wireResp.Usage.TotalTokens, - CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens, - CacheWriteTokens: wireResp.Usage.CacheWriteTokens, - }, - }, nil -} - -func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) { - model := strings.TrimSpace(req.Target.Model) - if model == "" { - model = strings.TrimSpace(defaultModel) - } - if model == "" { - return openAIChatRequest{}, errors.New("model is required") - } - - wireReq := openAIChatRequest{ - Model: model, - } - if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" { - if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength { - return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength) - } - wireReq.SessionID = sessionID - } - - wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages)) - for _, msg := range req.Prompt.Messages { - wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg)) - } - - if req.Target.Temperature != 0 || req.TargetPresence.Temperature { - wireReq.Temperature = &req.Target.Temperature - } - if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens { - wireReq.MaxTokens = &req.Target.MaxTokens - } - if req.Target.TopP != 0 || req.TargetPresence.TopP { - wireReq.TopP = &req.Target.TopP - } - if strings.TrimSpace(req.Target.ServiceTier) != "" { - wireReq.ServiceTier = req.Target.ServiceTier - } - if strings.TrimSpace(req.Target.ReasoningEffort) != "" { - wireReq.ReasoningEffort = req.Target.ReasoningEffort - } - if len(req.Target.ExtraParams) > 0 { - wireReq.ExtraParams = req.Target.ExtraParams - } - if req.StructuredOutput != nil { - responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput) - if err != nil { - return openAIChatRequest{}, err - } - wireReq.ResponseFormat = responseFormat - } - - return wireReq, nil -} - -type openAIChatRequest struct { - Model string `json:"model"` - SessionID string `json:"session_id,omitempty"` - Messages []openAIChatRequestMessage `json:"messages"` - Temperature *float64 `json:"temperature,omitempty"` - MaxTokens *int `json:"max_tokens,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - ServiceTier string `json:"service_tier,omitempty"` - ReasoningEffort string `json:"reasoning_effort,omitempty"` - ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"` - ExtraParams map[string]any `json:"-"` -} - -func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) { - out := map[string]any{ - "model": req.Model, - "messages": req.Messages, - } - if req.SessionID != "" { - out["session_id"] = req.SessionID - } - if req.Temperature != nil { - out["temperature"] = *req.Temperature - } - if req.MaxTokens != nil { - out["max_tokens"] = *req.MaxTokens - } - if req.TopP != nil { - out["top_p"] = *req.TopP - } - if req.ServiceTier != "" { - out["service_tier"] = req.ServiceTier - } - if req.ReasoningEffort != "" { - out["reasoning_effort"] = req.ReasoningEffort - } - if req.ResponseFormat != nil { - out["response_format"] = req.ResponseFormat - } - - for key, value := range req.ExtraParams { - if key == "" { - return nil, errors.New("extra_params key must not be empty") - } - if _, reserved := reservedOpenAIChatRequestFields[key]; reserved { - return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key) - } - if _, err := json.Marshal(value); err != nil { - return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err) - } - out[key] = value - } - - return out, nil -} - -var reservedOpenAIChatRequestFields = map[string]struct{}{ - "model": {}, - "session_id": {}, - "messages": {}, - "temperature": {}, - "max_tokens": {}, - "top_p": {}, - "service_tier": {}, - "reasoning_effort": {}, - "response_format": {}, -} - -type openAIChatRequestMessage struct { - Role string `json:"role"` - Content any `json:"content"` -} - -type openAIChatTextContentBlock struct { - Type string `json:"type"` - Text string `json:"text"` - CacheControl *openAICacheControl `json:"cache_control,omitempty"` -} - -type openAICacheControl struct { - Type string `json:"type"` - TTL string `json:"ttl,omitempty"` -} - -type openAIChatResponseMessage struct { - Role string `json:"role"` - Content string `json:"content"` -} - -type openAIChatResponse struct { - Choices []struct { - Message openAIChatResponseMessage `json:"message"` - } `json:"choices"` - Usage struct { - PromptTokens int `json:"prompt_tokens"` - CompletionTokens int `json:"completion_tokens"` - TotalTokens int `json:"total_tokens"` - PromptTokensDetails struct { - CachedTokens int `json:"cached_tokens"` - } `json:"prompt_tokens_details"` - CacheWriteTokens int `json:"cache_write_tokens"` - } `json:"usage"` -} - -type openAIResponseFormat struct { - Type string `json:"type"` - JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"` -} - -type openAIJSONSchemaEnvelope struct { - Name string `json:"name"` - Strict bool `json:"strict"` - Schema any `json:"schema"` -} - -func openAIChatRequestMessageFromRenderedMessage(msg domain.RenderedMessage) openAIChatRequestMessage { - wireMsg := openAIChatRequestMessage{ - Role: msg.Role, - Content: msg.Content, - } - if msg.CacheControl == nil { - return wireMsg - } - - wireMsg.Content = []openAIChatTextContentBlock{ - { - Type: "text", - Text: msg.Content, - CacheControl: &openAICacheControl{ - Type: string(msg.CacheControl.Type), - TTL: msg.CacheControl.TTL, - }, - }, - } - return wireMsg -} - -func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) { - if spec == nil { - return nil, nil - } - - switch spec.Type { - case domain.StructuredOutputJSONSchema: - if spec.JSONSchema == nil { - return nil, errors.New("json_schema structured output requires schema payload") - } - if strings.TrimSpace(spec.JSONSchema.Name) == "" { - return nil, errors.New("json_schema structured output requires non-empty schema name") - } - if spec.JSONSchema.Schema == nil { - return nil, errors.New("json_schema structured output requires schema document") - } - return &openAIResponseFormat{ - Type: "json_schema", - JSONSchema: &openAIJSONSchemaEnvelope{ - Name: spec.JSONSchema.Name, - Strict: spec.JSONSchema.Strict, - Schema: spec.JSONSchema.Schema, - }, - }, nil - default: - return nil, fmt.Errorf("unsupported structured output type %q", spec.Type) - } -} diff --git a/internal/llm/openai_compatible_client_test.go b/internal/llm/openai_compatible_client_test.go deleted file mode 100644 index 1a32bc9..0000000 --- a/internal/llm/openai_compatible_client_test.go +++ /dev/null @@ -1,1069 +0,0 @@ -package llm - -import ( - "context" - "encoding/json" - "errors" - "math" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) { - transport := http.DefaultTransport - supplied := &http.Client{Transport: transport} - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ - HTTPClient: supplied, - }) - if err != nil { - t.Fatalf("unexpected constructor error: %v", err) - } - - if supplied.Timeout != 0 { - t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout) - } - if client.httpClient == supplied { - t.Fatal("expected constructed client to use a cloned HTTP client") - } - if client.httpClient.Timeout <= 0 { - t.Fatalf("expected constructed client to use a positive default timeout, got %v", client.httpClient.Timeout) - } - if client.httpClient.Transport != transport { - t.Fatal("expected cloned client to preserve the supplied transport") - } -} - -func TestNewOpenAICompatibleClientDoesNotMutateSuppliedNonzeroTimeoutClient(t *testing.T) { - transport := http.DefaultTransport - suppliedTimeout := 37 * time.Second - supplied := &http.Client{ - Timeout: suppliedTimeout, - Transport: transport, - } - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ - Timeout: 2 * time.Second, - HTTPClient: supplied, - }) - if err != nil { - t.Fatalf("unexpected constructor error: %v", err) - } - - if supplied.Timeout != suppliedTimeout { - t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout) - } - if client.httpClient == supplied { - t.Fatal("expected constructed client to use a cloned HTTP client") - } - if client.httpClient.Timeout != suppliedTimeout { - t.Fatalf("expected cloned client timeout %v, got %v", suppliedTimeout, client.httpClient.Timeout) - } - if client.httpClient.Transport != transport { - t.Fatal("expected cloned client to preserve the supplied transport") - } -} - -func TestNewOpenAICompatibleClientTreatsSuppliedNegativeTimeoutAsUnset(t *testing.T) { - transport := http.DefaultTransport - supplied := &http.Client{ - Timeout: -time.Second, - Transport: transport, - } - configuredTimeout := 23 * time.Second - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ - Timeout: configuredTimeout, - HTTPClient: supplied, - }) - if err != nil { - t.Fatalf("unexpected constructor error: %v", err) - } - - if supplied.Timeout != -time.Second { - t.Fatalf("expected supplied client timeout to remain negative, got %v", supplied.Timeout) - } - if client.httpClient == supplied { - t.Fatal("expected constructed client to use a cloned HTTP client") - } - if client.httpClient.Timeout != configuredTimeout { - t.Fatalf("expected cloned client timeout %v, got %v", configuredTimeout, client.httpClient.Timeout) - } - if client.httpClient.Transport != transport { - t.Fatal("expected cloned client to preserve the supplied transport") - } -} - -func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) { - type observedRequest struct { - Authorization string - Body map[string]any - } - obs := &observedRequest{} - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - obs.Authorization = r.Header.Get("Authorization") - if r.URL.Path != "/v1/chat/completions" { - t.Fatalf("unexpected path: %s", r.URL.Path) - } - if ct := r.Header.Get("Content-Type"); ct != "application/json" { - t.Fatalf("unexpected content type: %s", ct) - } - - defer r.Body.Close() - if err := json.NewDecoder(r.Body).Decode(&obs.Body); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "choices": [{"message": {"role": "assistant", "content": "hello from model"}}], - "usage": {"prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33} -}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ - BaseURL: ts.URL + "/v1", - Timeout: 2 * time.Second, - }) - if err != nil { - t.Fatalf("unexpected constructor error: %v", err) - } - t.Setenv("SCRIPTORIUM_TEST_API_KEY", "secret-key") - - resp, err := client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{ - {Role: "system", Content: "You are helpful."}, - {Role: "user", Content: "Say hello"}, - }}, - Target: domain.ExecutionTarget{ - Model: "gpt-test", - Temperature: 0.4, - MaxTokens: 123, - TopP: 0.7, - ServiceTier: "priority", - APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY", - }, - StructuredOutput: &domain.StructuredOutputSpec{ - Type: domain.StructuredOutputJSONSchema, - JSONSchema: &domain.StructuredOutputJSONSpec{ - Name: "weather_schema", - Strict: true, - Schema: map[string]any{ - "type": "object", - "properties": map[string]any{ - "location": map[string]any{"type": "string"}, - }, - "required": []any{"location"}, - }, - }, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - if resp.Content != "hello from model" { - t.Fatalf("unexpected content: %q", resp.Content) - } - if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 { - t.Fatalf("unexpected usage: %+v", resp.Usage) - } - if resp.Usage.CachedTokens != 0 || resp.Usage.CacheWriteTokens != 0 { - t.Fatalf("expected absent cache usage fields to remain zero, got %+v", resp.Usage) - } - - if obs.Authorization != "Bearer secret-key" { - t.Fatalf("unexpected Authorization header: %q", obs.Authorization) - } - if got, ok := obs.Body["model"].(string); !ok || got != "gpt-test" { - t.Fatalf("unexpected model payload: %#v", obs.Body["model"]) - } - if got, ok := obs.Body["temperature"].(float64); !ok || got != 0.4 { - t.Fatalf("unexpected temperature payload: %#v", obs.Body["temperature"]) - } - if got, ok := obs.Body["max_tokens"].(float64); !ok || got != 123 { - t.Fatalf("unexpected max_tokens payload: %#v", obs.Body["max_tokens"]) - } - if got, ok := obs.Body["top_p"].(float64); !ok || got != 0.7 { - t.Fatalf("unexpected top_p payload: %#v", obs.Body["top_p"]) - } - if got, ok := obs.Body["service_tier"].(string); !ok || got != "priority" { - t.Fatalf("unexpected service_tier payload: %#v", obs.Body["service_tier"]) - } - - msgs, ok := obs.Body["messages"].([]any) - if !ok || len(msgs) != 2 { - t.Fatalf("unexpected messages payload: %#v", obs.Body["messages"]) - } - msg0 := msgs[0].(map[string]any) - if msg0["role"] != "system" || msg0["content"] != "You are helpful." { - t.Fatalf("unexpected first message: %#v", msg0) - } - msg1 := msgs[1].(map[string]any) - if msg1["role"] != "user" || msg1["content"] != "Say hello" { - t.Fatalf("unexpected second message: %#v", msg1) - } - - responseFormat, ok := obs.Body["response_format"].(map[string]any) - if !ok { - t.Fatalf("expected response_format payload, got %#v", obs.Body["response_format"]) - } - if responseFormat["type"] != "json_schema" { - t.Fatalf("expected response_format.type=json_schema, got %#v", responseFormat["type"]) - } - jsonSchema, ok := responseFormat["json_schema"].(map[string]any) - if !ok { - t.Fatalf("expected response_format.json_schema map, got %#v", responseFormat["json_schema"]) - } - if jsonSchema["name"] != "weather_schema" { - t.Fatalf("expected json_schema.name weather_schema, got %#v", jsonSchema["name"]) - } - if jsonSchema["strict"] != true { - t.Fatalf("expected json_schema.strict=true, got %#v", jsonSchema["strict"]) - } - if _, ok := jsonSchema["schema"].(map[string]any); !ok { - t.Fatalf("expected json_schema.schema object, got %#v", jsonSchema["schema"]) - } -} - -func TestOpenAICompatibleClientDirectAPIKeyPreferredOverEnv(t *testing.T) { - const directKey = "direct-llm-key" - t.Setenv("SCRIPTORIUM_TEST_API_KEY", "env-key") - - var gotAuth string - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotAuth = r.Header.Get("Authorization") - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{ - Model: "model", - APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY", - APIKey: directKey, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if gotAuth != "Bearer "+directKey { - t.Fatalf("unexpected Authorization header: %q", gotAuth) - } -} - -func TestOpenAICompatibleClientSerializesCacheControlledMessageAsContentBlock(t *testing.T) { - var observedBody map[string]any - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{ - { - Role: "system", - Content: "Stable instructions.", - CacheControl: &domain.CacheControl{ - Type: domain.CacheControlEphemeral, - TTL: "1h", - }, - }, - {Role: "user", Content: "Dynamic request."}, - }}, - Target: domain.ExecutionTarget{Model: "model"}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - for _, forbidden := range []string{"cache_control", "extra_params"} { - if _, exists := observedBody[forbidden]; exists { - t.Fatalf("expected top-level %s to be omitted, got %#v", forbidden, observedBody[forbidden]) - } - } - - msgs, ok := observedBody["messages"].([]any) - if !ok || len(msgs) != 2 { - t.Fatalf("unexpected messages payload: %#v", observedBody["messages"]) - } - msg0 := msgs[0].(map[string]any) - if msg0["role"] != "system" { - t.Fatalf("unexpected first message role: %#v", msg0["role"]) - } - contentBlocks, ok := msg0["content"].([]any) - if !ok || len(contentBlocks) != 1 { - t.Fatalf("expected first message content block array, got %#v", msg0["content"]) - } - block := contentBlocks[0].(map[string]any) - if block["type"] != "text" || block["text"] != "Stable instructions." { - t.Fatalf("unexpected text content block: %#v", block) - } - cacheControl, ok := block["cache_control"].(map[string]any) - if !ok { - t.Fatalf("expected cache_control on content block, got %#v", block) - } - if cacheControl["type"] != string(domain.CacheControlEphemeral) || cacheControl["ttl"] != "1h" { - t.Fatalf("unexpected cache_control payload: %#v", cacheControl) - } - - msg1 := msgs[1].(map[string]any) - if msg1["role"] != "user" || msg1["content"] != "Dynamic request." { - t.Fatalf("expected uncached message to keep string content, got %#v", msg1) - } -} - -func TestOpenAICompatibleClientOmitsEmptyCacheControlTTL(t *testing.T) { - var observedBody map[string]any - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{ - { - Role: "system", - Content: "Stable instructions.", - CacheControl: &domain.CacheControl{ - Type: domain.CacheControlEphemeral, - }, - }, - }}, - Target: domain.ExecutionTarget{Model: "model"}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - msgs := observedBody["messages"].([]any) - msg0 := msgs[0].(map[string]any) - contentBlocks := msg0["content"].([]any) - block := contentBlocks[0].(map[string]any) - cacheControl := block["cache_control"].(map[string]any) - if cacheControl["type"] != string(domain.CacheControlEphemeral) { - t.Fatalf("unexpected cache_control type: %#v", cacheControl) - } - if _, exists := cacheControl["ttl"]; exists { - t.Fatalf("expected empty ttl to be omitted, got %#v", cacheControl) - } -} - -func TestOpenAICompatibleClientSerializesSessionID(t *testing.T) { - var observedBody map[string]any - var observedSessionHeader string - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - observedSessionHeader = r.Header.Get("x-session-id") - defer r.Body.Close() - if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{ - SessionID: " session-123 ", - Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}, - }, - Target: domain.ExecutionTarget{Model: "model"}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if observedBody["session_id"] != "session-123" { - t.Fatalf("expected top-level session_id, got %#v", observedBody["session_id"]) - } - if observedSessionHeader != "" { - t.Fatalf("did not expect x-session-id header, got %q", observedSessionHeader) - } -} - -func TestOpenAICompatibleClientOmitsEmptySessionID(t *testing.T) { - var observedBody map[string]any - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{ - SessionID: " ", - Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}, - }, - Target: domain.ExecutionTarget{Model: "model"}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if _, exists := observedBody["session_id"]; exists { - t.Fatalf("expected empty session_id to be omitted, got %#v", observedBody["session_id"]) - } -} - -func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) { - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ - BaseURL: "http://example.com/v1", - Model: "model", - }) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{ - SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1), - Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}, - }, - }) - if err == nil { - t.Fatal("expected invalid request error") - } - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } -} - -func TestOpenAICompatibleClientParsesCacheUsage(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{ - "choices": [{"message": {"role": "assistant", "content": "ok"}}], - "usage": { - "prompt_tokens": 100, - "completion_tokens": 20, - "total_tokens": 120, - "prompt_tokens_details": {"cached_tokens": 80}, - "cache_write_tokens": 60 - } -}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"}) - if err != nil { - t.Fatal(err) - } - - resp, err := client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if resp.Usage.PromptTokens != 100 || resp.Usage.CompletionTokens != 20 || resp.Usage.TotalTokens != 120 { - t.Fatalf("unexpected base usage fields: %+v", resp.Usage) - } - if resp.Usage.CachedTokens != 80 || resp.Usage.CacheWriteTokens != 60 { - t.Fatalf("unexpected cache usage fields: %+v", resp.Usage) - } -} - -func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) { - var observedBody map[string]any - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{Model: "model"}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if _, exists := observedBody["response_format"]; exists { - t.Fatalf("expected response_format omitted, got %#v", observedBody["response_format"]) - } - if _, exists := observedBody["service_tier"]; exists { - t.Fatalf("expected service_tier omitted, got %#v", observedBody["service_tier"]) - } -} - -func TestOpenAICompatibleClientSerializesReasoningEffortAndExtraParams(t *testing.T) { - var observedBody map[string]any - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{ - Model: "model", - ReasoningEffort: "high", - ExtraParams: map[string]any{ - "string_value": "on", - "number_value": 42, - "boolean_value": true, - "object_value": map[string]any{"nested": "value", "count": 2}, - "array_value": []any{"first", 3, false}, - }, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if observedBody["reasoning_effort"] != "high" { - t.Fatalf("expected reasoning_effort high, got %#v", observedBody["reasoning_effort"]) - } - if observedBody["string_value"] != "on" { - t.Fatalf("unexpected string extra param: %#v", observedBody["string_value"]) - } - if observedBody["number_value"] != float64(42) { - t.Fatalf("unexpected number extra param: %#v", observedBody["number_value"]) - } - if observedBody["boolean_value"] != true { - t.Fatalf("unexpected boolean extra param: %#v", observedBody["boolean_value"]) - } - objectValue, ok := observedBody["object_value"].(map[string]any) - if !ok || objectValue["nested"] != "value" || objectValue["count"] != float64(2) { - t.Fatalf("unexpected object extra param: %#v", observedBody["object_value"]) - } - if _, exists := observedBody["extra_params"]; exists { - t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"]) - } - arrayValue, ok := observedBody["array_value"].([]any) - if !ok || len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false { - t.Fatalf("unexpected array extra param: %#v", observedBody["array_value"]) - } -} - -func TestOpenAICompatibleClientOmitsReasoningEffortWhenUnset(t *testing.T) { - var observedBody map[string]any - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{Model: "model"}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if _, exists := observedBody["reasoning_effort"]; exists { - t.Fatalf("expected reasoning_effort omitted, got %#v", observedBody["reasoning_effort"]) - } - if _, exists := observedBody["extra_params"]; exists { - t.Fatalf("expected extra_params wrapper omitted, got %#v", observedBody["extra_params"]) - } -} - -func TestOpenAICompatibleClientSerializesExplicitZeroNumericOverrides(t *testing.T) { - var observedBody map[string]any - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{Model: "model"}, - TargetPresence: domain.ExecutionTargetPresence{ - Temperature: true, - MaxTokens: true, - TopP: true, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if observedBody["temperature"] != float64(0) { - t.Fatalf("expected explicit zero temperature, got %#v", observedBody["temperature"]) - } - if observedBody["max_tokens"] != float64(0) { - t.Fatalf("expected explicit zero max_tokens, got %#v", observedBody["max_tokens"]) - } - if observedBody["top_p"] != float64(0) { - t.Fatalf("expected explicit zero top_p, got %#v", observedBody["top_p"]) - } -} - -func TestOpenAICompatibleClientOmitsImplicitZeroNumericFields(t *testing.T) { - var observedBody map[string]any - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{Model: "model"}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - for _, field := range []string{"temperature", "max_tokens", "top_p"} { - if _, exists := observedBody[field]; exists { - t.Fatalf("expected implicit zero field %q to be omitted, got body %#v", field, observedBody) - } - } -} - -func TestOpenAICompatibleClientOmittedTimeoutUsesClientTimeout(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(20 * time.Millisecond) - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ - BaseURL: ts.URL + "/v1", - Timeout: time.Nanosecond, - }) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0}, - }) - if err == nil { - t.Fatal("expected omitted timeout to use client timeout") - } - if !errors.Is(err, ErrRequestFailed) { - t.Fatalf("expected ErrRequestFailed, got %v", err) - } -} - -func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *testing.T) { - tests := []struct { - name string - extraParams map[string]any - want string - }{ - {name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"}, - {name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"}, - } - for _, key := range []string{ - "model", - "session_id", - "messages", - "temperature", - "max_tokens", - "top_p", - "service_tier", - "reasoning_effort", - "response_format", - } { - tests = append(tests, struct { - name string - extraParams map[string]any - want string - }{ - name: "reserved key " + key, - extraParams: map[string]any{key: "collision"}, - want: "reserved request field", - }) - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - called := false - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - called = true - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams}, - }) - if err == nil { - t.Fatal("expected invalid request error") - } - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if !strings.Contains(err.Error(), tc.want) { - t.Fatalf("expected error to contain %q, got %v", tc.want, err) - } - if called { - t.Fatal("provider should not be called for invalid extra_params") - } - }) - } -} - -func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) { - hadAuth := false - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - hadAuth = r.Header.Get("Authorization") != "" - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{Model: "model"}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if hadAuth { - t.Fatal("did not expect Authorization header") - } -} - -func TestOpenAICompatibleClientAPIKeyEnvMissing(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"}, - }) - if err == nil { - t.Fatal("expected missing API key env error") - } - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } -} - -func TestOpenAICompatibleClientModelFallbackFromConfig(t *testing.T) { - gotModel := "" - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var body map[string]any - _ = json.NewDecoder(r.Body).Decode(&body) - if m, ok := body["model"].(string); ok { - gotModel = m - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "default-model"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if gotModel != "default-model" { - t.Fatalf("expected default model, got %q", gotModel) - } -} - -func TestOpenAICompatibleClientEndpointOverride(t *testing.T) { - defaultHit := false - overrideHit := false - - defaultServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defaultHit = true - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"default"}}]}`)) - })) - defer defaultServer.Close() - - overrideServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - overrideHit = true - if r.URL.Path != "/v1/chat/completions" { - t.Fatalf("unexpected path: %s", r.URL.Path) - } - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"override"}}]}`)) - })) - defer overrideServer.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: defaultServer.URL + "/v1", Model: "m"}) - if err != nil { - t.Fatal(err) - } - - resp, err := client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{Endpoint: overrideServer.URL + "/v1"}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if resp.Content != "override" { - t.Fatalf("expected override response, got %q", resp.Content) - } - if defaultHit { - t.Fatal("default endpoint should not have been called") - } - if !overrideHit { - t.Fatal("override endpoint should have been called") - } -} - -func TestOpenAICompatibleClientNon2xxError(t *testing.T) { - const sensitiveBody = `provider-secret-fragment request_payload_details` - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"` + sensitiveBody + `"}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "m"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - }) - if err == nil { - t.Fatal("expected non-2xx error") - } - if !errors.Is(err, ErrUnexpectedStatus) { - t.Fatalf("expected ErrUnexpectedStatus, got %v", err) - } - if !strings.Contains(err.Error(), "status=400") { - t.Fatalf("expected status detail, got %v", err) - } - if strings.Contains(err.Error(), sensitiveBody) { - t.Fatalf("expected provider response body to be redacted, got %v", err) - } -} - -func TestOpenAICompatibleClientMalformedResponseInvalidJSON(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{not valid json`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "m"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - }) - if err == nil { - t.Fatal("expected malformed response error") - } - if !errors.Is(err, ErrMalformedResponse) { - t.Fatalf("expected ErrMalformedResponse, got %v", err) - } -} - -func TestOpenAICompatibleClientMalformedResponseMissingChoices(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{"choices": []}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "m"}) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - }) - if err == nil { - t.Fatal("expected malformed response error") - } - if !errors.Is(err, ErrMalformedResponse) { - t.Fatalf("expected ErrMalformedResponse, got %v", err) - } -} - -func TestOpenAICompatibleClientTimeout(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(250 * time.Millisecond) - _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) - })) - defer ts.Close() - - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ - BaseURL: ts.URL + "/v1", - Model: "m", - Timeout: 50 * time.Millisecond, - }) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - }) - if err == nil { - t.Fatal("expected timeout error") - } - if !errors.Is(err, ErrRequestFailed) { - t.Fatalf("expected ErrRequestFailed, got %v", err) - } -} - -func TestOpenAICompatibleClientNegativeTimeoutRejected(t *testing.T) { - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ - BaseURL: "http://example.com/v1", - Model: "m", - }) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{TimeoutSeconds: -1}, - }) - if err == nil { - t.Fatal("expected invalid request error") - } - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } -} - -func TestOpenAICompatibleClientAllowsEmptyConfiguredBaseURL(t *testing.T) { - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ - BaseURL: "", - Model: "m", - }) - if err != nil { - t.Fatalf("expected empty configured base URL to be allowed, got %v", err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{Endpoint: "http://localhost:9999/v1"}, - }) - if err == nil { - t.Fatal("expected request failure due to unreachable endpoint") - } - if !errors.Is(err, ErrRequestFailed) { - t.Fatalf("expected ErrRequestFailed with request endpoint override, got %v", err) - } -} - -func TestOpenAICompatibleClientRequiresEndpointWhenUnsetEverywhere(t *testing.T) { - client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ - BaseURL: "", - Model: "m", - }) - if err != nil { - t.Fatal(err) - } - - _, err = client.Generate(context.Background(), domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{}, - }) - if err == nil { - t.Fatal("expected endpoint-required error") - } - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } -} diff --git a/internal/profile/builtin/assets/aion-labs/aion-2.yml b/internal/profile/builtin/assets/aion-labs/aion-2.yml deleted file mode 100644 index 785a6fd..0000000 --- a/internal/profile/builtin/assets/aion-labs/aion-2.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: aion-2 -endpoint: https://openrouter.ai/api/v1 -model: aion-labs/aion-2.0 -temperature: 0.72 -reasoning_effort: high -top_p: 0.95 -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-fable-latest.yml b/internal/profile/builtin/assets/anthropic/claude-fable-latest.yml deleted file mode 100644 index 1a1e7aa..0000000 --- a/internal/profile/builtin/assets/anthropic/claude-fable-latest.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: claude-fable-latest -endpoint: https://openrouter.ai/api/v1 -model: "~anthropic/claude-fable-latest" -reasoning_effort: high -timeout_seconds: 600 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml b/internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml deleted file mode 100644 index f22bf15..0000000 --- a/internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: claude-haiku-latest -endpoint: https://openrouter.ai/api/v1 -model: "~anthropic/claude-haiku-latest" -reasoning_effort: medium -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-opus-latest.yml b/internal/profile/builtin/assets/anthropic/claude-opus-latest.yml deleted file mode 100644 index c192c3a..0000000 --- a/internal/profile/builtin/assets/anthropic/claude-opus-latest.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: claude-opus-latest -endpoint: https://openrouter.ai/api/v1 -model: "~anthropic/claude-opus-latest" -reasoning_effort: high -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml b/internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml deleted file mode 100644 index c7be449..0000000 --- a/internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: claude-sonnet-latest -endpoint: https://openrouter.ai/api/v1 -model: "~anthropic/claude-sonnet-latest" -reasoning_effort: high -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/deepseek/deepseek-3-2.yml b/internal/profile/builtin/assets/deepseek/deepseek-3-2.yml deleted file mode 100644 index 1f27fbc..0000000 --- a/internal/profile/builtin/assets/deepseek/deepseek-3-2.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: deepseek-3-2 -endpoint: https://openrouter.ai/api/v1 -model: deepseek/deepseek-v3.2 -reasoning_effort: high -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/deepseek/deepseek-4-flash.yml b/internal/profile/builtin/assets/deepseek/deepseek-4-flash.yml deleted file mode 100644 index a2edf84..0000000 --- a/internal/profile/builtin/assets/deepseek/deepseek-4-flash.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: deepseek-4-flash -endpoint: https://openrouter.ai/api/v1 -model: deepseek/deepseek-v4-flash -#reasoning_effort: medium -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml b/internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml deleted file mode 100644 index c7af1ff..0000000 --- a/internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: deepseek-4-pro -endpoint: https://openrouter.ai/api/v1 -model: deepseek/deepseek-v4-pro -reasoning_effort: high -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-2-flash-lite.yml b/internal/profile/builtin/assets/google/gemini-2-flash-lite.yml deleted file mode 100644 index e648e70..0000000 --- a/internal/profile/builtin/assets/google/gemini-2-flash-lite.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: gemini-2-flash-lite -endpoint: https://openrouter.ai/api/v1 -model: "google/gemini-2.5-flash-lite" -#temperature: 0.15 -reasoning_effort: high -#top_p: 0.98 -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-2-flash.yml b/internal/profile/builtin/assets/google/gemini-2-flash.yml deleted file mode 100644 index 3b1267e..0000000 --- a/internal/profile/builtin/assets/google/gemini-2-flash.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: gemini-2-flash -endpoint: https://openrouter.ai/api/v1 -model: "google/gemini-2.5-flash" -#temperature: 0.15 -reasoning_effort: high -#top_p: 0.98 -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-2-pro.yml b/internal/profile/builtin/assets/google/gemini-2-pro.yml deleted file mode 100644 index b779db6..0000000 --- a/internal/profile/builtin/assets/google/gemini-2-pro.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: gemini-2-pro -endpoint: https://openrouter.ai/api/v1 -model: "google/gemini-2.5-pro" -#temperature: 0.15 -reasoning_effort: high -#top_p: 0.98 -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-3-flash-lite.yml b/internal/profile/builtin/assets/google/gemini-3-flash-lite.yml deleted file mode 100644 index ae44f66..0000000 --- a/internal/profile/builtin/assets/google/gemini-3-flash-lite.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: gemini-3-flash-lite -endpoint: https://openrouter.ai/api/v1 -model: "google/gemini-3.1-flash-lite" -#temperature: 0.15 -reasoning_effort: high -#top_p: 0.98 -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-flash-latest.yml b/internal/profile/builtin/assets/google/gemini-flash-latest.yml deleted file mode 100644 index 2bcda6e..0000000 --- a/internal/profile/builtin/assets/google/gemini-flash-latest.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: gemini-flash-latest -endpoint: https://openrouter.ai/api/v1 -model: "~google/gemini-flash-latest" -#temperature: 0.15 -reasoning_effort: high -#top_p: 0.98 -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-pro-latest.yml b/internal/profile/builtin/assets/google/gemini-pro-latest.yml deleted file mode 100644 index 2e77cbc..0000000 --- a/internal/profile/builtin/assets/google/gemini-pro-latest.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: gemini-pro-latest -endpoint: https://openrouter.ai/api/v1 -model: "~google/gemini-pro-latest" -#temperature: 0.15 -reasoning_effort: high -#top_p: 0.98 -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemma-4-31b.yml b/internal/profile/builtin/assets/google/gemma-4-31b.yml deleted file mode 100644 index f8ff113..0000000 --- a/internal/profile/builtin/assets/google/gemma-4-31b.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: gemma-4-31b -endpoint: https://openrouter.ai/api/v1 -model: google/gemma-4-31b-it:exacto -temperature: 0.15 -reasoning_effort: high -top_p: 0.98 -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/minimax/minimax-m2.yml b/internal/profile/builtin/assets/minimax/minimax-m2.yml deleted file mode 100644 index 6ce7fdf..0000000 --- a/internal/profile/builtin/assets/minimax/minimax-m2.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: minimax-m2 -endpoint: https://openrouter.ai/api/v1 -model: minimax/minimax-m2.5 -temperature: 0.5 -reasoning_effort: high -top_p: 0.95 -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/minimax/minimax-m3.yml b/internal/profile/builtin/assets/minimax/minimax-m3.yml deleted file mode 100644 index da9a3af..0000000 --- a/internal/profile/builtin/assets/minimax/minimax-m3.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: minimax-m3 -endpoint: https://openrouter.ai/api/v1 -model: minimax/minimax-m3 -#temperature: 0.5 -reasoning_effort: high -#top_p: 0.95 -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/mistral/mistral-large-2512.yml b/internal/profile/builtin/assets/mistral/mistral-large-2512.yml deleted file mode 100644 index 106ea2e..0000000 --- a/internal/profile/builtin/assets/mistral/mistral-large-2512.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: mistral-large-2512 -endpoint: https://openrouter.ai/api/v1 -model: mistralai/mistral-large-2512 -temperature: 0.15 -top_p: 0.98 -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml b/internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml deleted file mode 100644 index 762149f..0000000 --- a/internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml +++ /dev/null @@ -1,8 +0,0 @@ -id: mistral-medium-3-5 -endpoint: https://openrouter.ai/api/v1 -model: mistralai/mistral-medium-3-5 -temperature: 0.15 -reasoning_effort: high -top_p: 0.98 -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/mistral/mistral-small-3.yml b/internal/profile/builtin/assets/mistral/mistral-small-3.yml deleted file mode 100644 index d077918..0000000 --- a/internal/profile/builtin/assets/mistral/mistral-small-3.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: mistral-small-3 -endpoint: https://openrouter.ai/api/v1 -model: mistralai/mistral-small-3.2-24b-instruct -temperature: 0.05 -top_p: 1.0 -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/mistral/mistral-small-4.yml b/internal/profile/builtin/assets/mistral/mistral-small-4.yml deleted file mode 100644 index fd0fce6..0000000 --- a/internal/profile/builtin/assets/mistral/mistral-small-4.yml +++ /dev/null @@ -1,8 +0,0 @@ -id: mistral-small-4 -endpoint: https://openrouter.ai/api/v1 -model: mistralai/mistral-small-2603 -temperature: 0.1 -reasoning_effort: high -top_p: 0.98 -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml b/internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml deleted file mode 100644 index bb55536..0000000 --- a/internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: nemotron-3-ultra -endpoint: https://openrouter.ai/api/v1 -model: nvidia/nemotron-3-ultra-550b-a55b -reasoning_effort: high -timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/openai/gpt-5-mini.yml b/internal/profile/builtin/assets/openai/gpt-5-mini.yml deleted file mode 100644 index 72ea4b7..0000000 --- a/internal/profile/builtin/assets/openai/gpt-5-mini.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: gpt-5-mini -endpoint: https://openrouter.ai/api/v1 -model: "openai/gpt-5.4-mini" -reasoning_effort: high -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/assets/openai/gpt-5-nano.yml b/internal/profile/builtin/assets/openai/gpt-5-nano.yml deleted file mode 100644 index 069a3d5..0000000 --- a/internal/profile/builtin/assets/openai/gpt-5-nano.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: gpt-5-nano -endpoint: https://openrouter.ai/api/v1 -model: "openai/gpt-5.4-nano" -reasoning_effort: high -timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY -service_tier: flex diff --git a/internal/profile/builtin/repository.go b/internal/profile/builtin/repository.go deleted file mode 100644 index f553352..0000000 --- a/internal/profile/builtin/repository.go +++ /dev/null @@ -1,31 +0,0 @@ -package builtin - -import ( - "embed" - "strings" - - "gitea.maximumdirect.net/eric/scriptorium/internal/profile" -) - -const assetRoot = "assets" - -//go:embed assets/**/*.yml -var assets embed.FS - -func NewRepository() profile.Repository { - return profile.NewFSRepository(assets, assetRoot) -} - -func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository { - if primary == nil { - return NewRepository() - } - return profile.NewOverlayRepository(primary, NewRepository()) -} - -func NewRepositoryWithDirectory(dir string) profile.Repository { - if strings.TrimSpace(dir) == "" { - return NewRepository() - } - return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir)) -} diff --git a/internal/profile/builtin/repository_test.go b/internal/profile/builtin/repository_test.go deleted file mode 100644 index 5973d9a..0000000 --- a/internal/profile/builtin/repository_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package builtin - -import ( - "context" - "errors" - "io/fs" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" - "gitea.maximumdirect.net/eric/scriptorium/internal/profile" - "gopkg.in/yaml.v3" -) - -func TestBuiltInProfilesValidateThroughRepository(t *testing.T) { - repo := NewRepository() - ids := loadBuiltInProfileIDs(t) - if len(ids) == 0 { - t.Fatal("expected built-in profiles") - } - - for id := range ids { - t.Run(id, func(t *testing.T) { - p, err := repo.GetProfile(context.Background(), id) - if err != nil { - t.Fatalf("expected built-in profile %q to load, got %v", id, err) - } - if p.ID != id { - t.Fatalf("expected profile id %q, got %q", id, p.ID) - } - }) - } -} - -func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) { - loadBuiltInProfileIDs(t) -} - -func loadBuiltInProfileIDs(t *testing.T) map[string]string { - t.Helper() - - ids := map[string]string{} - err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !strings.HasSuffix(name, ".yml") { - return nil - } - - data, err := assets.ReadFile(name) - if err != nil { - t.Fatalf("failed to read built-in profile %s: %v", name, err) - } - - var raw map[string]any - if err := yaml.Unmarshal(data, &raw); err != nil { - t.Fatalf("failed to decode built-in profile %s: %v", name, err) - } - if _, ok := raw["api_key"]; ok { - t.Fatalf("built-in profile %s contains raw api_key", name) - } - id, ok := raw["id"].(string) - if !ok || strings.TrimSpace(id) == "" { - t.Fatalf("built-in profile %s has missing id", name) - } - if previous, ok := ids[id]; ok { - t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name) - } - ids[id] = name - return nil - }) - if err != nil { - t.Fatalf("failed to walk built-in profiles: %v", err) - } - return ids -} - -func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) { - repo := NewRepositoryWithPrimary(staticProfileRepo{ - profiles: map[string]string{"mistral-small-3": "custom-model"}, - }) - - p, err := repo.GetProfile(context.Background(), "mistral-small-3") - if err != nil { - t.Fatalf("expected profile to load, got %v", err) - } - if p.Model != "custom-model" { - t.Fatalf("expected primary profile to override built-in, got %+v", p) - } -} - -func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) { - repo := NewRepositoryWithPrimary(staticProfileRepo{}) - - p, err := repo.GetProfile(context.Background(), "mistral-small-3") - if err != nil { - t.Fatalf("expected built-in profile to load, got %v", err) - } - if p.ID != "mistral-small-3" { - t.Fatalf("unexpected profile: %+v", p) - } -} - -func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) { - repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile}) - - _, err := repo.GetProfile(context.Background(), "mistral-small-3") - if !errors.Is(err, profile.ErrInvalidProfile) { - t.Fatalf("expected primary error, got %v", err) - } -} - -type staticProfileRepo struct { - profiles map[string]string - err error -} - -func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) { - if r.err != nil { - return nil, r.err - } - if model, ok := r.profiles[id]; ok { - return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil - } - return nil, profile.ErrProfileNotFound -} diff --git a/internal/profile/filesystem_repository.go b/internal/profile/filesystem_repository.go deleted file mode 100644 index c6508ea..0000000 --- a/internal/profile/filesystem_repository.go +++ /dev/null @@ -1,213 +0,0 @@ -package profile - -import ( - "bytes" - "context" - "errors" - "fmt" - "io/fs" - "os" - "path" - "strings" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" - "gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog" - "gopkg.in/yaml.v3" -) - -var ( - ErrProfileNotFound = errors.New("execution profile not found") - ErrInvalidYAML = errors.New("invalid YAML format") - ErrInvalidProfile = errors.New("invalid execution profile configuration") - ErrRawAPIKeyNotAllowed = errors.New("raw api_key is not allowed; use api_key_env") -) - -type filesystemRepository struct { - dir string -} - -func NewFilesystemRepository(dir string) Repository { - return &filesystemRepository{dir: dir} -} - -func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - return loadProfile(ctx, os.DirFS(r.dir), ".", id) -} - -type fsRepository struct { - fsys fs.FS - root string -} - -func NewFSRepository(fsys fs.FS, root string) Repository { - return &fsRepository{fsys: fsys, root: root} -} - -func (r *fsRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - return loadProfile(ctx, r.fsys, r.root, id) -} - -type overlayRepository struct { - primary Repository - fallback Repository -} - -func NewOverlayRepository(primary, fallback Repository) Repository { - return &overlayRepository{primary: primary, fallback: fallback} -} - -func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - if r.primary != nil { - prof, err := r.primary.GetProfile(ctx, id) - if err == nil { - return prof, nil - } - if !errors.Is(err, ErrProfileNotFound) { - return nil, err - } - } - if r.fallback == nil { - return nil, ErrProfileNotFound - } - return r.fallback.GetProfile(ctx, id) -} - -func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) { - if strings.TrimSpace(id) == "" { - return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile) - } - if fsys == nil { - return nil, fmt.Errorf("failed to read profile directory: filesystem is nil") - } - - files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root) - if err != nil { - return nil, fmt.Errorf("failed to read profile directory: %w", err) - } - - var matches []profileMatch - for _, fullPath := range files { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - relPath := filecatalog.DisplayPath(root, fullPath) - fileMatch := filecatalog.Stem(path.Base(fullPath)) == id - data, err := fs.ReadFile(fsys, fullPath) - if err != nil { - return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err) - } - metadata := readProfileFileMetadata(data) - idMatch := fileMatch || metadata.id == id - if metadata.hasRawAPIKey { - if idMatch { - return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath) - } - continue - } - - var prof domain.ExecutionProfile - decoder := yaml.NewDecoder(bytes.NewReader(data)) - decoder.KnownFields(true) - if err := decoder.Decode(&prof); err != nil { - if idMatch { - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err) - } - continue - } - - if prof.ID != id { - continue - } - if err := validateProfile(&prof); err != nil { - if errors.Is(err, ErrRawAPIKeyNotAllowed) { - return nil, fmt.Errorf("%w: %s", err, relPath) - } - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err) - } - matches = append(matches, profileMatch{ - profile: &prof, - path: relPath, - }) - } - - if len(matches) > 1 { - paths := make([]string, 0, len(matches)) - for _, match := range matches { - paths = append(paths, match.path) - } - return nil, fmt.Errorf("%w: duplicate execution profile id %q found in: %s", ErrInvalidProfile, id, strings.Join(paths, ", ")) - } - - if len(matches) == 1 { - return matches[0].profile, nil - } - - return nil, ErrProfileNotFound -} - -type profileMatch struct { - profile *domain.ExecutionProfile - path string -} - -type profileFileMetadata struct { - id string - hasRawAPIKey bool -} - -func readProfileFileMetadata(data []byte) profileFileMetadata { - var node yaml.Node - if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil { - return profileFileMetadata{} - } - if node.Kind != yaml.DocumentNode || len(node.Content) == 0 { - return profileFileMetadata{} - } - mapping := node.Content[0] - if mapping.Kind != yaml.MappingNode { - return profileFileMetadata{} - } - - var metadata profileFileMetadata - for i := 0; i+1 < len(mapping.Content); i += 2 { - key := mapping.Content[i] - value := mapping.Content[i+1] - switch key.Value { - case "id": - metadata.id = strings.TrimSpace(value.Value) - case "api_key": - metadata.hasRawAPIKey = true - } - } - return metadata -} - -func validateProfile(p *domain.ExecutionProfile) error { - if strings.TrimSpace(p.ID) == "" { - return errors.New("id is required") - } - if strings.TrimSpace(p.Endpoint) == "" { - return errors.New("endpoint is required") - } - if strings.TrimSpace(p.Model) == "" { - return errors.New("model is required") - } - - if p.Temperature < 0 || p.Temperature > 2 { - return errors.New("temperature must be between 0 and 2") - } - if p.MaxTokens < 0 { - return errors.New("max_tokens must be greater than or equal to 0") - } - if p.TopP < 0 || p.TopP > 1 { - return errors.New("top_p must be between 0 and 1") - } - if p.TimeoutSeconds < 0 { - return errors.New("timeout_seconds must be greater than or equal to 0") - } - - return nil -} diff --git a/internal/profile/repository.go b/internal/profile/repository.go deleted file mode 100644 index e177717..0000000 --- a/internal/profile/repository.go +++ /dev/null @@ -1,12 +0,0 @@ -package profile - -import ( - "context" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -// Repository loads execution profiles. -type Repository interface { - GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) -} diff --git a/internal/profile/repository_test.go b/internal/profile/repository_test.go deleted file mode 100644 index f06b3d2..0000000 --- a/internal/profile/repository_test.go +++ /dev/null @@ -1,479 +0,0 @@ -package profile - -import ( - "context" - "encoding/json" - "errors" - "os" - "path/filepath" - "strings" - "testing" - "testing/fstest" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -func TestFilesystemRepository_GetProfile(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "execution_profile_test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpDir) - - files, err := os.ReadDir("testdata") - if err != nil { - t.Fatalf("failed to read testdata: %v", err) - } - for _, f := range files { - src := filepath.Join("testdata", f.Name()) - dst := filepath.Join(tmpDir, f.Name()) - data, err := os.ReadFile(src) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(dst, data, 0644); err != nil { - t.Fatal(err) - } - } - - repo := NewFilesystemRepository(tmpDir) - ctx := context.Background() - - t.Run("valid local profile", func(t *testing.T) { - p, err := repo.GetProfile(ctx, "local-default") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.ID != "local-default" { - t.Fatalf("unexpected id: %q", p.ID) - } - if p.Endpoint == "" || p.Model == "" { - t.Fatalf("expected endpoint/model to be set: %+v", p) - } - }) - - t.Run("valid profile with api_key_env", func(t *testing.T) { - p, err := repo.GetProfile(ctx, "local-secure") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.APIKeyEnv != "SCRIPTORIUM_API_KEY" { - t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv) - } - if p.ReasoningEffort != "medium" { - t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort) - } - if p.ServiceTier != "priority" { - t.Fatalf("unexpected service_tier: %q", p.ServiceTier) - } - }) - - t.Run("valid nested profile", func(t *testing.T) { - nestedDir := filepath.Join(tmpDir, "local") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writeProfileTestFile(t, filepath.Join(nestedDir, "nested-local.yaml"), ` -id: nested-local -endpoint: http://localhost:8000/v1 -model: nested-model -temperature: 0.1 -`) - - p, err := repo.GetProfile(ctx, "nested-local") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.Model != "nested-model" { - t.Fatalf("unexpected model: %q", p.Model) - } - }) - - t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) { - writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), ` -id: json-extra-params -endpoint: http://localhost:8000/v1 -model: nested-model -extra_params: - string_value: enabled - number_value: 42 - boolean_value: true - object_value: - nested: value - count: 2 - array_value: - - first - - 3 - - false -`) - - p, err := repo.GetProfile(ctx, "json-extra-params") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - var got map[string]any - encoded, err := json.Marshal(p.ExtraParams) - if err != nil { - t.Fatalf("expected extra_params to marshal as JSON, got %v", err) - } - if err := json.Unmarshal(encoded, &got); err != nil { - t.Fatalf("expected extra_params JSON to decode, got %v", err) - } - - if got["string_value"] != "enabled" { - t.Fatalf("unexpected string extra param: %#v", got["string_value"]) - } - if got["number_value"] != float64(42) { - t.Fatalf("unexpected number extra param: %#v", got["number_value"]) - } - if got["boolean_value"] != true { - t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"]) - } - objectValue, ok := got["object_value"].(map[string]any) - if !ok { - t.Fatalf("expected object extra param, got %#v", got["object_value"]) - } - if objectValue["nested"] != "value" || objectValue["count"] != float64(2) { - t.Fatalf("unexpected object extra param: %#v", objectValue) - } - arrayValue, ok := got["array_value"].([]any) - if !ok { - t.Fatalf("expected array extra param, got %#v", got["array_value"]) - } - if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false { - t.Fatalf("unexpected array extra param: %#v", arrayValue) - } - }) - - t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) { - writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), ` -id: duplicate-profile -endpoint: http://localhost:8000/v1 -model: first-model -`) - nestedDir := filepath.Join(tmpDir, "duplicates") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writeProfileTestFile(t, filepath.Join(nestedDir, "duplicate-profile-b.yaml"), ` -id: duplicate-profile -endpoint: http://localhost:8000/v1 -model: second-model -`) - - _, err := repo.GetProfile(ctx, "duplicate-profile") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected duplicate profile to return ErrInvalidProfile, got %v", err) - } - for _, want := range []string{"duplicate execution profile id", "duplicate-profile-a.yaml", filepath.Join("duplicates", "duplicate-profile-b.yaml")} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("expected error to contain %q, got %v", want, err) - } - } - }) - - t.Run("nested raw api_key rejected for likely target file", func(t *testing.T) { - nestedDir := filepath.Join(tmpDir, "secure") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writeProfileTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), ` -id: nested_raw_api_key -endpoint: http://localhost:8000/v1 -model: m -api_key: secret -`) - - _, err := repo.GetProfile(ctx, "nested_raw_api_key") - if !errors.Is(err, ErrRawAPIKeyNotAllowed) { - t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err) - } - if !strings.Contains(err.Error(), filepath.Join("secure", "not_named_like_id.yaml")) { - t.Fatalf("expected nested path in error, got %v", err) - } - }) - - t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) { - writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), ` -id: raw-api-key-non-target -endpoint: http://localhost:8000/v1 -model: m -api_key: secret -`) - - _, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err) - } - }) - - t.Run("invalid yaml", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "invalid_yaml") - if !errors.Is(err, ErrInvalidYAML) { - t.Fatalf("expected ErrInvalidYAML, got %v", err) - } - }) - - t.Run("missing id", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "missing_id") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound, got %v", err) - } - }) - - t.Run("missing endpoint", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "missing-endpoint") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - }) - - t.Run("missing model", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "missing-model") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - }) - - t.Run("unknown field", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "unknown_field") - if !errors.Is(err, ErrInvalidYAML) { - t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err) - } - }) - - t.Run("raw api_key rejected", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "raw_api_key") - if !errors.Is(err, ErrRawAPIKeyNotAllowed) { - t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err) - } - }) - - t.Run("profile not found", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "does-not-exist") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound, got %v", err) - } - }) -} - -func writeProfileTestFile(t *testing.T, path string, content string) { - t.Helper() - if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil { - t.Fatalf("failed to write profile test file %q: %v", path, err) - } -} - -func TestFSRepository(t *testing.T) { - ctx := context.Background() - - t.Run("loads valid profiles from nested directories", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/provider/nested.yaml": profileMapFile(` -id: nested-profile -endpoint: http://localhost:8000/v1 -model: nested-model -temperature: 0.1 -`), - }, "profiles") - - p, err := repo.GetProfile(ctx, "nested-profile") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.ID != "nested-profile" || p.Model != "nested-model" { - t.Fatalf("unexpected profile: %+v", p) - } - }) - - t.Run("rejects unknown YAML fields", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/unknown.yaml": profileMapFile(` -id: unknown-profile -endpoint: http://localhost:8000/v1 -model: model -unknown: value -`), - }, "profiles") - - _, err := repo.GetProfile(ctx, "unknown-profile") - if !errors.Is(err, ErrInvalidYAML) { - t.Fatalf("expected ErrInvalidYAML, got %v", err) - } - }) - - t.Run("rejects raw api_key in selected profile", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/raw.yaml": profileMapFile(` -id: raw-profile -endpoint: http://localhost:8000/v1 -model: model -api_key: secret -`), - }, "profiles") - - _, err := repo.GetProfile(ctx, "raw-profile") - if !errors.Is(err, ErrRawAPIKeyNotAllowed) { - t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err) - } - }) - - t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/raw.yaml": profileMapFile(` -id: raw-profile -endpoint: http://localhost:8000/v1 -model: model -api_key: secret -`), - "profiles/valid.yaml": profileMapFile(` -id: valid-profile -endpoint: http://localhost:8000/v1 -model: model -`), - }, "profiles") - - p, err := repo.GetProfile(ctx, "valid-profile") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.ID != "valid-profile" { - t.Fatalf("unexpected profile: %+v", p) - } - }) - - t.Run("rejects duplicate IDs within one source", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/a.yaml": profileMapFile(` -id: duplicate-profile -endpoint: http://localhost:8000/v1 -model: first -`), - "profiles/nested/b.yaml": profileMapFile(` -id: duplicate-profile -endpoint: http://localhost:8000/v1 -model: second -`), - }, "profiles") - - _, err := repo.GetProfile(ctx, "duplicate-profile") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("expected error to contain %q, got %v", want, err) - } - } - }) -} - -func TestOverlayRepository(t *testing.T) { - ctx := context.Background() - primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"} - fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"} - - t.Run("returns primary matches before fallback matches", func(t *testing.T) { - repo := NewOverlayRepository( - staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}}, - staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}}, - ) - - p, err := repo.GetProfile(ctx, "shared") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.Model != "primary" { - t.Fatalf("expected primary profile, got %+v", p) - } - }) - - t.Run("falls back on primary not found", func(t *testing.T) { - repo := NewOverlayRepository( - staticProfileRepo{}, - staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}}, - ) - - p, err := repo.GetProfile(ctx, "shared") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.Model != "fallback" { - t.Fatalf("expected fallback profile, got %+v", p) - } - }) - - t.Run("does not fall back after primary load errors", func(t *testing.T) { - for _, tc := range []struct { - name string - err error - }{ - {name: "invalid yaml", err: ErrInvalidYAML}, - {name: "invalid profile", err: ErrInvalidProfile}, - {name: "raw api key", err: ErrRawAPIKeyNotAllowed}, - } { - t.Run(tc.name, func(t *testing.T) { - repo := NewOverlayRepository( - staticProfileRepo{err: tc.err}, - staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}}, - ) - - _, err := repo.GetProfile(ctx, "shared") - if !errors.Is(err, tc.err) { - t.Fatalf("expected %v, got %v", tc.err, err) - } - }) - } - }) - - t.Run("returns not found when both sources miss", func(t *testing.T) { - repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{}) - - _, err := repo.GetProfile(ctx, "missing") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound, got %v", err) - } - }) - - t.Run("nil primary uses fallback", func(t *testing.T) { - repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}}) - - p, err := repo.GetProfile(ctx, "shared") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.Model != "fallback" { - t.Fatalf("expected fallback profile, got %+v", p) - } - }) - - t.Run("nil fallback returns not found after primary miss", func(t *testing.T) { - repo := NewOverlayRepository(staticProfileRepo{}, nil) - - _, err := repo.GetProfile(ctx, "missing") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound, got %v", err) - } - }) -} - -func profileMapFile(content string) *fstest.MapFile { - return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))} -} - -type staticProfileRepo struct { - profiles map[string]*domain.ExecutionProfile - err error -} - -func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) { - if r.err != nil { - return nil, r.err - } - if p, ok := r.profiles[id]; ok { - cp := *p - return &cp, nil - } - return nil, ErrProfileNotFound -} diff --git a/internal/profile/testdata/invalid_yaml.yaml b/internal/profile/testdata/invalid_yaml.yaml deleted file mode 100644 index 6ba79f8..0000000 --- a/internal/profile/testdata/invalid_yaml.yaml +++ /dev/null @@ -1,3 +0,0 @@ -id: invalid_yaml -endpoint: http://localhost:8000/v1 -model: [broken diff --git a/internal/profile/testdata/missing_endpoint.yaml b/internal/profile/testdata/missing_endpoint.yaml deleted file mode 100644 index 443c8d6..0000000 --- a/internal/profile/testdata/missing_endpoint.yaml +++ /dev/null @@ -1,2 +0,0 @@ -id: missing-endpoint -model: gpt-4o-mini diff --git a/internal/profile/testdata/missing_id.yaml b/internal/profile/testdata/missing_id.yaml deleted file mode 100644 index 4c1421d..0000000 --- a/internal/profile/testdata/missing_id.yaml +++ /dev/null @@ -1,2 +0,0 @@ -endpoint: http://localhost:8000/v1 -model: gpt-4o-mini diff --git a/internal/profile/testdata/missing_model.yaml b/internal/profile/testdata/missing_model.yaml deleted file mode 100644 index 838092b..0000000 --- a/internal/profile/testdata/missing_model.yaml +++ /dev/null @@ -1,2 +0,0 @@ -id: missing-model -endpoint: http://localhost:8000/v1 diff --git a/internal/profile/testdata/raw_api_key.yaml b/internal/profile/testdata/raw_api_key.yaml deleted file mode 100644 index a1b61ea..0000000 --- a/internal/profile/testdata/raw_api_key.yaml +++ /dev/null @@ -1,4 +0,0 @@ -id: raw-api-key -endpoint: http://localhost:8000/v1 -model: gpt-4o-mini -api_key: super-secret-should-not-be-here diff --git a/internal/profile/testdata/unknown_field.yaml b/internal/profile/testdata/unknown_field.yaml deleted file mode 100644 index 9b1916a..0000000 --- a/internal/profile/testdata/unknown_field.yaml +++ /dev/null @@ -1,4 +0,0 @@ -id: unknown-field -endpoint: http://localhost:8000/v1 -model: gpt-4o-mini -foo: bar diff --git a/internal/profile/testdata/valid_local_profile.yaml b/internal/profile/testdata/valid_local_profile.yaml deleted file mode 100644 index 6430a37..0000000 --- a/internal/profile/testdata/valid_local_profile.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: local-default -endpoint: http://localhost:8000/v1 -model: gpt-4o-mini -temperature: 0.2 -max_tokens: 700 -top_p: 1.0 -timeout_seconds: 120 diff --git a/internal/profile/testdata/valid_with_api_key_env.yaml b/internal/profile/testdata/valid_with_api_key_env.yaml deleted file mode 100644 index 8059ca0..0000000 --- a/internal/profile/testdata/valid_with_api_key_env.yaml +++ /dev/null @@ -1,8 +0,0 @@ -id: local-secure -endpoint: http://localhost:8000/v1 -model: gpt-4o-mini -api_key_env: SCRIPTORIUM_API_KEY -service_tier: priority -reasoning_effort: medium -extra_params: - provider: local diff --git a/internal/prompt/go_renderer.go b/internal/prompt/go_renderer.go deleted file mode 100644 index eecc2c7..0000000 --- a/internal/prompt/go_renderer.go +++ /dev/null @@ -1,125 +0,0 @@ -package prompt - -import ( - "bytes" - "context" - "errors" - "fmt" - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" - "strings" - "text/template" - "unicode/utf8" -) - -var ( - ErrMissingRequiredInput = errors.New("missing required input artifact") - ErrUnknownInput = errors.New("referenced unknown input artifact") - ErrInvalidTemplate = errors.New("invalid prompt template") - ErrRenderFailure = errors.New("prompt render failure") - ErrInvalidMessageRole = errors.New("invalid or empty message role") -) - -type goRenderer struct{} - -func NewGoRenderer() Renderer { - return &goRenderer{} -} - -func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) { - if definition == nil { - return nil, fmt.Errorf("%w: nil prompt definition", ErrRenderFailure) - } - - // 1. Verify required inputs - for _, in := range definition.Inputs { - if !in.Required { - continue - } - art, ok := inputs[in.Name] - if !ok || art == nil { - return nil, fmt.Errorf("%w: %s", ErrMissingRequiredInput, in.Name) - } - } - - // 2. Setup template functions - funcs := template.FuncMap{ - "input": func(name string) (string, error) { - art, ok := inputs[name] - if !ok || art == nil { - return "", fmt.Errorf("%w: %s", ErrUnknownInput, name) - } - return string(art.Body), nil - }, - } - - sessionID, err := renderSessionID(definition.SessionID, funcs, vars) - if err != nil { - return nil, err - } - - var renderedMessages []domain.RenderedMessage - - for i, tmplMsg := range definition.Templates { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - if tmplMsg.Role == "" { - return nil, fmt.Errorf("%w: message %d", ErrInvalidMessageRole, i) - } - - // Parse and execute template - tmpl, err := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content) - if err != nil { - return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, err) - } - - var buf bytes.Buffer - if err := tmpl.Execute(&buf, vars); err != nil { - return nil, fmt.Errorf("%w: message %d: %w", ErrRenderFailure, i, err) - } - - renderedMessages = append(renderedMessages, domain.RenderedMessage{ - Role: tmplMsg.Role, - Content: buf.String(), - CacheControl: cloneCacheControl(tmplMsg.CacheControl), - }) - } - - return &domain.RenderedPrompt{ - SessionID: sessionID, - Messages: renderedMessages, - }, nil -} - -func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) { - if strings.TrimSpace(raw) == "" { - return "", nil - } - - tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw) - if err != nil { - return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err) - } - - var buf bytes.Buffer - if err := tmpl.Execute(&buf, vars); err != nil { - return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err) - } - - sessionID := strings.TrimSpace(buf.String()) - if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength { - return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength) - } - return sessionID, nil -} - -func cloneCacheControl(in *domain.CacheControl) *domain.CacheControl { - if in == nil { - return nil - } - out := *in - return &out -} diff --git a/internal/prompt/renderer.go b/internal/prompt/renderer.go deleted file mode 100644 index 9e4469d..0000000 --- a/internal/prompt/renderer.go +++ /dev/null @@ -1,11 +0,0 @@ -package prompt - -import ( - "context" - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -// Renderer renders prompt templates using named artifacts and variables. -type Renderer interface { - Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) -} diff --git a/internal/prompt/renderer_test.go b/internal/prompt/renderer_test.go deleted file mode 100644 index a2dee8a..0000000 --- a/internal/prompt/renderer_test.go +++ /dev/null @@ -1,345 +0,0 @@ -package prompt - -import ( - "context" - "errors" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -func TestGoRenderer_Render(t *testing.T) { - renderer := NewGoRenderer() - ctx := context.Background() - - inputs := map[string]*domain.Artifact{ - "transcript": {Body: []byte("The quick brown fox.")}, - } - vars := map[string]string{ - "role": "helpful assistant", - "tone": "concise", - } - - t.Run("rendering inline message content", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "user", Content: "Analyze this: {{input \"transcript\"}}"}, - }, - } - - res, err := renderer.Render(ctx, def, inputs, vars) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(res.Messages) != 1 { - t.Fatalf("expected 1 message, got %d", len(res.Messages)) - } - if res.Messages[0].Content != "Analyze this: The quick brown fox." { - t.Fatalf("unexpected rendered content: %q", res.Messages[0].Content) - } - }) - - t.Run("rendering file-backed message content loaded into prompt definition", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "user", Content: "From file: {{input \"transcript\"}}", ContentFile: "/tmp/user.tmpl"}, - }, - } - - res, err := renderer.Render(ctx, def, inputs, vars) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got := res.Messages[0].Content; got != "From file: The quick brown fox." { - t.Fatalf("unexpected file-backed render result: %q", got) - } - }) - - t.Run("rendering system and user messages", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "system", Content: "You are a {{.role}}."}, - {Role: "user", Content: "Analyze this: {{input \"transcript\"}}"}, - }, - } - - res, err := renderer.Render(ctx, def, inputs, vars) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(res.Messages) != 2 { - t.Fatalf("expected 2 messages, got %d", len(res.Messages)) - } - if res.Messages[0].Role != "system" || res.Messages[1].Role != "user" { - t.Fatalf("unexpected roles: %#v", res.Messages) - } - }) - - t.Run("copying cache control to rendered messages", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - { - Role: "system", - Content: "You are concise.", - CacheControl: &domain.CacheControl{ - Type: domain.CacheControlEphemeral, - TTL: "1h", - }, - }, - {Role: "user", Content: "Analyze this: {{input \"transcript\"}}"}, - }, - } - - res, err := renderer.Render(ctx, def, inputs, vars) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(res.Messages) != 2 { - t.Fatalf("expected 2 messages, got %d", len(res.Messages)) - } - if res.Messages[0].CacheControl == nil { - t.Fatal("expected rendered cache control") - } - if res.Messages[0].CacheControl.Type != domain.CacheControlEphemeral { - t.Fatalf("unexpected cache control type: %q", res.Messages[0].CacheControl.Type) - } - if res.Messages[0].CacheControl.TTL != "1h" { - t.Fatalf("unexpected cache control ttl: %q", res.Messages[0].CacheControl.TTL) - } - if res.Messages[1].CacheControl != nil { - t.Fatalf("expected no cache control on second message, got %#v", res.Messages[1].CacheControl) - } - }) - - t.Run("rendered cache control does not alias source template", func(t *testing.T) { - source := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"} - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "system", Content: "You are concise.", CacheControl: source}, - }, - } - - res, err := renderer.Render(ctx, def, inputs, vars) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.Messages[0].CacheControl == source { - t.Fatal("expected rendered cache control to be cloned") - } - - res.Messages[0].CacheControl.TTL = "" - if source.TTL != "1h" { - t.Fatalf("source cache control was mutated, ttl=%q", source.TTL) - } - }) - - t.Run("accessing vars", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "system", Content: "Speak in a {{.tone}} tone."}, - }, - } - - res, err := renderer.Render(ctx, def, inputs, vars) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.Messages[0].Content != "Speak in a concise tone." { - t.Fatalf("unexpected vars rendering: %q", res.Messages[0].Content) - } - }) - - t.Run("rendering session id from vars", func(t *testing.T) { - def := &domain.PromptDefinition{ - SessionID: " {{ .session_id }} ", - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "system", Content: "Speak in a {{.tone}} tone."}, - }, - } - - res, err := renderer.Render(ctx, def, inputs, map[string]string{ - "tone": "concise", - "session_id": "agent-session-123", - }) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.SessionID != "agent-session-123" { - t.Fatalf("unexpected session id: %q", res.SessionID) - } - }) - - t.Run("empty rendered session id is omitted", func(t *testing.T) { - def := &domain.PromptDefinition{ - SessionID: " ", - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "system", Content: "Speak in a {{.tone}} tone."}, - }, - } - - res, err := renderer.Render(ctx, def, inputs, vars) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.SessionID != "" { - t.Fatalf("expected empty session id, got %q", res.SessionID) - } - }) - - t.Run("missing session id var fails rendering", func(t *testing.T) { - def := &domain.PromptDefinition{ - SessionID: "{{ .session_id }}", - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "system", Content: "Speak in a {{.tone}} tone."}, - }, - } - - _, err := renderer.Render(ctx, def, inputs, vars) - if !errors.Is(err, ErrRenderFailure) { - t.Fatalf("expected ErrRenderFailure, got %v", err) - } - }) - - t.Run("too long rendered session id fails rendering", func(t *testing.T) { - def := &domain.PromptDefinition{ - SessionID: "{{ .session_id }}", - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "system", Content: "Speak in a {{.tone}} tone."}, - }, - } - - _, err := renderer.Render(ctx, def, inputs, map[string]string{ - "tone": "concise", - "session_id": strings.Repeat("x", domain.SessionIDMaxLength+1), - }) - if !errors.Is(err, ErrRenderFailure) { - t.Fatalf("expected ErrRenderFailure, got %v", err) - } - }) - - t.Run("inserting required input artifact", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "user", Content: "{{input \"transcript\"}}"}, - }, - } - - res, err := renderer.Render(ctx, def, inputs, vars) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.Messages[0].Content != "The quick brown fox." { - t.Fatalf("unexpected required input rendering: %q", res.Messages[0].Content) - } - }) - - t.Run("optional input absent and not referenced", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{ - {Name: "transcript", Required: true}, - {Name: "glossary", Required: false}, - }, - Templates: []domain.PromptMessageTemplate{ - {Role: "user", Content: "Transcript: {{input \"transcript\"}}"}, - }, - } - - res, err := renderer.Render(ctx, def, inputs, vars) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(res.Messages) != 1 { - t.Fatalf("expected one rendered message, got %d", len(res.Messages)) - } - }) - - t.Run("optional input absent but referenced, expecting failure", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{ - {Name: "transcript", Required: true}, - {Name: "glossary", Required: false}, - }, - Templates: []domain.PromptMessageTemplate{ - {Role: "user", Content: "Glossary: {{input \"glossary\"}}"}, - }, - } - - _, err := renderer.Render(ctx, def, inputs, vars) - if !errors.Is(err, ErrRenderFailure) { - t.Fatalf("expected ErrRenderFailure, got %v", err) - } - if !errors.Is(err, ErrUnknownInput) { - t.Fatalf("expected ErrUnknownInput, got %v", err) - } - }) - - t.Run("required input missing, expecting failure", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "user", Content: "Analyze this: {{input \"transcript\"}}"}, - }, - } - - _, err := renderer.Render(ctx, def, map[string]*domain.Artifact{}, vars) - if !errors.Is(err, ErrMissingRequiredInput) { - t.Fatalf("expected ErrMissingRequiredInput, got %v", err) - } - }) - - t.Run("invalid template syntax", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "user", Content: "Hello {{.unclosed"}, - }, - } - - _, err := renderer.Render(ctx, def, inputs, vars) - if !errors.Is(err, ErrInvalidTemplate) { - t.Fatalf("expected ErrInvalidTemplate, got %v", err) - } - }) - - t.Run("unknown input reference", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "user", Content: "Hello {{input \"ghost\"}}"}, - }, - } - - _, err := renderer.Render(ctx, def, inputs, vars) - if !errors.Is(err, ErrRenderFailure) { - t.Fatalf("expected ErrRenderFailure, got %v", err) - } - if !errors.Is(err, ErrUnknownInput) { - t.Fatalf("expected ErrUnknownInput, got %v", err) - } - }) - - t.Run("empty message role", func(t *testing.T) { - def := &domain.PromptDefinition{ - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{ - {Role: "", Content: "Hello"}, - }, - } - _, err := renderer.Render(ctx, def, inputs, vars) - if !errors.Is(err, ErrInvalidMessageRole) { - t.Fatalf("expected ErrInvalidMessageRole, got %v", err) - } - }) -} diff --git a/internal/promptdef/filesystem_repository.go b/internal/promptdef/filesystem_repository.go deleted file mode 100644 index 54009e7..0000000 --- a/internal/promptdef/filesystem_repository.go +++ /dev/null @@ -1,484 +0,0 @@ -package promptdef - -import ( - "bytes" - "context" - "errors" - "fmt" - "io/fs" - "os" - "path" - "path/filepath" - "strings" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" - "gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog" - "gopkg.in/yaml.v3" -) - -var ( - ErrPromptDefinitionNotFound = errors.New("prompt definition not found") - ErrInvalidYAML = errors.New("invalid YAML format") - ErrInvalidPromptDefinition = errors.New("invalid prompt definition configuration") -) - -type filesystemRepository struct { - dir string -} - -type fsRepository struct { - fsys fs.FS - root string -} - -type promptDefinitionFile struct { - ID string `yaml:"id"` - Version string `yaml:"version"` - DefaultProfile *string `yaml:"default_profile"` - Description string `yaml:"description"` - SessionID string `yaml:"session_id"` - Inputs []promptInputFile `yaml:"inputs"` - Messages []promptMessageFile `yaml:"messages"` - Output promptOutputContractFile `yaml:"output"` -} - -type promptInputFile struct { - Name string `yaml:"name"` - Required bool `yaml:"required"` - ContentType string `yaml:"content_type"` - Description string `yaml:"description"` -} - -type promptMessageFile struct { - Role string `yaml:"role"` - Content string `yaml:"content"` - ContentFile string `yaml:"content_file"` - CacheControl *cacheControlFile `yaml:"cache_control"` -} - -type cacheControlFile struct { - Type string `yaml:"type"` - TTL string `yaml:"ttl"` -} - -type promptOutputContractFile struct { - Format domain.OutputFormat `yaml:"format"` - ValidationMode domain.ValidationMode `yaml:"validation_mode"` - SchemaPath string `yaml:"schema_path"` - RepairAttempts int `yaml:"repair_attempts"` -} - -func NewFilesystemRepository(dir string) Repository { - return &filesystemRepository{dir: dir} -} - -func NewFSRepository(fsys fs.FS, root string) Repository { - return &fsRepository{fsys: fsys, root: root} -} - -func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) { - if strings.TrimSpace(id) == "" { - return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition) - } - - files, err := filecatalog.FindYAMLFiles(ctx, r.dir) - if err != nil { - return nil, fmt.Errorf("failed to read prompt definition directory: %w", err) - } - - var matches []promptDefinitionMatch - for _, fullPath := range files { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - relPath := filecatalog.RelativePath(r.dir, fullPath) - fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id - - raw, err := loadPromptDefinitionFile(fullPath) - if err != nil { - if fileMatch || promptDefinitionFileHasID(fullPath, id) { - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err) - } - continue - } - - def, err := normalizePromptDefinition(raw, fullPath) - if err != nil { - if fileMatch || strings.TrimSpace(raw.ID) == id { - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err) - } - continue - } - - if def.ID != id { - continue - } - if version != "" && def.Version != version { - continue - } - matches = append(matches, promptDefinitionMatch{ - def: def, - path: relPath, - }) - } - - if len(matches) > 1 { - paths := make([]string, 0, len(matches)) - for _, match := range matches { - paths = append(paths, match.path) - } - if version != "" { - return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", ")) - } - return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", ")) - } - - if len(matches) == 1 { - return matches[0].def, nil - } - - return nil, ErrPromptDefinitionNotFound -} - -func (r *fsRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) { - return loadPromptDefinition(ctx, r.fsys, r.root, id, version) -} - -type promptDefinitionMatch struct { - def *domain.PromptDefinition - path string -} - -func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read prompt definition file: %w", err) - } - - var raw promptDefinitionFile - decoder := yaml.NewDecoder(bytes.NewReader(data)) - decoder.KnownFields(true) - if err := decoder.Decode(&raw); err != nil { - return nil, err - } - return &raw, nil -} - -func promptDefinitionFileHasID(path string, id string) bool { - data, err := os.ReadFile(path) - if err != nil { - return false - } - var raw struct { - ID string `yaml:"id"` - } - if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil { - return false - } - return strings.TrimSpace(raw.ID) == id -} - -func loadPromptDefinition(ctx context.Context, fsys fs.FS, root string, id string, version string) (*domain.PromptDefinition, error) { - if strings.TrimSpace(id) == "" { - return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition) - } - if fsys == nil { - return nil, fmt.Errorf("failed to read prompt definition directory: filesystem is nil") - } - - files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root) - if err != nil { - return nil, fmt.Errorf("failed to read prompt definition directory: %w", err) - } - cleanRoot := filecatalog.CleanFSRoot(root) - rootInfo, err := fs.Stat(fsys, cleanRoot) - if err != nil { - return nil, fmt.Errorf("failed to read prompt definition directory: %w", err) - } - - var matches []promptDefinitionMatch - for _, fullPath := range files { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - relPath := filecatalog.DisplayPath(root, fullPath) - fileMatch := filecatalog.Stem(path.Base(fullPath)) == id - data, err := fs.ReadFile(fsys, fullPath) - if err != nil { - if fileMatch { - return nil, fmt.Errorf("%w: %s: failed to read prompt definition file: %v", ErrInvalidYAML, relPath, err) - } - continue - } - - raw, err := decodePromptDefinition(data) - if err != nil { - if fileMatch || promptDefinitionDataHasID(data, id) { - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err) - } - continue - } - - def, err := normalizePromptDefinitionFromFS(raw, fsys, root, fullPath, rootInfo.IsDir()) - if err != nil { - if fileMatch || strings.TrimSpace(raw.ID) == id { - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err) - } - continue - } - - if def.ID != id { - continue - } - if version != "" && def.Version != version { - continue - } - matches = append(matches, promptDefinitionMatch{ - def: def, - path: relPath, - }) - } - - if len(matches) > 1 { - paths := make([]string, 0, len(matches)) - for _, match := range matches { - paths = append(paths, match.path) - } - if version != "" { - return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", ")) - } - return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", ")) - } - - if len(matches) == 1 { - return matches[0].def, nil - } - - return nil, ErrPromptDefinitionNotFound -} - -func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) { - var raw promptDefinitionFile - decoder := yaml.NewDecoder(bytes.NewReader(data)) - decoder.KnownFields(true) - if err := decoder.Decode(&raw); err != nil { - return nil, err - } - return &raw, nil -} - -func promptDefinitionDataHasID(data []byte, id string) bool { - var raw struct { - ID string `yaml:"id"` - } - if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil { - return false - } - return strings.TrimSpace(raw.ID) == id -} - -func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) { - promptDir := filepath.Dir(sourcePath) - return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) { - resolvedPath := strings.TrimSpace(contentFile) - if !filepath.IsAbs(resolvedPath) { - resolvedPath = filepath.Join(promptDir, resolvedPath) - } - resolvedPath = filepath.Clean(resolvedPath) - - body, err := os.ReadFile(resolvedPath) - if err != nil { - return "", "", err - } - return string(body), resolvedPath, nil - }) -} - -func normalizePromptDefinitionFromFS(raw *promptDefinitionFile, fsys fs.FS, root string, sourcePath string, rootIsDir bool) (*domain.PromptDefinition, error) { - promptDir := path.Dir(sourcePath) - return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) { - var resolvedPath string - if rootIsDir { - var err error - resolvedPath, _, err = filecatalog.ResolveFSPath(root, promptDir, contentFile) - if err != nil { - return "", "", err - } - } else { - resolvedPath = strings.TrimSpace(contentFile) - if !path.IsAbs(resolvedPath) { - resolvedPath = path.Join(promptDir, resolvedPath) - } - resolvedPath = strings.TrimPrefix(path.Clean(resolvedPath), "/") - } - - body, err := fs.ReadFile(fsys, resolvedPath) - if err != nil { - return "", "", err - } - return string(body), resolvedPath, nil - }) -} - -func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContentFile func(string) (string, string, error)) (*domain.PromptDefinition, error) { - if raw == nil { - return nil, errors.New("prompt definition is nil") - } - - id := strings.TrimSpace(raw.ID) - if id == "" { - return nil, errors.New("id is required") - } - - version := strings.TrimSpace(raw.Version) - if version == "" { - return nil, errors.New("version is required") - } - - if len(raw.Messages) == 0 { - return nil, errors.New("at least one message is required") - } - - inputs := make([]domain.PromptInput, 0, len(raw.Inputs)) - seenInputNames := make(map[string]struct{}, len(raw.Inputs)) - for i, in := range raw.Inputs { - name := strings.TrimSpace(in.Name) - if name == "" { - return nil, fmt.Errorf("input %d has empty name", i) - } - if _, exists := seenInputNames[name]; exists { - return nil, fmt.Errorf("duplicate input name %q", name) - } - seenInputNames[name] = struct{}{} - - inputs = append(inputs, domain.PromptInput{ - Name: name, - Required: in.Required, - ContentType: strings.TrimSpace(in.ContentType), - Description: strings.TrimSpace(in.Description), - }) - } - - templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages)) - for i, msg := range raw.Messages { - role := strings.TrimSpace(msg.Role) - if role == "" { - return nil, fmt.Errorf("message %d role is required", i) - } - - hasContent := strings.TrimSpace(msg.Content) != "" - hasContentFile := strings.TrimSpace(msg.ContentFile) != "" - if hasContent == hasContentFile { - return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role) - } - - cacheControl, err := normalizeCacheControl(msg.CacheControl) - if err != nil { - return nil, fmt.Errorf("message %d (%s) cache_control: %w", i, role, err) - } - - templateContent := msg.Content - resolvedContentFile := "" - if hasContentFile { - body, resolvedPath, err := readContentFile(msg.ContentFile) - if err != nil { - return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err) - } - templateContent = body - resolvedContentFile = resolvedPath - } - - templates = append(templates, domain.PromptMessageTemplate{ - Role: role, - Content: templateContent, - ContentFile: resolvedContentFile, - CacheControl: cacheControl, - }) - } - - if !isValidOutputFormat(raw.Output.Format) { - return nil, fmt.Errorf("invalid output format: %q", raw.Output.Format) - } - if !isValidValidationMode(raw.Output.ValidationMode) { - return nil, fmt.Errorf("invalid validation mode: %q", raw.Output.ValidationMode) - } - if raw.Output.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(raw.Output.SchemaPath) == "" { - return nil, errors.New("output.schema_path is required when output.validation_mode is json_schema") - } - if raw.Output.RepairAttempts < 0 { - return nil, errors.New("output.repair_attempts must be greater than or equal to 0") - } - - defaultProfile := "" - if raw.DefaultProfile != nil { - defaultProfile = strings.TrimSpace(*raw.DefaultProfile) - if defaultProfile == "" { - return nil, errors.New("default_profile must be a non-empty string when set") - } - } - - return &domain.PromptDefinition{ - ID: id, - Version: version, - DefaultProfile: defaultProfile, - Description: strings.TrimSpace(raw.Description), - SessionID: strings.TrimSpace(raw.SessionID), - Inputs: inputs, - Templates: templates, - OutputFormat: raw.Output.Format, - Validation: domain.OutputContract{ - Format: raw.Output.Format, - ValidationMode: raw.Output.ValidationMode, - SchemaPath: strings.TrimSpace(raw.Output.SchemaPath), - RepairAttempts: raw.Output.RepairAttempts, - }, - }, nil -} - -func normalizeCacheControl(raw *cacheControlFile) (*domain.CacheControl, error) { - if raw == nil { - return nil, nil - } - - cacheType := strings.TrimSpace(raw.Type) - if cacheType == "" { - return nil, errors.New("type is required") - } - if domain.CacheControlType(cacheType) != domain.CacheControlEphemeral { - return nil, fmt.Errorf("unsupported type %q", cacheType) - } - - ttl := strings.TrimSpace(raw.TTL) - if ttl != "" && ttl != "1h" { - return nil, fmt.Errorf("unsupported ttl %q", ttl) - } - - return &domain.CacheControl{ - Type: domain.CacheControlType(cacheType), - TTL: ttl, - }, nil -} - -func isValidOutputFormat(f domain.OutputFormat) bool { - switch f { - case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON: - return true - default: - return false - } -} - -func isValidValidationMode(m domain.ValidationMode) bool { - switch m { - case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema: - return true - default: - return false - } -} diff --git a/internal/promptdef/repository.go b/internal/promptdef/repository.go deleted file mode 100644 index 69be356..0000000 --- a/internal/promptdef/repository.go +++ /dev/null @@ -1,12 +0,0 @@ -package promptdef - -import ( - "context" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -// Repository loads prompt definitions. -type Repository interface { - GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) -} diff --git a/internal/promptdef/repository_test.go b/internal/promptdef/repository_test.go deleted file mode 100644 index 2bbeb12..0000000 --- a/internal/promptdef/repository_test.go +++ /dev/null @@ -1,526 +0,0 @@ -package promptdef - -import ( - "context" - "errors" - "io/fs" - "os" - "path/filepath" - "strings" - "testing" - "testing/fstest" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -func TestFilesystemRepository_GetPromptDefinition(t *testing.T) { - tmpDir := t.TempDir() - if err := copyTree("testdata", tmpDir); err != nil { - t.Fatalf("failed to copy testdata: %v", err) - } - - repo := NewFilesystemRepository(tmpDir) - ctx := context.Background() - - t.Run("valid inline prompt", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "valid-inline", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.ID != "valid-inline" { - t.Fatalf("unexpected id: %q", p.ID) - } - if p.Version != "1.0.0" { - t.Fatalf("unexpected version: %q", p.Version) - } - if p.OutputFormat != domain.FormatMarkdown { - t.Fatalf("unexpected output format: %q", p.OutputFormat) - } - if p.Validation.ValidationMode != domain.ValidationBasic { - t.Fatalf("unexpected validation mode: %q", p.Validation.ValidationMode) - } - if len(p.Templates) != 2 { - t.Fatalf("expected 2 messages, got %d", len(p.Templates)) - } - if len(p.Inputs) != 1 { - t.Fatalf("expected 1 input, got %d", len(p.Inputs)) - } - if p.Inputs[0].ContentType != "text/markdown" { - t.Fatalf("expected input content_type to be preserved, got %q", p.Inputs[0].ContentType) - } - }) - - t.Run("valid file-backed prompt", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "valid-file-backed", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(p.Templates) != 2 { - t.Fatalf("expected 2 messages, got %d", len(p.Templates)) - } - if !strings.Contains(p.Templates[1].Content, "{{input \"transcript\"}}") { - t.Fatalf("expected content_file template body to be loaded, got %q", p.Templates[1].Content) - } - if p.Templates[1].ContentFile == "" { - t.Fatal("expected ContentFile source metadata to be preserved") - } - if !filepath.IsAbs(p.Templates[1].ContentFile) { - t.Fatalf("expected resolved content_file path to be absolute, got %q", p.Templates[1].ContentFile) - } - }) - - t.Run("valid cache control with ttl", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-ttl", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(p.Templates) != 2 { - t.Fatalf("expected 2 messages, got %d", len(p.Templates)) - } - assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "1h") - if p.Templates[1].CacheControl != nil { - t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl) - } - }) - - t.Run("valid cache control without ttl", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-without-ttl", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(p.Templates) != 2 { - t.Fatalf("expected 2 messages, got %d", len(p.Templates)) - } - assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "") - if p.Templates[1].CacheControl != nil { - t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl) - } - }) - - t.Run("valid session id template", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "valid-session-id", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.SessionID != "{{ .session_id }}" { - t.Fatalf("expected trimmed session_id template, got %q", p.SessionID) - } - }) - - t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) { - nestedDir := filepath.Join(tmpDir, "dnd", "recap") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writePromptTestFile(t, filepath.Join(nestedDir, "nested_recap.yaml"), ` -id: nested-recap -version: "1.0.0" -messages: - - role: user - content_file: ./nested_recap.user.tmpl -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - writePromptTestFile(t, filepath.Join(nestedDir, "nested_recap.user.tmpl"), `Nested recap: {{input "transcript"}}`) - - p, err := repo.GetPromptDefinition(ctx, "nested-recap", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(p.Templates) != 1 { - t.Fatalf("expected one template, got %d", len(p.Templates)) - } - if !strings.Contains(p.Templates[0].Content, "Nested recap") { - t.Fatalf("expected nested content file body, got %q", p.Templates[0].Content) - } - if !strings.Contains(p.Templates[0].ContentFile, filepath.Join("dnd", "recap", "nested_recap.user.tmpl")) { - t.Fatalf("expected nested content file path, got %q", p.Templates[0].ContentFile) - } - }) - - t.Run("prompt with default_profile", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "with-default-profile", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.DefaultProfile != "local-default" { - t.Fatalf("unexpected default profile: %q", p.DefaultProfile) - } - if len(p.Inputs) != 1 { - t.Fatalf("expected one input, got %d", len(p.Inputs)) - } - if p.Inputs[0].ContentType != "" { - t.Fatalf("expected missing content_type to remain empty, got %q", p.Inputs[0].ContentType) - } - }) - - t.Run("duplicate prompt IDs fail as ambiguous", func(t *testing.T) { - writePromptTestFile(t, filepath.Join(tmpDir, "duplicate_a.yaml"), ` -id: duplicate-prompt -version: "1.0.0" -messages: - - role: user - content: First duplicate. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - nestedDir := filepath.Join(tmpDir, "nested") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writePromptTestFile(t, filepath.Join(nestedDir, "duplicate_b.yaml"), ` -id: duplicate-prompt -version: "2.0.0" -messages: - - role: user - content: Second duplicate. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - - _, err := repo.GetPromptDefinition(ctx, "duplicate-prompt", "") - if !errors.Is(err, ErrInvalidPromptDefinition) { - t.Fatalf("expected duplicate prompt to return ErrInvalidPromptDefinition, got %v", err) - } - for _, want := range []string{"duplicate prompt definition id", "duplicate_a.yaml", filepath.Join("nested", "duplicate_b.yaml")} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("expected error to contain %q, got %v", want, err) - } - } - }) - - t.Run("duplicate prompt ID and requested version fails as ambiguous", func(t *testing.T) { - writePromptTestFile(t, filepath.Join(tmpDir, "version_duplicate_a.yaml"), ` -id: duplicate-version-prompt -version: "1.0.0" -messages: - - role: user - content: First duplicate version. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - nestedDir := filepath.Join(tmpDir, "versioned") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writePromptTestFile(t, filepath.Join(nestedDir, "version_duplicate_b.yaml"), ` -id: duplicate-version-prompt -version: "1.0.0" -messages: - - role: user - content: Second duplicate version. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - - _, err := repo.GetPromptDefinition(ctx, "duplicate-version-prompt", "1.0.0") - if !errors.Is(err, ErrInvalidPromptDefinition) { - t.Fatalf("expected duplicate prompt version to return ErrInvalidPromptDefinition, got %v", err) - } - for _, want := range []string{"duplicate prompt definition id", "version \"1.0.0\"", "version_duplicate_a.yaml", filepath.Join("versioned", "version_duplicate_b.yaml")} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("expected error to contain %q, got %v", want, err) - } - } - }) - - t.Run("non-matching malformed nested prompt is ignored for not found lookup", func(t *testing.T) { - nestedDir := filepath.Join(tmpDir, "broken") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writePromptTestFile(t, filepath.Join(nestedDir, "unrelated.yaml"), "id: [") - - _, err := repo.GetPromptDefinition(ctx, "does-not-exist-even-with-broken-nested-file", "") - if !errors.Is(err, ErrPromptDefinitionNotFound) { - t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err) - } - }) - - t.Run("strict decode failure in nested prompt matches by YAML ID", func(t *testing.T) { - nestedDir := filepath.Join(tmpDir, "strict") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writePromptTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), ` -id: nested-strict-error -version: "1.0.0" -unknown_field: true -messages: - - role: user - content: Invalid because of unknown field. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - - _, err := repo.GetPromptDefinition(ctx, "nested-strict-error", "") - if !errors.Is(err, ErrInvalidYAML) { - t.Fatalf("expected ErrInvalidYAML, got %v", err) - } - if !strings.Contains(err.Error(), filepath.Join("strict", "not_named_like_id.yaml")) { - t.Fatalf("expected nested path in error, got %v", err) - } - }) - - t.Run("version lookup", func(t *testing.T) { - _, err := repo.GetPromptDefinition(ctx, "valid-inline", "9.9.9") - if !errors.Is(err, ErrPromptDefinitionNotFound) { - t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err) - } - }) - - cases := []struct { - name string - id string - targetErr error - errSubstrs []string - }{ - {name: "invalid YAML", id: "invalid_yaml", targetErr: ErrInvalidYAML}, - {name: "missing id", id: "missing_id", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"id is required"}}, - {name: "no messages", id: "no_messages", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"at least one message is required"}}, - {name: "both content and content_file", id: "both_content_and_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}}, - {name: "neither content nor content_file", id: "neither_content_nor_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}}, - {name: "missing content_file", id: "missing_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"failed to read content_file"}}, - {name: "duplicate input names", id: "duplicate_input_names", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"duplicate input name"}}, - {name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}}, - {name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}}, - {name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}}, - {name: "empty cache control type", id: "empty_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}}, - {name: "unsupported cache control type", id: "unsupported_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}}, - {name: "unsupported cache control ttl", id: "unsupported_cache_control_ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}}, - {name: "unknown cache control field", id: "unknown_cache_control_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - _, err := repo.GetPromptDefinition(ctx, tc.id, "") - if !errors.Is(err, tc.targetErr) { - t.Fatalf("expected %v, got %v", tc.targetErr, err) - } - for _, sub := range tc.errSubstrs { - if !strings.Contains(err.Error(), sub) { - t.Fatalf("expected error to contain %q, got %v", sub, err) - } - } - }) - } - - t.Run("prompt definition not found", func(t *testing.T) { - _, err := repo.GetPromptDefinition(ctx, "does-not-exist", "") - if !errors.Is(err, ErrPromptDefinitionNotFound) { - t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err) - } - }) -} - -func TestFSRepositoryGetPromptDefinition(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(` -id: fs-prompt -version: "1.0.0" -inputs: - - name: transcript - required: true -messages: - - role: user - content_file: ./messages/user.tmpl -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`)}, - "prompts/nested/messages/user.tmpl": &fstest.MapFile{Data: []byte(`Summarize {{input "transcript"}}.`)}, - }, "prompts") - - got, err := repo.GetPromptDefinition(context.Background(), "fs-prompt", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if got.ID != "fs-prompt" { - t.Fatalf("unexpected prompt id: %q", got.ID) - } - if len(got.Templates) != 1 || !strings.Contains(got.Templates[0].Content, `{{input "transcript"}}`) { - t.Fatalf("expected content_file body to be loaded, got %+v", got.Templates) - } - if got.Templates[0].ContentFile != "prompts/nested/messages/user.tmpl" { - t.Fatalf("unexpected content file path: %q", got.Templates[0].ContentFile) - } -} - -func TestFSRepositoryContentFileContainment(t *testing.T) { - t.Run("nested prompt can reference file inside root", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(` -id: fs-contained-prompt -version: "1.0.0" -messages: - - role: user - content_file: ../shared/user.tmpl -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`)}, - "prompts/shared/user.tmpl": &fstest.MapFile{Data: []byte(`Inside root.`)}, - }, "prompts") - - got, err := repo.GetPromptDefinition(context.Background(), "fs-contained-prompt", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(got.Templates) != 1 || got.Templates[0].Content != "Inside root." { - t.Fatalf("expected contained content file, got %+v", got.Templates) - } - }) - - tests := []struct { - name string - contentFile string - wantErr string - }{ - {name: "parent escape rejected", contentFile: "../outside.tmpl", wantErr: "escapes source root"}, - {name: "absolute path rejected", contentFile: "/outside.tmpl", wantErr: "must be relative"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "prompts/prompt.yaml": &fstest.MapFile{Data: []byte(` -id: fs-escaped-prompt -version: "1.0.0" -messages: - - role: user - content_file: ` + tc.contentFile + ` -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`)}, - "outside.tmpl": &fstest.MapFile{Data: []byte(`Outside root.`)}, - }, "prompts") - - _, err := repo.GetPromptDefinition(context.Background(), "fs-escaped-prompt", "") - if !errors.Is(err, ErrInvalidPromptDefinition) { - t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err) - } - if !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err) - } - }) - } -} - -func TestFSRepositoryRejectsDuplicatePromptIDs(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "one.yaml": &fstest.MapFile{Data: []byte(` -id: duplicate-fs-prompt -version: "1.0.0" -messages: - - role: user - content: First. -output: - format: text - validation_mode: none - repair_attempts: 0 -`)}, - "nested/two.yaml": &fstest.MapFile{Data: []byte(` -id: duplicate-fs-prompt -version: "1.0.0" -messages: - - role: user - content: Second. -output: - format: text - validation_mode: none - repair_attempts: 0 -`)}, - }, ".") - - _, err := repo.GetPromptDefinition(context.Background(), "duplicate-fs-prompt", "") - if !errors.Is(err, ErrInvalidPromptDefinition) { - t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err) - } - if !strings.Contains(err.Error(), "one.yaml") || !strings.Contains(err.Error(), "nested/two.yaml") { - t.Fatalf("expected duplicate paths in error, got %v", err) - } -} - -func TestFSRepositoryRejectsUnknownYAMLFields(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "not_named_like_id.yaml": &fstest.MapFile{Data: []byte(` -id: strict-fs-prompt -version: "1.0.0" -unknown: true -messages: - - role: user - content: Invalid. -output: - format: text - validation_mode: none - repair_attempts: 0 -`)}, - }, ".") - - _, err := repo.GetPromptDefinition(context.Background(), "strict-fs-prompt", "") - if !errors.Is(err, ErrInvalidYAML) { - t.Fatalf("expected ErrInvalidYAML, got %v", err) - } -} - -func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) { - t.Helper() - if got == nil { - t.Fatal("expected cache control, got nil") - } - if got.Type != wantType { - t.Fatalf("unexpected cache control type: got %q want %q", got.Type, wantType) - } - if got.TTL != wantTTL { - t.Fatalf("unexpected cache control ttl: got %q want %q", got.TTL, wantTTL) - } -} - -func writePromptTestFile(t *testing.T, path string, content string) { - t.Helper() - if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil { - t.Fatalf("failed to write prompt test file %q: %v", path, err) - } -} - -func copyTree(src, dst string) error { - return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - rel, err := filepath.Rel(src, path) - if err != nil { - return err - } - if rel == "." { - return nil - } - - target := filepath.Join(dst, rel) - if d.IsDir() { - return os.MkdirAll(target, 0o755) - } - - data, err := os.ReadFile(path) - if err != nil { - return err - } - return os.WriteFile(target, data, 0o644) - }) -} diff --git a/internal/promptdef/testdata/both_content_and_content_file.yaml b/internal/promptdef/testdata/both_content_and_content_file.yaml deleted file mode 100644 index de11155..0000000 --- a/internal/promptdef/testdata/both_content_and_content_file.yaml +++ /dev/null @@ -1,10 +0,0 @@ -id: both-content-and-content-file -version: "1.0.0" -messages: - - role: user - content: "Hi" - content_file: ./messages/user_prompt.tmpl -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/duplicate_input_names.yaml b/internal/promptdef/testdata/duplicate_input_names.yaml deleted file mode 100644 index 9412e4d..0000000 --- a/internal/promptdef/testdata/duplicate_input_names.yaml +++ /dev/null @@ -1,14 +0,0 @@ -id: duplicate-input-names -version: "1.0.0" -inputs: - - name: transcript - required: true - - name: transcript - required: false -messages: - - role: user - content: "Hi" -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/empty_cache_control_type.yaml b/internal/promptdef/testdata/empty_cache_control_type.yaml deleted file mode 100644 index 03330ec..0000000 --- a/internal/promptdef/testdata/empty_cache_control_type.yaml +++ /dev/null @@ -1,10 +0,0 @@ -id: empty-cache-control-type -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: {} -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/invalid_validation_mode.yaml b/internal/promptdef/testdata/invalid_validation_mode.yaml deleted file mode 100644 index 478abc5..0000000 --- a/internal/promptdef/testdata/invalid_validation_mode.yaml +++ /dev/null @@ -1,9 +0,0 @@ -id: invalid-validation-mode -version: "1.0.0" -messages: - - role: user - content: "Hi" -output: - format: text - validation_mode: nope - repair_attempts: 0 diff --git a/internal/promptdef/testdata/invalid_yaml.yaml b/internal/promptdef/testdata/invalid_yaml.yaml deleted file mode 100644 index 8cc64e8..0000000 --- a/internal/promptdef/testdata/invalid_yaml.yaml +++ /dev/null @@ -1,9 +0,0 @@ -id: invalid-yaml -version: "1.0.0" -messages: - - role: user - content: [broken -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/json_schema_without_schema_path.yaml b/internal/promptdef/testdata/json_schema_without_schema_path.yaml deleted file mode 100644 index 6dbea5b..0000000 --- a/internal/promptdef/testdata/json_schema_without_schema_path.yaml +++ /dev/null @@ -1,9 +0,0 @@ -id: json-schema-without-schema-path -version: "1.0.0" -messages: - - role: user - content: "Return JSON" -output: - format: json - validation_mode: json_schema - repair_attempts: 0 diff --git a/internal/promptdef/testdata/messages/user_prompt.tmpl b/internal/promptdef/testdata/messages/user_prompt.tmpl deleted file mode 100644 index c8fc459..0000000 --- a/internal/promptdef/testdata/messages/user_prompt.tmpl +++ /dev/null @@ -1,2 +0,0 @@ -Use transcript: -{{input "transcript"}} diff --git a/internal/promptdef/testdata/missing_content_file.yaml b/internal/promptdef/testdata/missing_content_file.yaml deleted file mode 100644 index 42e44fa..0000000 --- a/internal/promptdef/testdata/missing_content_file.yaml +++ /dev/null @@ -1,9 +0,0 @@ -id: missing-content-file -version: "1.0.0" -messages: - - role: user - content_file: ./messages/does_not_exist.tmpl -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/missing_id.yaml b/internal/promptdef/testdata/missing_id.yaml deleted file mode 100644 index 6d21ccc..0000000 --- a/internal/promptdef/testdata/missing_id.yaml +++ /dev/null @@ -1,8 +0,0 @@ -version: "1.0.0" -messages: - - role: user - content: "Hi" -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/neither_content_nor_content_file.yaml b/internal/promptdef/testdata/neither_content_nor_content_file.yaml deleted file mode 100644 index 525419b..0000000 --- a/internal/promptdef/testdata/neither_content_nor_content_file.yaml +++ /dev/null @@ -1,8 +0,0 @@ -id: neither-content-nor-content-file -version: "1.0.0" -messages: - - role: user -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/no_messages.yaml b/internal/promptdef/testdata/no_messages.yaml deleted file mode 100644 index b4b2bbf..0000000 --- a/internal/promptdef/testdata/no_messages.yaml +++ /dev/null @@ -1,6 +0,0 @@ -id: no-messages -version: "1.0.0" -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/unknown_cache_control_field.yaml b/internal/promptdef/testdata/unknown_cache_control_field.yaml deleted file mode 100644 index ac8193a..0000000 --- a/internal/promptdef/testdata/unknown_cache_control_field.yaml +++ /dev/null @@ -1,12 +0,0 @@ -id: unknown-cache-control-field -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: - type: ephemeral - unexpected: true -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/unknown_input_field.yaml b/internal/promptdef/testdata/unknown_input_field.yaml deleted file mode 100644 index d8dfa35..0000000 --- a/internal/promptdef/testdata/unknown_input_field.yaml +++ /dev/null @@ -1,13 +0,0 @@ -id: unknown-input-field -version: "1.0.0" -inputs: - - name: transcript - required: true - unknown_input_setting: true -messages: - - role: user - content: "Hi" -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/unsupported_cache_control_ttl.yaml b/internal/promptdef/testdata/unsupported_cache_control_ttl.yaml deleted file mode 100644 index d6f81d4..0000000 --- a/internal/promptdef/testdata/unsupported_cache_control_ttl.yaml +++ /dev/null @@ -1,12 +0,0 @@ -id: unsupported-cache-control-ttl -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: - type: ephemeral - ttl: 5m -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/unsupported_cache_control_type.yaml b/internal/promptdef/testdata/unsupported_cache_control_type.yaml deleted file mode 100644 index 875a9b0..0000000 --- a/internal/promptdef/testdata/unsupported_cache_control_type.yaml +++ /dev/null @@ -1,11 +0,0 @@ -id: unsupported-cache-control-type -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: - type: persistent -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/valid_cache_control_ttl.yaml b/internal/promptdef/testdata/valid_cache_control_ttl.yaml deleted file mode 100644 index 4136ac4..0000000 --- a/internal/promptdef/testdata/valid_cache_control_ttl.yaml +++ /dev/null @@ -1,14 +0,0 @@ -id: valid-cache-control-ttl -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: - type: ephemeral - ttl: 1h - - role: user - content: "Summarize the input." -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/valid_cache_control_without_ttl.yaml b/internal/promptdef/testdata/valid_cache_control_without_ttl.yaml deleted file mode 100644 index 48e6c37..0000000 --- a/internal/promptdef/testdata/valid_cache_control_without_ttl.yaml +++ /dev/null @@ -1,13 +0,0 @@ -id: valid-cache-control-without-ttl -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: - type: ephemeral - - role: user - content: "Summarize the input." -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/valid_file_backed.yaml b/internal/promptdef/testdata/valid_file_backed.yaml deleted file mode 100644 index f57f56a..0000000 --- a/internal/promptdef/testdata/valid_file_backed.yaml +++ /dev/null @@ -1,14 +0,0 @@ -id: valid-file-backed -version: "1.0.0" -inputs: - - name: transcript - required: true -messages: - - role: system - content: "Return markdown." - - role: user - content_file: ./messages/user_prompt.tmpl -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/valid_inline.yaml b/internal/promptdef/testdata/valid_inline.yaml deleted file mode 100644 index 3f61059..0000000 --- a/internal/promptdef/testdata/valid_inline.yaml +++ /dev/null @@ -1,18 +0,0 @@ -id: valid-inline -version: "1.0.0" -inputs: - - name: transcript - required: true - content_type: text/markdown - description: Transcript content -messages: - - role: system - content: "You are concise." - - role: user - content: | - Summarize: - {{input "transcript"}} -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/valid_session_id.yaml b/internal/promptdef/testdata/valid_session_id.yaml deleted file mode 100644 index b477662..0000000 --- a/internal/promptdef/testdata/valid_session_id.yaml +++ /dev/null @@ -1,10 +0,0 @@ -id: valid-session-id -version: "1.0.0" -session_id: " {{ .session_id }} " -messages: - - role: user - content: Hello. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/with_default_profile.yaml b/internal/promptdef/testdata/with_default_profile.yaml deleted file mode 100644 index 1628b41..0000000 --- a/internal/promptdef/testdata/with_default_profile.yaml +++ /dev/null @@ -1,13 +0,0 @@ -id: with-default-profile -version: "1.0.0" -default_profile: local-default -inputs: - - name: transcript - required: true -messages: - - role: user - content: "Write output" -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/usecase/repairer.go b/internal/usecase/repairer.go deleted file mode 100644 index 475878e..0000000 --- a/internal/usecase/repairer.go +++ /dev/null @@ -1,76 +0,0 @@ -package usecase - -import ( - "context" - "errors" - "fmt" - "strings" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" - "gitea.maximumdirect.net/eric/scriptorium/internal/llm" -) - -type OutputRepairer interface { - Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) -} - -type RepairRequest struct { - PreviousOutput string - ValidationErrors []string - Target domain.ExecutionTarget - StructuredOutput *domain.StructuredOutputSpec - Attempt int - MaxAttempts int - Mode domain.ValidationMode -} - -type defaultOutputRepairer struct { - llm llm.Client -} - -func NewDefaultOutputRepairer(llmClient llm.Client) OutputRepairer { - return &defaultOutputRepairer{llm: llmClient} -} - -func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) { - if r.llm == nil { - return nil, errors.New("llm client is required for repair") - } - - errs := "(none provided)" - if len(req.ValidationErrors) > 0 { - errs = strings.Join(req.ValidationErrors, "\n") - } - - prompt := domain.RenderedPrompt{Messages: []domain.RenderedMessage{ - { - Role: "system", - Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.", - }, - { - Role: "user", - Content: fmt.Sprintf( - "Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.", - req.Attempt, - req.MaxAttempts, - req.Mode, - errs, - req.PreviousOutput, - ), - }, - }} - - resp, err := r.llm.Generate(ctx, domain.GenerateRequest{ - Prompt: prompt, - Target: req.Target, - StructuredOutput: req.StructuredOutput, - }) - if err != nil { - return nil, err - } - if resp == nil { - return nil, errors.New("repair llm returned nil response") - } - - return resp, nil -} diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go deleted file mode 100644 index c7dc40c..0000000 --- a/internal/usecase/runner.go +++ /dev/null @@ -1,576 +0,0 @@ -package usecase - -import ( - "context" - "crypto/rand" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "os" - "strings" - "time" - "unicode" - - "gitea.maximumdirect.net/eric/scriptorium/internal/artifact" - "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" - "gitea.maximumdirect.net/eric/scriptorium/internal/llm" - "gitea.maximumdirect.net/eric/scriptorium/internal/profile" - "gitea.maximumdirect.net/eric/scriptorium/internal/prompt" - "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" - "gitea.maximumdirect.net/eric/scriptorium/internal/validate" -) - -var ( - ErrInvalidRequest = errors.New("invalid run request") - ErrProfileRequired = errors.New("profile selection is required") - ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable") - ErrAPIKeyRequired = errors.New("api key is required") - ErrPromptLoad = errors.New("failed to load prompt definition") - ErrProfileLoad = errors.New("failed to load execution profile") - ErrArtifactLoad = errors.New("failed to load artifact") - ErrPromptRender = errors.New("failed to render prompt") - ErrLLMGenerate = errors.New("failed to generate output") - ErrValidation = errors.New("failed to validate output") -) - -// Runner executes the Scriptorium core use case. -type Runner struct { - promptDefs promptdef.Repository - profiles profile.Repository - artifacts artifact.Reader - renderer prompt.Renderer - llm llm.Client - validator validate.Validator - repairer OutputRepairer -} - -func NewRunner( - promptDefs promptdef.Repository, - profiles profile.Repository, - artifacts artifact.Reader, - renderer prompt.Renderer, - llmClient llm.Client, - validator validate.Validator, -) *Runner { - return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil) -} - -func NewRunnerWithRepairer( - promptDefs promptdef.Repository, - profiles profile.Repository, - artifacts artifact.Reader, - renderer prompt.Renderer, - llmClient llm.Client, - validator validate.Validator, - repairer OutputRepairer, -) *Runner { - return &Runner{ - promptDefs: promptDefs, - profiles: profiles, - artifacts: artifacts, - renderer: renderer, - llm: llmClient, - validator: validator, - repairer: repairer, - } -} - -func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) { - runID, err := newRunID() - if err != nil { - return nil, fmt.Errorf("failed to create run id: %w", err) - } - - start := time.Now().UTC() - - prepared, err := r.Prepare(ctx, req) - if err != nil { - return nil, err - } - - genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{ - Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages}, - Target: prepared.EffectiveModelParams, - TargetPresence: prepared.TargetPresence, - StructuredOutput: prepared.StructuredOutput, - }) - if err != nil { - if errors.Is(err, llm.ErrInvalidRequest) { - return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) - } - return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err) - } - - outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format) - validationResult, err := r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, 0) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrValidation, err) - } - - if r.shouldAttemptRepair(prepared.OutputContract, validationResult) { - attemptsUsed := 0 - for attemptsUsed < prepared.OutputContract.RepairAttempts && validationResult.Status == domain.ValidationFailed { - attemptsUsed++ - - repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{ - PreviousOutput: genResp.Content, - ValidationErrors: validationResult.Errors, - Target: prepared.EffectiveModelParams, - StructuredOutput: prepared.StructuredOutput, - Attempt: attemptsUsed, - MaxAttempts: prepared.OutputContract.RepairAttempts, - Mode: prepared.OutputContract.ValidationMode, - }) - if repairErr != nil { - return nil, fmt.Errorf("%w: %w", ErrValidation, repairErr) - } - if repairResp == nil { - return nil, fmt.Errorf("%w: repairer returned nil response", ErrValidation) - } - - genResp = repairResp - outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format) - - validationResult, err = r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, attemptsUsed) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrValidation, err) - } - } - } - - end := time.Now().UTC() - - return &domain.RunResult{ - RunID: runID, - Artifact: outputArtifact, - RawOutput: genResp.Content, - Validation: validationResult, - PromptID: prepared.PromptID, - PromptVersion: prepared.PromptVersion, - PromptHash: prepared.PromptHash, - RenderedPromptHash: prepared.RenderedPromptHash, - SelectedProfileID: prepared.SelectedProfileID, - ModelName: prepared.EffectiveModelParams.Model, - Endpoint: prepared.EffectiveModelParams.Endpoint, - EffectiveModelParams: prepared.EffectiveModelParams, - InputHashes: prepared.InputHashes, - Usage: genResp.Usage, - StartTime: start, - EndTime: end, - Duration: end.Sub(start), - }, nil -} - -func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.PreparedRun, error) { - if strings.TrimSpace(req.PromptID) == "" { - return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest) - } - - start := time.Now().UTC() - - def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) - } - promptDefinitionHash, err := hashPromptDefinition(def) - if err != nil { - return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err) - } - - selectedProfileID := strings.TrimSpace(req.ProfileID) - if selectedProfileID == "" { - selectedProfileID = strings.TrimSpace(def.DefaultProfile) - } - if selectedProfileID == "" { - return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired) - } - - execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) - } - - effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) - } - effectiveModel.APIKey = req.APIKey - if strings.TrimSpace(effectiveModel.Endpoint) == "" { - return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest) - } - if strings.TrimSpace(effectiveModel.Model) == "" { - return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest) - } - if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) - } - - effectiveContract := resolveOutputContract(def, req.Validation) - structuredOutput, err := r.resolveStructuredOutput(ctx, def, effectiveContract) - if err != nil { - return nil, err - } - - resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs)) - inputHashes := make(map[string]string, len(req.Inputs)) - for name, ref := range req.Inputs { - art, readErr := r.artifacts.Read(ctx, ref) - if readErr != nil { - return nil, fmt.Errorf("%w: input %q: %w", ErrArtifactLoad, name, readErr) - } - if art.Name == "" { - art.Name = name - } - resolvedInputs[name] = art - inputHashes[name] = art.Hash - } - - renderedPrompt, err := r.renderer.Render(ctx, def, resolvedInputs, req.Vars) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrPromptRender, err) - } - - end := time.Now().UTC() - return &domain.PreparedRun{ - PromptID: def.ID, - PromptVersion: def.Version, - PromptHash: promptDefinitionHash, - SelectedProfileID: selectedProfileID, - EffectiveModelParams: effectiveModel, - TargetPresence: targetPresence, - OutputContract: effectiveContract, - StructuredOutput: structuredOutput, - InputHashes: inputHashes, - SessionID: renderedPrompt.SessionID, - RenderedPromptHash: hashRenderedPrompt(*renderedPrompt), - Messages: renderedPrompt.Messages, - StartTime: start, - EndTime: end, - DurationMS: end.Sub(start).Milliseconds(), - }, nil -} - -func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) { - if contract.ValidationMode != domain.ValidationJSONSchema { - return nil, nil - } - - loader, ok := r.validator.(validate.SchemaDocumentLoader) - if !ok || loader == nil { - return nil, fmt.Errorf("%w: json_schema output requires schema document loader", ErrValidation) - } - - schemaDoc, err := loader.LoadSchemaDocument(ctx, contract.SchemaPath) - if err != nil { - return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err) - } - - return &domain.StructuredOutputSpec{ - Type: domain.StructuredOutputJSONSchema, - JSONSchema: &domain.StructuredOutputJSONSpec{ - Name: deriveStructuredSchemaName(def.ID, def.Version), - Strict: true, - Schema: schemaDoc, - }, - }, nil -} - -func deriveStructuredSchemaName(promptID string, promptVersion string) string { - raw := strings.TrimSpace(promptID) - if v := strings.TrimSpace(promptVersion); v != "" { - if raw == "" { - raw = v - } else { - raw = raw + "_" + v - } - } - - var b strings.Builder - for _, r := range raw { - if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == '-' { - b.WriteRune(r) - } else { - b.WriteRune('_') - } - } - - name := strings.Trim(b.String(), "_-") - if name == "" { - return "scriptorium_schema" - } - return name -} - -func (r *Runner) validateOutput(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, attemptsUsed int) (domain.ValidationResult, error) { - if r.validator == nil || contract.ValidationMode == domain.ValidationNone { - return domain.ValidationResult{ - Status: domain.ValidationSkipped, - Mode: contract.ValidationMode, - SchemaPath: contract.SchemaPath, - RepairAttempts: attemptsUsed, - IsValid: true, - }, nil - } - - res, err := r.validator.Validate(ctx, artifact, contract) - if err != nil { - return domain.ValidationResult{}, err - } - res.RepairAttempts = attemptsUsed - return res, nil -} - -func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationResult domain.ValidationResult) bool { - if r.repairer == nil { - return false - } - if contract.RepairAttempts <= 0 { - return false - } - if validationResult.Status != domain.ValidationFailed { - return false - } - return contract.ValidationMode == domain.ValidationJSON || contract.ValidationMode == domain.ValidationJSONSchema -} - -func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget { - out := base - if override.Endpoint != "" { - out.Endpoint = override.Endpoint - } - if override.Model != "" { - out.Model = override.Model - } - if override.Temperature != 0 { - out.Temperature = override.Temperature - } - if override.MaxTokens != 0 { - out.MaxTokens = override.MaxTokens - } - if override.TopP != 0 { - out.TopP = override.TopP - } - if override.TimeoutSeconds != 0 { - out.TimeoutSeconds = override.TimeoutSeconds - } - if strings.TrimSpace(override.ServiceTier) != "" { - out.ServiceTier = override.ServiceTier - } - if strings.TrimSpace(override.ReasoningEffort) != "" { - out.ReasoningEffort = override.ReasoningEffort - } - if strings.TrimSpace(override.APIKeyEnv) != "" { - out.APIKeyEnv = override.APIKeyEnv - } - if override.APIKeyRequired { - out.APIKeyRequired = true - } - if len(override.ExtraParams) > 0 { - out.ExtraParams = copyExtraParams(override.ExtraParams) - } - return out -} - -func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) { - out := base - var presence domain.ExecutionTargetPresence - if override.Endpoint != "" { - out.Endpoint = override.Endpoint - } - if override.Model != "" { - out.Model = override.Model - } - if override.Temperature != nil { - if *override.Temperature < 0 || *override.Temperature > 2 { - return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("temperature must be between 0 and 2") - } - out.Temperature = *override.Temperature - presence.Temperature = true - } - if override.MaxTokens != nil { - if *override.MaxTokens < 0 { - return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("max_tokens must be greater than or equal to 0") - } - out.MaxTokens = *override.MaxTokens - presence.MaxTokens = true - } - if override.TopP != nil { - if *override.TopP < 0 || *override.TopP > 1 { - return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("top_p must be between 0 and 1") - } - out.TopP = *override.TopP - presence.TopP = true - } - if override.TimeoutSeconds != nil { - if *override.TimeoutSeconds < 0 { - return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, errors.New("timeout_seconds must be greater than or equal to 0") - } - out.TimeoutSeconds = *override.TimeoutSeconds - presence.TimeoutSeconds = true - } - if strings.TrimSpace(override.ServiceTier) != "" { - out.ServiceTier = override.ServiceTier - } - if strings.TrimSpace(override.ReasoningEffort) != "" { - out.ReasoningEffort = override.ReasoningEffort - } - if strings.TrimSpace(override.APIKeyEnv) != "" { - out.APIKeyEnv = override.APIKeyEnv - } - if len(override.ExtraParams) > 0 { - out.ExtraParams = copyExtraParams(override.ExtraParams) - } - return out, presence, nil -} - -func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) { - out := defaults.ExecutionTargetDefault() - out = mergeExecutionTarget(out, executionProfileToTarget(profileValue)) - var presence domain.ExecutionTargetPresence - if override != nil { - var err error - out, presence, err = mergeExecutionTargetOverride(out, *override) - if err != nil { - return domain.ExecutionTarget{}, domain.ExecutionTargetPresence{}, err - } - } - return out, presence, nil -} - -func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error { - if strings.TrimSpace(apiKey) != "" { - return nil - } - envName := strings.TrimSpace(apiKeyEnv) - if envName == "" { - if apiKeyRequired { - return ErrAPIKeyRequired - } - return nil - } - if strings.TrimSpace(os.Getenv(envName)) == "" { - return fmt.Errorf("%w: api key environment variable %q is not set", ErrAPIKeyEnvMissing, envName) - } - return nil -} - -func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget { - if p == nil { - return domain.ExecutionTarget{} - } - return domain.ExecutionTarget{ - Endpoint: p.Endpoint, - Model: p.Model, - Temperature: p.Temperature, - MaxTokens: p.MaxTokens, - TopP: p.TopP, - TimeoutSeconds: p.TimeoutSeconds, - ServiceTier: p.ServiceTier, - ReasoningEffort: p.ReasoningEffort, - APIKeyEnv: p.APIKeyEnv, - APIKeyRequired: p.APIKeyRequired, - ExtraParams: copyExtraParams(p.ExtraParams), - } -} - -func copyExtraParams(src map[string]any) map[string]any { - if len(src) == 0 { - return nil - } - cp := make(map[string]any, len(src)) - for k, v := range src { - cp[k] = v - } - return cp -} - -func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract { - contract := def.Validation - if contract.Format == "" { - contract.Format = def.OutputFormat - } - if override != nil { - contract = *override - } - if contract.Format == "" { - contract.Format = domain.FormatText - } - return contract -} - -func hashRenderedPrompt(p domain.RenderedPrompt) string { - var b strings.Builder - if p.SessionID != "" { - b.WriteString("session_id=") - b.WriteString(p.SessionID) - b.WriteString("\n---\n") - } - for _, msg := range p.Messages { - b.WriteString(msg.Role) - b.WriteByte('\n') - b.WriteString(msg.Content) - if msg.CacheControl != nil { - b.WriteString("\ncache_control.type=") - b.WriteString(string(msg.CacheControl.Type)) - if msg.CacheControl.TTL != "" { - b.WriteString("\ncache_control.ttl=") - b.WriteString(msg.CacheControl.TTL) - } - } - b.WriteString("\n---\n") - } - h := sha256.Sum256([]byte(b.String())) - return hex.EncodeToString(h[:]) -} - -func buildOutputArtifact(content string, format domain.OutputFormat) domain.Artifact { - body := []byte(content) - hash := sha256.Sum256(body) - - contentType := defaults.ContentTypeTextPlain - switch format { - case domain.FormatMarkdown: - contentType = defaults.ContentTypeTextMarkdown - case domain.FormatJSON: - contentType = defaults.ContentTypeApplicationJSON - } - - return domain.Artifact{ - Name: defaults.OutputArtifactName, - ContentType: contentType, - Body: body, - Size: int64(len(body)), - Hash: hex.EncodeToString(hash[:]), - } -} - -func hashPromptDefinition(def *domain.PromptDefinition) (string, error) { - b, err := json.Marshal(def) - if err != nil { - return "", err - } - sum := sha256.Sum256(b) - return hex.EncodeToString(sum[:]), nil -} - -func newRunID() (string, error) { - var b [16]byte - if _, err := rand.Read(b[:]); err != nil { - return "", err - } - - // UUID v4 (RFC 4122 variant). - b[6] = (b[6] & 0x0f) | 0x40 - b[8] = (b[8] & 0x3f) | 0x80 - - return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", - b[0:4], - b[4:6], - b[6:8], - b[8:10], - b[10:16], - ), nil -} diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go deleted file mode 100644 index 2ef2af8..0000000 --- a/internal/usecase/runner_test.go +++ /dev/null @@ -1,1810 +0,0 @@ -package usecase - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "path/filepath" - "reflect" - "regexp" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" - "gitea.maximumdirect.net/eric/scriptorium/internal/llm" - "gitea.maximumdirect.net/eric/scriptorium/internal/profile" - "gitea.maximumdirect.net/eric/scriptorium/internal/prompt" - "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" - "gitea.maximumdirect.net/eric/scriptorium/internal/validate" -) - -type fakePromptRepo struct { - def *domain.PromptDefinition - err error - lastID string - lastVersion string -} - -type fakeExecutionProfileRepo struct { - profiles map[string]*domain.ExecutionProfile - err error - lastID string -} - -func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - f.lastID = id - if f.err != nil { - return nil, f.err - } - if p, ok := f.profiles[id]; ok { - cp := *p - return &cp, nil - } - return nil, errors.New("profile not found") -} - -func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) { - f.lastID = id - f.lastVersion = version - if f.err != nil { - return nil, f.err - } - return f.def, nil -} - -type fakeArtifactReader struct { - artifactsByURI map[string]*domain.Artifact - errByURI map[string]error -} - -func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { - if err, ok := f.errByURI[ref.URI]; ok { - return nil, err - } - if art, ok := f.artifactsByURI[ref.URI]; ok { - cp := *art - return &cp, nil - } - return nil, errors.New("artifact not found") -} - -type fakeRenderer struct { - rendered *domain.RenderedPrompt - err error -} - -func (f *fakeRenderer) Render(ctx context.Context, def *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) { - if f.err != nil { - return nil, f.err - } - return f.rendered, nil -} - -type fakeLLM struct { - resp *domain.GenerateResponse - err error - lastReq domain.GenerateRequest - calls int - forbid bool -} - -func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { - f.calls++ - f.lastReq = req - if f.forbid { - return nil, errors.New("llm should not be called") - } - if f.err != nil { - return nil, f.err - } - return f.resp, nil -} - -type fakeValidator struct { - result domain.ValidationResult - err error - schemaDoc any - schemaErr error - schemaLoadPath string - schemaLoads int -} - -func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { - if f.err != nil { - return domain.ValidationResult{}, f.err - } - return f.result, nil -} - -func (f *fakeValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { - f.schemaLoads++ - f.schemaLoadPath = schemaPath - if f.schemaErr != nil { - return nil, f.schemaErr - } - if f.schemaDoc != nil { - return f.schemaDoc, nil - } - return map[string]any{"type": "object"}, nil -} - -type fakeRepairer struct { - responses []*domain.GenerateResponse - err error - calls int - reqs []RepairRequest -} - -func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) { - f.calls++ - f.reqs = append(f.reqs, req) - if f.err != nil { - return nil, f.err - } - if len(f.responses) == 0 { - return nil, errors.New("no repair response configured") - } - idx := f.calls - 1 - if idx >= len(f.responses) { - idx = len(f.responses) - 1 - } - return f.responses[idx], nil -} - -func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} - reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ - "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, - "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, - }} - renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} - llmClient := &fakeLLM{forbid: true} - - runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) - prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - PromptVersion: "1", - ProfileID: "exec", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, - "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, - }, - Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if prepared.PromptID != "p" || prepared.PromptVersion != "1" { - t.Fatalf("unexpected prepared prompt metadata: %+v", prepared) - } - if prepared.SelectedProfileID != "exec" { - t.Fatalf("expected selected profile exec, got %q", prepared.SelectedProfileID) - } - if prepared.PromptHash == "" || prepared.RenderedPromptHash == "" { - t.Fatal("expected prompt hashes") - } - if prepared.EffectiveModelParams.Model != "m" || prepared.EffectiveModelParams.Endpoint != "http://override/v1" { - t.Fatalf("unexpected model params: %+v", prepared.EffectiveModelParams) - } - if prepared.OutputContract.Format != domain.FormatMarkdown { - t.Fatalf("expected output format markdown, got %q", prepared.OutputContract.Format) - } - if len(prepared.InputHashes) != 2 || prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" { - t.Fatalf("expected input hashes, got %#v", prepared.InputHashes) - } - if len(prepared.Messages) != 2 { - t.Fatalf("expected two messages, got %d", len(prepared.Messages)) - } - if prepared.SessionID != "session-123" { - t.Fatalf("expected prepared session id, got %q", prepared.SessionID) - } - if llmClient.calls != 0 { - t.Fatalf("prepare should not call llm, calls=%d", llmClient.calls) - } -} - -func TestRunnerPrepareUsesPromptDefaultProfileWhenNoExplicitProfileID(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - promptRepo.def.DefaultProfile = "from-prompt" - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "from-prompt": {ID: "from-prompt", Endpoint: "http://llm/v1", Model: "m"}, - }} - - runner := newMinimalRunner(promptRepo, execRepo) - prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if execRepo.lastID != "from-prompt" { - t.Fatalf("expected prompt default profile lookup, got %q", execRepo.lastID) - } - if prepared.SelectedProfileID != "from-prompt" { - t.Fatalf("expected selected profile from-prompt, got %q", prepared.SelectedProfileID) - } -} - -func TestRunnerPrepareMissingExplicitProfileAndMissingDefaultProfileFails(t *testing.T) { - repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - repo.def.DefaultProfile = "" - runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}) - _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()}) - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if !errors.Is(err, ErrProfileRequired) { - t.Fatalf("expected ErrProfileRequired, got %v", err) - } -} - -func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) { - repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - repo.def.DefaultProfile = "does-not-exist" - runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{}}) - _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()}) - if !errors.Is(err, ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } -} - -func TestRunnerPreparePromptLoadFailure(t *testing.T) { - runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil) - _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"}) - if !errors.Is(err, ErrPromptLoad) { - t.Fatalf("expected ErrPromptLoad, got %v", err) - } - if errors.Is(err, ErrProfileLoad) { - t.Fatalf("did not expect ErrProfileLoad, got %v", err) - } -} - -func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.2, - MaxTokens: 500, - TopP: 0.9, - TimeoutSeconds: 120, - ServiceTier: "priority", - }, - }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) - - prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - Execution: &domain.ExecutionTargetOverride{ - Endpoint: "http://override/v1", - Model: "override-model", - Temperature: float64Ptr(0.7), - TimeoutSeconds: intPtr(30), - ServiceTier: "flex", - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if prepared.EffectiveModelParams.Endpoint != "http://override/v1" || prepared.EffectiveModelParams.Model != "override-model" { - t.Fatalf("expected endpoint/model override to win, got %+v", prepared.EffectiveModelParams) - } - if prepared.EffectiveModelParams.TopP != 0.9 { - t.Fatalf("expected profile top_p to remain, got %v", prepared.EffectiveModelParams.TopP) - } - if prepared.EffectiveModelParams.ServiceTier != "flex" { - t.Fatalf("expected service_tier override to win, got %q", prepared.EffectiveModelParams.ServiceTier) - } -} - -func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) { - tests := []struct { - name string - override *domain.ExecutionTargetOverride - wantTemperature float64 - wantMaxTokens int - wantTopP float64 - wantTimeoutSecs int - wantPresence domain.ExecutionTargetPresence - }{ - { - name: "omitted preserves profile values", - override: &domain.ExecutionTargetOverride{}, - wantTemperature: 0.7, - wantMaxTokens: 321, - wantTopP: 0.8, - wantTimeoutSecs: 45, - }, - { - name: "explicit zero temperature", - override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(0)}, - wantTemperature: 0, - wantMaxTokens: 321, - wantTopP: 0.8, - wantTimeoutSecs: 45, - wantPresence: domain.ExecutionTargetPresence{Temperature: true}, - }, - { - name: "explicit zero max tokens", - override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(0)}, - wantTemperature: 0.7, - wantMaxTokens: 0, - wantTopP: 0.8, - wantTimeoutSecs: 45, - wantPresence: domain.ExecutionTargetPresence{MaxTokens: true}, - }, - { - name: "explicit zero top p", - override: &domain.ExecutionTargetOverride{TopP: float64Ptr(0)}, - wantTemperature: 0.7, - wantMaxTokens: 321, - wantTopP: 0, - wantTimeoutSecs: 45, - wantPresence: domain.ExecutionTargetPresence{TopP: true}, - }, - { - name: "explicit zero timeout", - override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(0)}, - wantTemperature: 0.7, - wantMaxTokens: 321, - wantTopP: 0.8, - wantTimeoutSecs: 0, - wantPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - runner := NewRunner( - &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.7, - MaxTokens: 321, - TopP: 0.8, - TimeoutSeconds: 45, - }, - }}, - defaultArtifactReader(), - defaultRenderer(), - &fakeLLM{forbid: true}, - nil, - ) - - prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - Execution: tc.override, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - got := prepared.EffectiveModelParams - if got.Temperature != tc.wantTemperature || - got.MaxTokens != tc.wantMaxTokens || - got.TopP != tc.wantTopP || - got.TimeoutSeconds != tc.wantTimeoutSecs { - t.Fatalf("unexpected effective numeric settings: %+v", got) - } - if prepared.TargetPresence != tc.wantPresence { - t.Fatalf("unexpected target presence: got %+v want %+v", prepared.TargetPresence, tc.wantPresence) - } - }) - } -} - -func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) { - tests := []struct { - name string - override *domain.ExecutionTargetOverride - }{ - {name: "temperature below range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(-0.1)}}, - {name: "temperature above range", override: &domain.ExecutionTargetOverride{Temperature: float64Ptr(2.1)}}, - {name: "max tokens below range", override: &domain.ExecutionTargetOverride{MaxTokens: intPtr(-1)}}, - {name: "top p below range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(-0.1)}}, - {name: "top p above range", override: &domain.ExecutionTargetOverride{TopP: float64Ptr(1.1)}}, - {name: "timeout below range", override: &domain.ExecutionTargetOverride{TimeoutSeconds: intPtr(-1)}}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - runner := NewRunner( - &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - defaultRenderer(), - &fakeLLM{forbid: true}, - nil, - ) - - _, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - Execution: tc.override, - }) - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - }) - } -} - -func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - TopP: 0.8, - TimeoutSeconds: 90, - ServiceTier: "priority", - }, - }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) - - prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if prepared.EffectiveModelParams.TopP != 0.8 { - t.Fatalf("expected profile top_p to beat default, got %v", prepared.EffectiveModelParams.TopP) - } - if prepared.EffectiveModelParams.TimeoutSeconds != 90 { - t.Fatalf("expected profile timeout to beat default, got %d", prepared.EffectiveModelParams.TimeoutSeconds) - } - if prepared.EffectiveModelParams.ServiceTier != "priority" { - t.Fatalf("expected profile service_tier to beat default, got %q", prepared.EffectiveModelParams.ServiceTier) - } -} - -func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) { - promptDir := filepath.Join("..", "promptdef", "testdata") - profileDir := filepath.Join("..", "profile", "testdata") - - reader := &fakeArtifactReader{ - artifactsByURI: map[string]*domain.Artifact{ - "a://transcript": { - Name: "transcript", - Body: []byte("Session transcript body."), - Hash: hashString("Session transcript body."), - }, - }, - } - llmClient := &fakeLLM{forbid: true} - runner := NewRunner( - promptdef.NewFilesystemRepository(promptDir), - profile.NewFilesystemRepository(profileDir), - reader, - prompt.NewGoRenderer(), - llmClient, - nil, - ) - - prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "valid-file-backed", - ProfileID: "local-default", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://transcript"}, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(prepared.Messages) != 2 { - t.Fatalf("expected two rendered messages, got %d", len(prepared.Messages)) - } - if !strings.Contains(prepared.Messages[1].Content, "Session transcript body.") { - t.Fatalf("expected file-backed template content to render input, got %q", prepared.Messages[1].Content) - } -} - -func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) { - def := promptDef(domain.FormatText, domain.ValidationNone, 0) - def.Templates = []domain.PromptMessageTemplate{{Role: "user", Content: `{{input "transcript"}}`}} - - runner := NewRunner( - &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - prompt.NewGoRenderer(), - &fakeLLM{forbid: true}, - nil, - ) - _, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: map[string]domain.ArtifactRef{}, - }) - if !errors.Is(err, ErrPromptRender) { - t.Fatalf("expected ErrPromptRender, got %v", err) - } - if !errors.Is(err, prompt.ErrMissingRequiredInput) { - t.Fatalf("expected ErrMissingRequiredInput, got %v", err) - } -} - -func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) { - def := promptDef(domain.FormatText, domain.ValidationNone, 0) - def.Inputs = []domain.PromptInput{{Name: "transcript", Required: false}} - def.Templates = []domain.PromptMessageTemplate{{Role: "user", Content: `{{input "ghost"}}`}} - - runner := NewRunner( - &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - prompt.NewGoRenderer(), - &fakeLLM{forbid: true}, - nil, - ) - _, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: map[string]domain.ArtifactRef{}, - }) - if !errors.Is(err, ErrPromptRender) { - t.Fatalf("expected ErrPromptRender, got %v", err) - } - if !errors.Is(err, prompt.ErrUnknownInput) { - t.Fatalf("expected ErrUnknownInput, got %v", err) - } -} - -func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) { - const envName = "SCRIPTORIUM_TEST_API_KEY" - const secret = "top-secret-value" - t.Setenv(envName, secret) - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, - }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) - - prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if prepared.EffectiveModelParams.APIKeyEnv != envName { - t.Fatalf("expected api key env name, got %q", prepared.EffectiveModelParams.APIKeyEnv) - } - metadataDump := fmt.Sprintf("%+v|%s|%s", prepared.EffectiveModelParams, prepared.PromptHash, prepared.RenderedPromptHash) - if strings.Contains(metadataDump, secret) { - t.Fatalf("unexpected api key value in prepared metadata dump: %s", metadataDump) - } -} - -func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) { - def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0) - def.Validation.SchemaPath = "events.schema.json" - validator := &fakeValidator{ - schemaDoc: map[string]any{ - "type": "object", - "properties": map[string]any{ - "events": map[string]any{"type": "array"}, - }, - }, - } - runner := NewRunner( - &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - defaultRenderer(), - &fakeLLM{forbid: true}, - validator, - ) - - prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if validator.schemaLoads != 1 { - t.Fatalf("expected one schema load, got %d", validator.schemaLoads) - } - if validator.schemaLoadPath != "events.schema.json" { - t.Fatalf("expected schema path events.schema.json, got %q", validator.schemaLoadPath) - } - if prepared.StructuredOutput == nil { - t.Fatal("expected structured output spec") - } - if prepared.StructuredOutput.Type != domain.StructuredOutputJSONSchema { - t.Fatalf("expected structured output type json_schema, got %q", prepared.StructuredOutput.Type) - } - if prepared.StructuredOutput.JSONSchema == nil { - t.Fatal("expected structured output json_schema payload") - } - if prepared.StructuredOutput.JSONSchema.Name != "p_1" { - t.Fatalf("expected derived schema name p_1, got %q", prepared.StructuredOutput.JSONSchema.Name) - } - if prepared.StructuredOutput.JSONSchema.Strict != true { - t.Fatalf("expected strict=true, got %v", prepared.StructuredOutput.JSONSchema.Strict) - } -} - -func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testing.T) { - def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0) - def.Validation.SchemaPath = "missing.schema.json" - validator := &fakeValidator{schemaErr: errors.New("schema unavailable")} - runner := NewRunner( - &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - defaultRenderer(), - &fakeLLM{forbid: true}, - validator, - ) - - _, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if !errors.Is(err, ErrValidation) { - t.Fatalf("expected ErrValidation, got %v", err) - } - if validator.schemaLoads != 1 { - t.Fatalf("expected one schema load attempt, got %d", validator.schemaLoads) - } - if validator.schemaLoadPath != "missing.schema.json" { - t.Fatalf("expected schema path missing.schema.json, got %q", validator.schemaLoadPath) - } -} - -func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) { - def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0) - def.Validation.SchemaPath = "missing.schema.json" - llmClient := &fakeLLM{forbid: true} - validator := &fakeValidator{schemaErr: errors.New("schema unavailable")} - runner := NewRunner( - &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - defaultRenderer(), - llmClient, - validator, - ) - - _, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if !errors.Is(err, ErrValidation) { - t.Fatalf("expected ErrValidation, got %v", err) - } - if llmClient.calls != 0 { - t.Fatalf("expected llm not called when schema loading fails, calls=%d", llmClient.calls) - } -} - -func TestDeriveStructuredSchemaName(t *testing.T) { - tests := []struct { - name string - id string - version string - want string - }{ - { - name: "sanitizes punctuation and keeps dashes", - id: "prompt.id/alpha", - version: "1.0.0-beta", - want: "prompt_id_alpha_1_0_0-beta", - }, - { - name: "fallback when empty", - id: "", - version: "", - want: "scriptorium_schema", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := deriveStructuredSchemaName(tc.id, tc.version) - if got != tc.want { - t.Fatalf("expected %q, got %q", tc.want, got) - } - }) - } -} - -func TestHashRenderedPromptIncludesCacheControlWhenPresent(t *testing.T) { - uncached := domain.RenderedPrompt{Messages: []domain.RenderedMessage{ - {Role: "system", Content: "sys"}, - {Role: "user", Content: "usr"}, - }} - wantLegacyHash := hashString("system\nsys\n---\nuser\nusr\n---\n") - if got := hashRenderedPrompt(uncached); got != wantLegacyHash { - t.Fatalf("expected no-cache hash to preserve legacy input, got %q want %q", got, wantLegacyHash) - } - - withCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{ - { - Role: "system", - Content: "sys", - CacheControl: &domain.CacheControl{ - Type: domain.CacheControlEphemeral, - TTL: "1h", - }, - }, - {Role: "user", Content: "usr"}, - }} - alsoWithCache := domain.RenderedPrompt{Messages: []domain.RenderedMessage{ - { - Role: "system", - Content: "sys", - CacheControl: &domain.CacheControl{ - Type: domain.CacheControlEphemeral, - TTL: "1h", - }, - }, - {Role: "user", Content: "usr"}, - }} - withoutTTL := domain.RenderedPrompt{Messages: []domain.RenderedMessage{ - { - Role: "system", - Content: "sys", - CacheControl: &domain.CacheControl{ - Type: domain.CacheControlEphemeral, - }, - }, - {Role: "user", Content: "usr"}, - }} - - cachedHash := hashRenderedPrompt(withCache) - if cachedHash == hashRenderedPrompt(uncached) { - t.Fatal("expected cache control to change rendered prompt hash") - } - if cachedHash != hashRenderedPrompt(alsoWithCache) { - t.Fatal("expected identical cache control metadata to produce stable hash") - } - if cachedHash == hashRenderedPrompt(withoutTTL) { - t.Fatal("expected ttl changes to affect rendered prompt hash") - } -} - -func TestHashRenderedPromptIncludesSessionIDWhenPresent(t *testing.T) { - withoutSession := domain.RenderedPrompt{Messages: []domain.RenderedMessage{ - {Role: "system", Content: "sys"}, - {Role: "user", Content: "usr"}, - }} - withSession := domain.RenderedPrompt{ - SessionID: "session-123", - Messages: []domain.RenderedMessage{ - {Role: "system", Content: "sys"}, - {Role: "user", Content: "usr"}, - }, - } - alsoWithSession := domain.RenderedPrompt{ - SessionID: "session-123", - Messages: []domain.RenderedMessage{ - {Role: "system", Content: "sys"}, - {Role: "user", Content: "usr"}, - }, - } - otherSession := domain.RenderedPrompt{ - SessionID: "session-456", - Messages: []domain.RenderedMessage{ - {Role: "system", Content: "sys"}, - {Role: "user", Content: "usr"}, - }, - } - - sessionHash := hashRenderedPrompt(withSession) - if sessionHash == hashRenderedPrompt(withoutSession) { - t.Fatal("expected session_id to change rendered prompt hash") - } - if sessionHash != hashRenderedPrompt(alsoWithSession) { - t.Fatal("expected identical session_id to produce stable hash") - } - if sessionHash == hashRenderedPrompt(otherSession) { - t.Fatal("expected session_id value changes to affect rendered prompt hash") - } -} - -func TestRunnerRunSuccessful(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} - reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ - "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, - "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, - }} - renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}} - - runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - PromptVersion: "1", - ProfileID: "exec", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, - "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, - }, - Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.PromptID != "p" || res.PromptVersion != "1" { - t.Fatalf("unexpected prompt metadata: %+v", res) - } - if res.SelectedProfileID != "exec" { - t.Fatalf("expected selected profile exec, got %q", res.SelectedProfileID) - } - if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok { - t.Fatalf("invalid run id: %q", res.RunID) - } - if res.PromptHash == "" || res.RenderedPromptHash == "" { - t.Fatal("expected prompt hashes") - } - if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://override/v1" { - t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams) - } - if res.Artifact.Name != defaults.OutputArtifactName { - t.Fatalf("expected default output artifact name %q, got %q", defaults.OutputArtifactName, res.Artifact.Name) - } - if res.Artifact.ContentType != defaults.ContentTypeTextMarkdown { - t.Fatalf("expected markdown content type %q, got %q", defaults.ContentTypeTextMarkdown, res.Artifact.ContentType) - } - if res.RawOutput != "# recap" { - t.Fatalf("expected raw output, got %q", res.RawOutput) - } - if res.Validation.Status != domain.ValidationSkipped { - t.Fatalf("expected skipped validation, got %q", res.Validation.Status) - } - if llmClient.lastReq.Target.TimeoutSeconds != 90 { - t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds) - } - if !llmClient.lastReq.TargetPresence.Temperature || !llmClient.lastReq.TargetPresence.TimeoutSeconds { - t.Fatalf("expected numeric override presence to be sent to llm, got %+v", llmClient.lastReq.TargetPresence) - } - if llmClient.lastReq.Prompt.SessionID != "session-123" { - t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID) - } -} - -func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) { - extraParams := map[string]any{ - "string_value": "enabled", - "number_value": 42, - "boolean_value": true, - "object_value": map[string]any{"nested": "value"}, - "array_value": []any{"first", 3, false}, - } - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - ExtraParams: extraParams, - }, - }} - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if !reflect.DeepEqual(res.EffectiveModelParams.ExtraParams, extraParams) { - t.Fatalf("expected run result extra_params to match profile values, got %#v", res.EffectiveModelParams.ExtraParams) - } - if !reflect.DeepEqual(llmClient.lastReq.Target.ExtraParams, extraParams) { - t.Fatalf("expected generate request extra_params to match profile values, got %#v", llmClient.lastReq.Target.ExtraParams) - } -} - -func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}} - reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ - "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, - }} - renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}} - runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) - - req := domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, - }, - Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "m", Temperature: float64Ptr(0.3), TimeoutSeconds: intPtr(90)}, - } - - prepared, err := runner.Prepare(context.Background(), req) - if err != nil { - t.Fatalf("prepare should succeed, got %v", err) - } - - res, err := runner.Run(context.Background(), req) - if err != nil { - t.Fatalf("run should succeed, got %v", err) - } - - if res.SelectedProfileID != prepared.SelectedProfileID { - t.Fatalf("expected selected profile to match prepare, run=%q prepare=%q", res.SelectedProfileID, prepared.SelectedProfileID) - } - if !reflect.DeepEqual(res.EffectiveModelParams, prepared.EffectiveModelParams) { - t.Fatalf("effective model params mismatch:\nrun=%+v\nprepare=%+v", res.EffectiveModelParams, prepared.EffectiveModelParams) - } - if !reflect.DeepEqual(res.InputHashes, prepared.InputHashes) { - t.Fatalf("input hashes mismatch:\nrun=%#v\nprepare=%#v", res.InputHashes, prepared.InputHashes) - } - if res.RenderedPromptHash != prepared.RenderedPromptHash { - t.Fatalf("expected rendered prompt hash to match prepare, run=%q prepare=%q", res.RenderedPromptHash, prepared.RenderedPromptHash) - } - if !reflect.DeepEqual(llmClient.lastReq.Prompt.Messages, prepared.Messages) { - t.Fatalf("expected run to send prepare-rendered messages to llm") - } -} - -func TestRunnerRunExplicitProfileIDIsUsed(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - promptRepo.def.DefaultProfile = "default-prof" - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "explicit-prof": {ID: "explicit-prof", Endpoint: "http://explicit/v1", Model: "explicit"}, - "default-prof": {ID: "default-prof", Endpoint: "http://default/v1", Model: "default"}, - }} - - runner := newMinimalRunner(promptRepo, execRepo) - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "explicit-prof", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if execRepo.lastID != "explicit-prof" { - t.Fatalf("expected explicit profile lookup, got %q", execRepo.lastID) - } - if res.SelectedProfileID != "explicit-prof" { - t.Fatalf("expected selected profile explicit-prof, got %q", res.SelectedProfileID) - } -} - -func TestRunnerRunPromptDefaultProfileIsUsedWhenNoExplicitProfileID(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - promptRepo.def.DefaultProfile = "from-prompt" - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "from-prompt": {ID: "from-prompt", Endpoint: "http://llm/v1", Model: "m"}, - }} - - runner := newMinimalRunner(promptRepo, execRepo) - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if execRepo.lastID != "from-prompt" { - t.Fatalf("expected prompt default profile lookup, got %q", execRepo.lastID) - } - if res.SelectedProfileID != "from-prompt" { - t.Fatalf("expected selected profile from-prompt, got %q", res.SelectedProfileID) - } -} - -func TestRunnerRunMissingExplicitProfileAndMissingDefaultProfileFails(t *testing.T) { - repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - repo.def.DefaultProfile = "" - runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}) - _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()}) - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } -} - -func TestRunnerRunInvalidDefaultProfileFails(t *testing.T) { - repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - repo.def.DefaultProfile = "does-not-exist" - runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{}}) - _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", Inputs: singleInputRef()}) - if !errors.Is(err, ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } -} - -func TestRunnerRunExecutionProfileLoadFailure(t *testing.T) { - repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - runner := newMinimalRunner(repo, &fakeExecutionProfileRepo{err: errors.New("load failed")}) - _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) - if !errors.Is(err, ErrProfileLoad) { - t.Fatalf("expected profile load failure, got %v", err) - } -} - -func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.2, - MaxTokens: 500, - TopP: 0.9, - TimeoutSeconds: 120, - ServiceTier: "priority", - }, - }} - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - Execution: &domain.ExecutionTargetOverride{ - Endpoint: "http://override/v1", - Model: "override-model", - Temperature: float64Ptr(0.7), - TimeoutSeconds: intPtr(30), - ServiceTier: "flex", - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Endpoint != "http://override/v1" || res.ModelName != "override-model" { - t.Fatalf("expected endpoint/model override to win, got endpoint=%q model=%q", res.Endpoint, res.ModelName) - } - if res.EffectiveModelParams.Temperature != 0.7 || res.EffectiveModelParams.TimeoutSeconds != 30 { - t.Fatalf("expected numeric override to win, got %+v", res.EffectiveModelParams) - } - if res.EffectiveModelParams.TopP != 0.9 { - t.Fatalf("expected non-overridden profile top_p to remain, got %v", res.EffectiveModelParams.TopP) - } - if res.EffectiveModelParams.ServiceTier != "flex" { - t.Fatalf("expected service_tier override to win, got %q", res.EffectiveModelParams.ServiceTier) - } -} - -func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - TopP: 0.8, - TimeoutSeconds: 90, - ServiceTier: "priority", - }, - }} - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.EffectiveModelParams.TopP != 0.8 { - t.Fatalf("expected profile top_p to beat default, got %v", res.EffectiveModelParams.TopP) - } - if res.EffectiveModelParams.TimeoutSeconds != 90 { - t.Fatalf("expected profile timeout to beat default, got %d", res.EffectiveModelParams.TimeoutSeconds) - } - if res.EffectiveModelParams.ServiceTier != "priority" { - t.Fatalf("expected profile service_tier to beat default, got %q", res.EffectiveModelParams.ServiceTier) - } -} - -func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"}, - }} - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.EffectiveModelParams.Temperature != defaults.ExecutionDefaultTemperature { - t.Fatalf("expected default temperature %v, got %v", defaults.ExecutionDefaultTemperature, res.EffectiveModelParams.Temperature) - } - if res.EffectiveModelParams.TopP != defaults.ExecutionDefaultTopP { - t.Fatalf("expected default top_p %v, got %v", defaults.ExecutionDefaultTopP, res.EffectiveModelParams.TopP) - } - if res.EffectiveModelParams.MaxTokens != defaults.ExecutionDefaultMaxTokens { - t.Fatalf("expected default max_tokens %d, got %d", defaults.ExecutionDefaultMaxTokens, res.EffectiveModelParams.MaxTokens) - } - if res.EffectiveModelParams.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds { - t.Fatalf("expected default timeout_seconds %d, got %d", defaults.ExecutionDefaultTimeoutSeconds, res.EffectiveModelParams.TimeoutSeconds) - } -} - -func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) { - t.Setenv("SCRIPTORIUM_TEST_API_KEY", "secret") - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY"}, - }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) - - res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.EffectiveModelParams.APIKeyEnv != "SCRIPTORIUM_TEST_API_KEY" { - t.Fatalf("expected api_key_env name in effective params, got %q", res.EffectiveModelParams.APIKeyEnv) - } -} - -func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"}, - }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) - - _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if !errors.Is(err, ErrAPIKeyEnvMissing) { - t.Fatalf("expected ErrAPIKeyEnvMissing, got %v", err) - } - if !strings.Contains(err.Error(), "SCRIPTORIUM_MISSING_KEY") { - t.Fatalf("expected missing env name in error, got %v", err) - } -} - -func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) { - const directKey = "direct-runner-key" - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"}, - }} - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) - - _, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - APIKey: directKey, - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if llmClient.lastReq.Target.APIKey != directKey { - t.Fatalf("expected direct API key to reach LLM request") - } - if llmClient.lastReq.Target.APIKeyEnv != "SCRIPTORIUM_MISSING_KEY" { - t.Fatalf("expected api_key_env name to remain on target, got %q", llmClient.lastReq.Target.APIKeyEnv) - } -} - -func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) { - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true}, - }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) - - _, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if !errors.Is(err, ErrAPIKeyRequired) { - t.Fatalf("expected ErrAPIKeyRequired, got %v", err) - } -} - -func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) { - const directKey = "direct-required-key" - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true}, - }} - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) - - _, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - APIKey: directKey, - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if llmClient.lastReq.Target.APIKey != directKey { - t.Fatalf("expected direct API key to reach LLM request") - } - if !llmClient.lastReq.Target.APIKeyRequired { - t.Fatalf("expected APIKeyRequired to be carried to target") - } -} - -func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) { - const envName = "SCRIPTORIUM_RUNTIME_API_KEY" - t.Setenv(envName, "runtime-secret") - - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"}, - }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - Execution: &domain.ExecutionTargetOverride{APIKeyEnv: envName}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.EffectiveModelParams.APIKeyEnv != envName { - t.Fatalf("expected runtime api_key_env override in effective params, got %q", res.EffectiveModelParams.APIKeyEnv) - } -} - -func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) { - const profileEnv = "SCRIPTORIUM_PROFILE_API_KEY" - const runtimeEnv = "SCRIPTORIUM_RUNTIME_API_KEY" - t.Setenv(runtimeEnv, "runtime-secret") - - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv}, - }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - Execution: &domain.ExecutionTargetOverride{APIKeyEnv: runtimeEnv}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.EffectiveModelParams.APIKeyEnv != runtimeEnv { - t.Fatalf("expected runtime override to beat profile api_key_env, got %q", res.EffectiveModelParams.APIKeyEnv) - } -} - -func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) { - const envName = "SCRIPTORIUM_TEST_API_KEY" - const secret = "top-secret-value" - t.Setenv(envName, secret) - promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, - }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) - - res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.EffectiveModelParams.APIKeyEnv != envName { - t.Fatalf("expected api key env name, got %q", res.EffectiveModelParams.APIKeyEnv) - } - metadataDump := fmt.Sprintf("%+v|%s|%s|%s|%s", res.EffectiveModelParams, res.Endpoint, res.ModelName, res.PromptHash, res.RenderedPromptHash) - if strings.Contains(metadataDump, secret) { - t.Fatalf("unexpected api key value in metadata dump: %s", metadataDump) - } -} - -func TestRunnerRunPromptLoadFailure(t *testing.T) { - runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil) - _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"}) - if !errors.Is(err, ErrPromptLoad) { - t.Fatalf("expected ErrPromptLoad, got %v", err) - } - if errors.Is(err, ErrProfileLoad) { - t.Fatalf("did not expect ErrProfileLoad, got %v", err) - } -} - -func TestRunnerRunArtifactLoadFailure(t *testing.T) { - runner := NewRunner( - &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - &fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}}, - &fakeRenderer{rendered: &domain.RenderedPrompt{}}, - &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, - nil, - ) - - _, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}}, - }) - if !errors.Is(err, ErrArtifactLoad) { - t.Fatalf("expected ErrArtifactLoad, got %v", err) - } -} - -func TestRunnerRunPromptRenderFailure(t *testing.T) { - runner := NewRunner( - &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - &fakeRenderer{err: errors.New("render failed")}, - &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, - nil, - ) - _, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if !errors.Is(err, ErrPromptRender) { - t.Fatalf("expected ErrPromptRender, got %v", err) - } -} - -func TestRunnerRunLLMFailure(t *testing.T) { - runner := NewRunner( - &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - defaultRenderer(), - &fakeLLM{err: errors.New("llm failed")}, - nil, - ) - _, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if !errors.Is(err, ErrLLMGenerate) { - t.Fatalf("expected ErrLLMGenerate, got %v", err) - } -} - -func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) { - runner := NewRunner( - &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - defaultRenderer(), - &fakeLLM{err: llm.ErrInvalidRequest}, - nil, - ) - _, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if errors.Is(err, ErrLLMGenerate) { - t.Fatalf("did not expect ErrLLMGenerate, got %v", err) - } -} - -func TestRunnerRunValidationStillWorks(t *testing.T) { - validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}} - runner := NewRunner( - &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - defaultRenderer(), - &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, - validator, - ) - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Validation.Status != domain.ValidationFailed || res.RawOutput != "raw output" { - t.Fatalf("unexpected validation/raw output: %+v", res) - } -} - -func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t *testing.T) { - repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}} - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}} - - runner := NewRunnerWithRepairer( - &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55}, - }}, - defaultArtifactReader(), - defaultRenderer(), - llmClient, - validate.NewStandardValidator("."), - repairer, - ) - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - Execution: &domain.ExecutionTargetOverride{Endpoint: "http://override/v1", Model: "override-model", TimeoutSeconds: intPtr(22)}, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if repairer.calls != 1 || res.Validation.RepairAttempts != 1 { - t.Fatalf("expected one bounded repair, calls=%d attempts=%d", repairer.calls, res.Validation.RepairAttempts) - } - if len(repairer.reqs) != 1 { - t.Fatalf("expected one repair request, got %d", len(repairer.reqs)) - } - if repairer.reqs[0].Target.Endpoint != "http://override/v1" || repairer.reqs[0].Target.Model != "override-model" { - t.Fatalf("expected repair to use effective target, got %+v", repairer.reqs[0].Target) - } - if repairer.reqs[0].Target.TimeoutSeconds != 22 { - t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds) - } -} - -func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) { - def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 1) - def.Validation.SchemaPath = "events.schema.json" - - validator := &fakeValidator{ - result: domain.ValidationResult{ - Status: domain.ValidationFailed, - Mode: domain.ValidationJSONSchema, - Errors: []string{"schema mismatch"}, - IsValid: false, - }, - schemaDoc: map[string]any{ - "type": "object", - "properties": map[string]any{ - "events": map[string]any{"type": "array"}, - }, - }, - } - repairer := &fakeRepairer{ - responses: []*domain.GenerateResponse{ - {Content: `{"events":[]}`}, - }, - } - llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}} - runner := NewRunnerWithRepairer( - &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - defaultArtifactReader(), - defaultRenderer(), - llmClient, - validator, - repairer, - ) - - _, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "p", - ProfileID: "exec", - Inputs: singleInputRef(), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if llmClient.lastReq.StructuredOutput == nil || llmClient.lastReq.StructuredOutput.JSONSchema == nil { - t.Fatalf("expected initial llm request to include structured output, got %+v", llmClient.lastReq.StructuredOutput) - } - if len(repairer.reqs) != 1 { - t.Fatalf("expected one repair request, got %d", len(repairer.reqs)) - } - if repairer.reqs[0].StructuredOutput == nil || repairer.reqs[0].StructuredOutput.JSONSchema == nil { - t.Fatalf("expected repair request structured output, got %+v", repairer.reqs[0].StructuredOutput) - } - if repairer.reqs[0].StructuredOutput.JSONSchema.Name != "p_1" { - t.Fatalf("expected derived schema name p_1, got %q", repairer.reqs[0].StructuredOutput.JSONSchema.Name) - } -} - -func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testing.T) { - src := &domain.ExecutionProfile{ - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.2, - MaxTokens: 123, - TopP: 0.75, - TimeoutSeconds: 90, - ServiceTier: "priority", - ReasoningEffort: "medium", - APIKeyEnv: "SCRIPTORIUM_API_KEY", - APIKeyRequired: true, - ExtraParams: map[string]any{ - "provider_option": "on", - }, - } - - target := executionProfileToTarget(src) - if target.Endpoint != src.Endpoint || - target.Model != src.Model || - target.Temperature != src.Temperature || - target.MaxTokens != src.MaxTokens || - target.TopP != src.TopP || - target.TimeoutSeconds != src.TimeoutSeconds || - target.ServiceTier != src.ServiceTier || - target.ReasoningEffort != src.ReasoningEffort || - target.APIKeyEnv != src.APIKeyEnv || - target.APIKeyRequired != src.APIKeyRequired { - t.Fatalf("expected all profile fields to populate target, got %+v", target) - } - if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) { - t.Fatalf("expected extra_params to match, got %#v", target.ExtraParams) - } - - src.ExtraParams["provider_option"] = "changed" - if target.ExtraParams["provider_option"] != "on" { - t.Fatalf("expected extra_params copy to be independent, got %#v", target.ExtraParams) - } -} - -func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testing.T) { - profileValue := &domain.ExecutionProfile{ - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.3, - MaxTokens: 222, - TopP: 0.6, - TimeoutSeconds: 77, - ServiceTier: "priority", - ReasoningEffort: "low", - APIKeyEnv: "PROFILE_KEY", - APIKeyRequired: true, - ExtraParams: map[string]any{ - "profile_option": "enabled", - }, - } - - target, presence, err := resolveExecutionTarget(profileValue, nil) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if presence != (domain.ExecutionTargetPresence{}) { - t.Fatalf("expected no request override presence, got %+v", presence) - } - if target.Endpoint != profileValue.Endpoint || - target.Model != profileValue.Model || - target.Temperature != profileValue.Temperature || - target.MaxTokens != profileValue.MaxTokens || - target.TopP != profileValue.TopP || - target.TimeoutSeconds != profileValue.TimeoutSeconds || - target.ServiceTier != profileValue.ServiceTier || - target.ReasoningEffort != profileValue.ReasoningEffort || - target.APIKeyEnv != profileValue.APIKeyEnv || - target.APIKeyRequired != profileValue.APIKeyRequired { - t.Fatalf("expected profile values to populate target, got %+v", target) - } - if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) { - t.Fatalf("expected profile extra_params in target, got %#v", target.ExtraParams) - } -} - -func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFields(t *testing.T) { - profileValue := &domain.ExecutionProfile{ - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.2, - MaxTokens: 200, - TopP: 0.8, - TimeoutSeconds: 90, - ServiceTier: "priority", - ReasoningEffort: "medium", - APIKeyEnv: "PROFILE_KEY", - ExtraParams: map[string]any{ - "profile_only": "yes", - }, - } - override := &domain.ExecutionTargetOverride{ - Endpoint: "http://override/v1", - Model: "override-model", - Temperature: float64Ptr(0.9), - MaxTokens: intPtr(111), - TopP: float64Ptr(0.5), - TimeoutSeconds: intPtr(30), - ServiceTier: "flex", - ReasoningEffort: "high", - APIKeyEnv: "RUNTIME_KEY", - ExtraParams: map[string]any{ - "runtime_only": "yes", - }, - } - - target, presence, err := resolveExecutionTarget(profileValue, override) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if presence != (domain.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}) { - t.Fatalf("unexpected override presence: %+v", presence) - } - if target.Endpoint != override.Endpoint || - target.Model != override.Model || - target.Temperature != *override.Temperature || - target.MaxTokens != *override.MaxTokens || - target.TopP != *override.TopP || - target.TimeoutSeconds != *override.TimeoutSeconds || - target.ServiceTier != override.ServiceTier || - target.ReasoningEffort != override.ReasoningEffort || - target.APIKeyEnv != override.APIKeyEnv { - t.Fatalf("expected runtime overrides to win for all fields, got %+v", target) - } - if !reflect.DeepEqual(target.ExtraParams, override.ExtraParams) { - t.Fatalf("expected runtime extra_params to replace profile extra_params, got %#v", target.ExtraParams) - } -} - -func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) { - base := domain.ExecutionTarget{ - Endpoint: "http://base/v1", - Model: "base-model", - ServiceTier: "priority", - ReasoningEffort: "medium", - APIKeyEnv: "BASE_KEY", - } - override := domain.ExecutionTarget{ - Endpoint: "http://override/v1", - Model: "override-model", - ServiceTier: " ", - ReasoningEffort: " ", - APIKeyEnv: "", - } - - merged := mergeExecutionTarget(base, override) - if merged.Endpoint != "http://override/v1" || merged.Model != "override-model" { - t.Fatalf("expected endpoint/model to override, got %+v", merged) - } - if merged.ServiceTier != "priority" { - t.Fatalf("expected empty service_tier override to be ignored, got %q", merged.ServiceTier) - } - if merged.ReasoningEffort != "medium" { - t.Fatalf("expected empty reasoning_effort override to be ignored, got %q", merged.ReasoningEffort) - } - if merged.APIKeyEnv != "BASE_KEY" { - t.Fatalf("expected empty api_key_env override to be ignored, got %q", merged.APIKeyEnv) - } -} - -func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) { - base := domain.ExecutionTarget{ - ExtraParams: map[string]any{ - "keep": "value", - }, - } - override := domain.ExecutionTarget{ - ExtraParams: map[string]any{}, - } - - merged := mergeExecutionTarget(base, override) - if !reflect.DeepEqual(merged.ExtraParams, base.ExtraParams) { - t.Fatalf("expected empty extra_params override not to erase base values, got %#v", merged.ExtraParams) - } -} - -func TestBuildOutputArtifactDefaults(t *testing.T) { - tests := []struct { - name string - format domain.OutputFormat - contentType string - }{ - {name: "text", format: domain.FormatText, contentType: defaults.ContentTypeTextPlain}, - {name: "markdown", format: domain.FormatMarkdown, contentType: defaults.ContentTypeTextMarkdown}, - {name: "json", format: domain.FormatJSON, contentType: defaults.ContentTypeApplicationJSON}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - art := buildOutputArtifact("body", tc.format) - if art.Name != defaults.OutputArtifactName { - t.Fatalf("expected artifact name %q, got %q", defaults.OutputArtifactName, art.Name) - } - if art.ContentType != tc.contentType { - t.Fatalf("expected content type %q, got %q", tc.contentType, art.ContentType) - } - }) - } -} - -func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts int) *domain.PromptDefinition { - return &domain.PromptDefinition{ - ID: "p", - Version: "1", - DefaultProfile: "exec", - Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, - Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, - OutputFormat: format, - Validation: domain.OutputContract{ - ValidationMode: mode, - RepairAttempts: attempts, - Format: format, - }, - } -} - -func hashString(s string) string { - sum := sha256.Sum256([]byte(s)) - return hex.EncodeToString(sum[:]) -} - -func defaultExecutionProfile() *domain.ExecutionProfile { - return &domain.ExecutionProfile{ - ID: "exec", - Endpoint: "http://llm/v1", - Model: "model-from-profile", - } -} - -func defaultArtifactReader() *fakeArtifactReader { - return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ - "a://ok": {Body: []byte("x"), Hash: hashString("x")}, - }} -} - -func defaultRenderer() *fakeRenderer { - return &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}} -} - -func singleInputRef() map[string]domain.ArtifactRef { - return map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}} -} - -func float64Ptr(v float64) *float64 { - return &v -} - -func intPtr(v int) *int { - return &v -} - -func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner { - return NewRunner( - promptRepo, - execRepo, - defaultArtifactReader(), - defaultRenderer(), - &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, - nil, - ) -} diff --git a/internal/validate/standard_validator.go b/internal/validate/standard_validator.go deleted file mode 100644 index 5e1ff7f..0000000 --- a/internal/validate/standard_validator.go +++ /dev/null @@ -1,292 +0,0 @@ -package validate - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io/fs" - "os" - "path" - "path/filepath" - "strings" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" - "gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog" - "github.com/santhosh-tekuri/jsonschema/v6" -) - -// StandardValidator provides basic, JSON, and JSON Schema output validation. -type StandardValidator struct { - schemaBaseDir string -} - -type FSValidator struct { - fsys fs.FS - root string -} - -func NewStandardValidator(schemaBaseDir string) Validator { - return &StandardValidator{schemaBaseDir: schemaBaseDir} -} - -func NewFSValidator(fsys fs.FS, root string) Validator { - return &FSValidator{fsys: fsys, root: root} -} - -func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { - return validateArtifact(ctx, artifact, contract, v.validateJSONSchema) -} - -func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { - return validateArtifact(ctx, artifact, contract, v.validateJSONSchema) -} - -type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error) - -func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) { - select { - case <-ctx.Done(): - return domain.ValidationResult{}, ctx.Err() - default: - } - - res := domain.ValidationResult{ - Mode: contract.ValidationMode, - SchemaPath: contract.SchemaPath, - RepairAttempts: contract.RepairAttempts, - } - - if artifact == nil { - return domain.ValidationResult{}, errors.New("artifact is required for validation") - } - - switch contract.ValidationMode { - case domain.ValidationNone: - res.Status = domain.ValidationSkipped - res.IsValid = true - return res, nil - case domain.ValidationBasic: - if strings.TrimSpace(string(artifact.Body)) == "" { - res.Status = domain.ValidationFailed - res.IsValid = false - res.Errors = []string{"output is empty"} - return res, nil - } - res.Status = domain.ValidationPassed - res.IsValid = true - return res, nil - case domain.ValidationJSON: - _, jsonErr := parseJSON(artifact.Body) - if jsonErr != nil { - res.Status = domain.ValidationFailed - res.IsValid = false - res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)} - return res, nil - } - res.Status = domain.ValidationPassed - res.IsValid = true - return res, nil - case domain.ValidationJSONSchema: - instance, jsonErr := parseJSON(artifact.Body) - if jsonErr != nil { - res.Status = domain.ValidationFailed - res.IsValid = false - res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)} - return res, nil - } - - validationErrors, err := validateSchema(instance, contract.SchemaPath) - if err != nil { - return domain.ValidationResult{}, err - } - if len(validationErrors) > 0 { - res.Status = domain.ValidationFailed - res.IsValid = false - res.Errors = validationErrors - return res, nil - } - - res.Status = domain.ValidationPassed - res.IsValid = true - return res, nil - default: - return domain.ValidationResult{}, fmt.Errorf("unsupported validation mode: %q", contract.ValidationMode) - } -} - -func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) { - resolvedSchemaPath, err := v.resolveSchemaPath(schemaPath) - if err != nil { - return nil, err - } - - compiler := jsonschema.NewCompiler() - schema, err := compiler.Compile(resolvedSchemaPath) - if err != nil { - return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err) - } - - if err := schema.Validate(instance); err != nil { - return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil - } - return nil, nil -} - -func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) { - schemaName, schemaDoc, err := v.loadSchemaDocument(schemaPath) - if err != nil { - return nil, err - } - - resourceURL := fsSchemaResourceURL(schemaName) - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource(resourceURL, schemaDoc); err != nil { - return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err) - } - schema, err := compiler.Compile(resourceURL) - if err != nil { - return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err) - } - - if err := schema.Validate(instance); err != nil { - return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil - } - return nil, nil -} - -func parseJSON(body []byte) (any, error) { - var v any - if err := json.Unmarshal(body, &v); err != nil { - return nil, err - } - return v, nil -} - -func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - resolved, err := v.resolveSchemaPath(schemaPath) - if err != nil { - return nil, err - } - - raw, err := os.ReadFile(resolved) - if err != nil { - return nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err) - } - - var doc any - if err := json.Unmarshal(raw, &doc); err != nil { - return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) - } - return doc, nil -} - -func (v *FSValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - _, doc, err := v.loadSchemaDocument(schemaPath) - if err != nil { - return nil, err - } - return doc, nil -} - -func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) { - if strings.TrimSpace(schemaPath) == "" { - return "", errors.New("schema path is required for json_schema validation") - } - - resolved := schemaPath - if !filepath.IsAbs(schemaPath) { - resolved = filepath.Join(v.schemaBaseDir, schemaPath) - } - - resolved = filepath.Clean(resolved) - if _, err := os.Stat(resolved); err != nil { - return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err) - } - - return resolved, nil -} - -func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) { - resolved, err := v.resolveSchemaPath(schemaPath) - if err != nil { - return "", nil, err - } - - raw, err := fs.ReadFile(v.fsys, resolved) - if err != nil { - return "", nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err) - } - - var doc any - if err := json.Unmarshal(raw, &doc); err != nil { - return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) - } - return resolved, doc, nil -} - -func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) { - if strings.TrimSpace(schemaPath) == "" { - return "", errors.New("schema path is required for json_schema validation") - } - if v.fsys == nil { - return "", errors.New("schema filesystem is nil") - } - - cleanRoot := filecatalog.CleanFSRoot(v.root) - rootInfo, err := fs.Stat(v.fsys, cleanRoot) - if err != nil { - return "", fmt.Errorf("failed to access schema source %q: %w", cleanRoot, err) - } - - var resolved string - if rootInfo.IsDir() { - resolvedPath, _, err := filecatalog.ResolveFSPath(cleanRoot, cleanRoot, schemaPath) - if err != nil { - return "", err - } - resolved = resolvedPath - } else { - cleanSchemaPath, err := cleanSchemaFSPath(schemaPath) - if err != nil { - return "", err - } - if cleanSchemaPath != path.Base(cleanRoot) { - return "", fmt.Errorf("schema path %q does not match schema file %q", cleanSchemaPath, path.Base(cleanRoot)) - } - resolved = cleanRoot - } - - if _, err := fs.Stat(v.fsys, resolved); err != nil { - return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err) - } - return resolved, nil -} - -func cleanSchemaFSPath(schemaPath string) (string, error) { - cleaned := strings.TrimSpace(schemaPath) - if cleaned == "" { - return "", errors.New("schema path is required for json_schema validation") - } - cleaned = path.Clean(cleaned) - if path.IsAbs(cleaned) { - return "", fmt.Errorf("schema path %q must be relative", schemaPath) - } - return cleaned, nil -} - -func fsSchemaResourceURL(schemaName string) string { - return "scriptorium-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/") -} diff --git a/internal/validate/standard_validator_test.go b/internal/validate/standard_validator_test.go deleted file mode 100644 index d404a5b..0000000 --- a/internal/validate/standard_validator_test.go +++ /dev/null @@ -1,383 +0,0 @@ -package validate - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - "testing/fstest" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -func TestStandardValidatorNoneSkipped(t *testing.T) { - v := NewStandardValidator("") - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("ignored")}, domain.OutputContract{ - ValidationMode: domain.ValidationNone, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationSkipped { - t.Fatalf("expected skipped, got %q", res.Status) - } - if !res.IsValid { - t.Fatal("expected valid=true for skipped") - } -} - -func TestStandardValidatorBasicSuccess(t *testing.T) { - v := NewStandardValidator("") - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("hello")}, domain.OutputContract{ - ValidationMode: domain.ValidationBasic, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationPassed || !res.IsValid { - t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) - } -} - -func TestStandardValidatorBasicFailureEmpty(t *testing.T) { - v := NewStandardValidator("") - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(" \n\t ")}, domain.OutputContract{ - ValidationMode: domain.ValidationBasic, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationFailed || res.IsValid { - t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid) - } - if len(res.Errors) == 0 { - t.Fatal("expected validation errors") - } -} - -func TestStandardValidatorJSONSuccess(t *testing.T) { - v := NewStandardValidator("") - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":true}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSON, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationPassed || !res.IsValid { - t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) - } -} - -func TestStandardValidatorJSONFailure(t *testing.T) { - v := NewStandardValidator("") - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSON, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationFailed || res.IsValid { - t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid) - } - if len(res.Errors) == 0 { - t.Fatal("expected parse errors") - } -} - -func TestStandardValidatorJSONSchemaSuccess(t *testing.T) { - tmp := t.TempDir() - schemaPath := filepath.Join(tmp, "schema.json") - if err := os.WriteFile(schemaPath, []byte(`{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": ["name"], - "properties": { - "name": {"type": "string"} - } -}`), 0644); err != nil { - t.Fatal(err) - } - - v := NewStandardValidator(tmp) - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: "schema.json", - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationPassed || !res.IsValid { - t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) - } -} - -func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) { - tmp := t.TempDir() - nestedDir := filepath.Join(tmp, "dnd") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(nestedDir, "schema.json"), []byte(`{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": ["name"], - "properties": { - "name": {"type": "string"} - } -}`), 0644); err != nil { - t.Fatal(err) - } - - v := NewStandardValidator(tmp) - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: filepath.Join("dnd", "schema.json"), - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationPassed || !res.IsValid { - t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) - } -} - -func TestStandardValidatorJSONSchemaNestedSchemaPathMissing(t *testing.T) { - v := NewStandardValidator(t.TempDir()) - - _, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: filepath.Join("dnd", "missing.json"), - }) - if err == nil { - t.Fatal("expected nested schema load error") - } -} - -func TestStandardValidatorJSONSchemaFailure(t *testing.T) { - tmp := t.TempDir() - schemaPath := filepath.Join(tmp, "schema.json") - if err := os.WriteFile(schemaPath, []byte(`{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": ["name"], - "properties": { - "name": {"type": "string"} - } -}`), 0644); err != nil { - t.Fatal(err) - } - - v := NewStandardValidator(tmp) - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"count":1}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: "schema.json", - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationFailed || res.IsValid { - t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid) - } - if len(res.Errors) == 0 { - t.Fatal("expected schema errors") - } -} - -func TestStandardValidatorJSONSchemaSchemaLoadError(t *testing.T) { - v := NewStandardValidator(t.TempDir()) - - _, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: "missing.json", - }) - if err == nil { - t.Fatal("expected schema load error") - } -} - -func TestStandardValidatorLoadSchemaDocumentSuccess(t *testing.T) { - tmp := t.TempDir() - if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{ - "type": "object", - "properties": { - "name": {"type": "string"} - } -}`), 0644); err != nil { - t.Fatal(err) - } - - v := NewStandardValidator(tmp) - loader, ok := v.(SchemaDocumentLoader) - if !ok { - t.Fatal("standard validator must implement SchemaDocumentLoader") - } - - doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - obj, ok := doc.(map[string]any) - if !ok { - t.Fatalf("expected object document, got %#v", doc) - } - if obj["type"] != "object" { - t.Fatalf("expected schema type=object, got %#v", obj["type"]) - } -} - -func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) { - tmp := t.TempDir() - if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{`), 0644); err != nil { - t.Fatal(err) - } - - v := NewStandardValidator(tmp) - loader, ok := v.(SchemaDocumentLoader) - if !ok { - t.Fatal("standard validator must implement SchemaDocumentLoader") - } - - _, err := loader.LoadSchemaDocument(context.Background(), "schema.json") - if err == nil { - t.Fatal("expected decode error") - } -} - -func TestFSValidatorJSONSchemaSuccess(t *testing.T) { - v := NewFSValidator(fstest.MapFS{ - "schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": ["events"], - "properties": { - "events": {"type": "array"} - } -}`)}, - }, "schemas") - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: "events.schema.json", - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationPassed || !res.IsValid { - t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) - } -} - -func TestFSValidatorJSONSchemaPathContainment(t *testing.T) { - t.Run("nested schema inside root succeeds", func(t *testing.T) { - v := NewFSValidator(fstest.MapFS{ - "schemas/nested/events.schema.json": &fstest.MapFile{Data: []byte(`{ - "type": "object", - "required": ["events"], - "properties": { - "events": {"type": "array"} - } -}`)}, - }, "schemas") - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: "nested/events.schema.json", - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationPassed || !res.IsValid { - t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) - } - }) - - tests := []struct { - name string - schemaPath string - wantErr string - }{ - {name: "parent escape rejected", schemaPath: "../outside.schema.json", wantErr: "escapes source root"}, - {name: "absolute path rejected", schemaPath: "/outside.schema.json", wantErr: "must be relative"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - v := NewFSValidator(fstest.MapFS{ - "schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, - "outside.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, - "schemas/outside.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, - }, "schemas") - - _, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: tc.schemaPath, - }) - if err == nil { - t.Fatal("expected schema path error") - } - if !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err) - } - }) - } -} - -func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) { - v := NewFSValidator(fstest.MapFS{ - "events.schema.json": &fstest.MapFile{Data: []byte(`{ - "type": "object", - "required": ["events"], - "properties": { - "events": {"type": "array"} - } -}`)}, - }, "events.schema.json") - - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: "events.schema.json", - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Status != domain.ValidationPassed || !res.IsValid { - t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) - } - - _, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: "other.schema.json", - }) - if err == nil { - t.Fatal("expected schema path mismatch error") - } -} - -func TestFSValidatorLoadSchemaDocument(t *testing.T) { - v := NewFSValidator(fstest.MapFS{ - "schemas/schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, - }, "schemas") - loader, ok := v.(SchemaDocumentLoader) - if !ok { - t.Fatal("fs validator must implement SchemaDocumentLoader") - } - - doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - obj, ok := doc.(map[string]any) - if !ok || obj["type"] != "object" { - t.Fatalf("unexpected schema document: %#v", doc) - } -} diff --git a/internal/validate/validator.go b/internal/validate/validator.go deleted file mode 100644 index e483df7..0000000 --- a/internal/validate/validator.go +++ /dev/null @@ -1,16 +0,0 @@ -package validate - -import ( - "context" - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -// Validator validates the generated artifact based on the output contract. -type Validator interface { - Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) -} - -// SchemaDocumentLoader loads JSON schema documents using validator path semantics. -type SchemaDocumentLoader interface { - LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) -} diff --git a/json_copy.go b/json_copy.go deleted file mode 100644 index d09e36c..0000000 --- a/json_copy.go +++ /dev/null @@ -1,218 +0,0 @@ -package scriptorium - -import ( - "encoding/json" - "fmt" - "math" - "reflect" - "strconv" -) - -const maxSafeJSONInteger = 1<<53 - 1 - -type jsonVisit struct { - typ reflect.Type - ptr uintptr -} - -func copyPublicJSONMap(src map[string]any) (map[string]any, error) { - if src == nil { - return nil, nil - } - copied, err := copyPublicJSONValue(reflect.ValueOf(src), "extra_params", make(map[jsonVisit]struct{})) - if err != nil { - return nil, err - } - out, ok := copied.(map[string]any) - if !ok { - return nil, fmt.Errorf("extra_params: expected object") - } - return out, nil -} - -func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) { - if !value.IsValid() { - return nil, nil - } - if value.Kind() == reflect.Interface { - if value.IsNil() { - return nil, nil - } - return copyPublicJSONValue(value.Elem(), path, seen) - } - if !value.CanInterface() { - return nil, fmt.Errorf("%s: value cannot be copied", path) - } - if number, ok := value.Interface().(json.Number); ok { - f, err := strconv.ParseFloat(number.String(), 64) - if err != nil || math.IsNaN(f) || math.IsInf(f, 0) { - return nil, fmt.Errorf("%s: invalid JSON number", path) - } - return number, nil - } - - switch value.Kind() { - case reflect.Bool, reflect.String: - return value.Interface(), nil - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - if value.Int() < -maxSafeJSONInteger || value.Int() > maxSafeJSONInteger { - return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path) - } - return value.Interface(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - if value.Uint() > maxSafeJSONInteger { - return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path) - } - return value.Interface(), nil - case reflect.Float32, reflect.Float64: - f := value.Convert(reflect.TypeOf(float64(0))).Float() - if math.IsNaN(f) || math.IsInf(f, 0) { - return nil, fmt.Errorf("%s: floating-point value must be finite", path) - } - return value.Interface(), nil - case reflect.Pointer: - if value.IsNil() { - return nil, nil - } - visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()} - if _, ok := seen[visit]; ok { - return nil, fmt.Errorf("%s: cyclic value is not supported", path) - } - seen[visit] = struct{}{} - defer delete(seen, visit) - return copyPublicJSONValue(value.Elem(), path, seen) - case reflect.Map: - return copyPublicJSONMapValue(value, path, seen) - case reflect.Slice: - if value.IsNil() { - return nil, nil - } - return copyPublicJSONSequenceValue(value, path, seen) - case reflect.Array: - return copyPublicJSONSequenceValue(value, path, seen) - default: - return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type()) - } -} - -func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) { - if value.IsNil() { - return nil, nil - } - if value.Type().Key().Kind() != reflect.String { - return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key()) - } - - visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()} - if _, ok := seen[visit]; ok { - return nil, fmt.Errorf("%s: cyclic value is not supported", path) - } - seen[visit] = struct{}{} - defer delete(seen, visit) - - type entry struct { - key reflect.Value - name string - value any - } - entries := make([]entry, 0, value.Len()) - preserveType := true - elemType := value.Type().Elem() - iter := value.MapRange() - for iter.Next() { - key := iter.Key() - name := key.String() - copied, err := copyPublicJSONValue(iter.Value(), path+"."+name, seen) - if err != nil { - return nil, err - } - entries = append(entries, entry{key: key, name: name, value: copied}) - if copied == nil { - if !canAssignNil(elemType) { - preserveType = false - } - continue - } - if !reflect.TypeOf(copied).AssignableTo(elemType) { - preserveType = false - } - } - - if preserveType { - out := reflect.MakeMapWithSize(value.Type(), len(entries)) - for _, entry := range entries { - if entry.value == nil { - out.SetMapIndex(entry.key, reflect.Zero(elemType)) - continue - } - out.SetMapIndex(entry.key, reflect.ValueOf(entry.value)) - } - return out.Interface(), nil - } - - out := make(map[string]any, len(entries)) - for _, entry := range entries { - out[entry.name] = entry.value - } - return out, nil -} - -func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) { - var visit jsonVisit - if value.Kind() == reflect.Slice { - visit = jsonVisit{typ: value.Type(), ptr: value.Pointer()} - if _, ok := seen[visit]; ok { - return nil, fmt.Errorf("%s: cyclic value is not supported", path) - } - seen[visit] = struct{}{} - defer delete(seen, visit) - } - - values := make([]any, value.Len()) - preserveType := true - elemType := value.Type().Elem() - for i := 0; i < value.Len(); i++ { - copied, err := copyPublicJSONValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen) - if err != nil { - return nil, err - } - values[i] = copied - if copied == nil { - if !canAssignNil(elemType) { - preserveType = false - } - continue - } - if !reflect.TypeOf(copied).AssignableTo(elemType) { - preserveType = false - } - } - - if preserveType { - out := reflect.New(value.Type()).Elem() - if value.Kind() == reflect.Slice { - out = reflect.MakeSlice(value.Type(), value.Len(), value.Len()) - } - for i, copied := range values { - if copied == nil { - out.Index(i).Set(reflect.Zero(elemType)) - continue - } - out.Index(i).Set(reflect.ValueOf(copied)) - } - return out.Interface(), nil - } - - out := make([]any, len(values)) - copy(out, values) - return out, nil -} - -func canAssignNil(typ reflect.Type) bool { - switch typ.Kind() { - case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: - return true - default: - return false - } -} diff --git a/llm_adapter.go b/llm_adapter.go deleted file mode 100644 index f5b9672..0000000 --- a/llm_adapter.go +++ /dev/null @@ -1,23 +0,0 @@ -package scriptorium - -import ( - "context" - "fmt" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" -) - -type publicLLMClientAdapter struct { - client LLMClient -} - -func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { - resp, err := a.client.Generate(ctx, fromDomainGenerateRequest(req)) - if err != nil { - return nil, err - } - if resp == nil { - return nil, fmt.Errorf("%w: llm client returned nil response", ErrLLMGenerate) - } - return toDomainGenerateResponse(resp), nil -} diff --git a/profiles.go b/profiles.go deleted file mode 100644 index 334638e..0000000 --- a/profiles.go +++ /dev/null @@ -1,124 +0,0 @@ -package scriptorium - -import ( - "context" - "errors" - "fmt" - "strings" - - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" - "gitea.maximumdirect.net/eric/scriptorium/internal/profile" -) - -// OpenAICompatibleProfile returns an ordinary in-memory Profile for an -// OpenAI-compatible chat-completions endpoint. -// -// It does not register global state, maintain a model catalog, or resolve -// credentials. If APIKeyRequired is true, callers satisfy it with -// RunRequest.APIKey. Raw API keys do not belong in profiles. -func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile { - return Profile{ - ID: cfg.ID, - Endpoint: cfg.Endpoint, - Model: cfg.Model, - Temperature: cfg.Temperature, - MaxTokens: cfg.MaxTokens, - TopP: cfg.TopP, - TimeoutSeconds: cfg.TimeoutSeconds, - ServiceTier: cfg.ServiceTier, - ReasoningEffort: cfg.ReasoningEffort, - APIKeyRequired: cfg.APIKeyRequired, - ExtraParams: copyShallowAnyMap(cfg.ExtraParams), - } -} - -func copyShallowAnyMap(src map[string]any) map[string]any { - if src == nil { - return nil - } - out := make(map[string]any, len(src)) - for k, v := range src { - out[k] = v - } - return out -} - -type memoryProfileRepository struct { - profiles map[string]domain.ExecutionProfile -} - -func newMemoryProfileRepository(profiles []Profile) (*memoryProfileRepository, error) { - repo := &memoryProfileRepository{profiles: make(map[string]domain.ExecutionProfile, len(profiles))} - for _, publicProfile := range profiles { - prof, err := toDomainProfile(publicProfile) - if err != nil { - return nil, err - } - if _, exists := repo.profiles[prof.ID]; exists { - return nil, fmt.Errorf("duplicate profile id %q", prof.ID) - } - repo.profiles[prof.ID] = prof - } - return repo, nil -} - -func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) { - if r == nil { - return nil, profile.ErrProfileNotFound - } - prof, ok := r.profiles[id] - if !ok { - return nil, profile.ErrProfileNotFound - } - prof.ExtraParams = copyAnyMap(prof.ExtraParams) - return &prof, nil -} - -func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) { - extraParams, err := copyPublicJSONMap(publicProfile.ExtraParams) - if err != nil { - return domain.ExecutionProfile{}, err - } - prof := domain.ExecutionProfile{ - ID: strings.TrimSpace(publicProfile.ID), - Endpoint: publicProfile.Endpoint, - Model: publicProfile.Model, - Temperature: publicProfile.Temperature, - MaxTokens: publicProfile.MaxTokens, - TopP: publicProfile.TopP, - TimeoutSeconds: publicProfile.TimeoutSeconds, - ServiceTier: publicProfile.ServiceTier, - ReasoningEffort: publicProfile.ReasoningEffort, - APIKeyRequired: publicProfile.APIKeyRequired, - ExtraParams: extraParams, - } - if err := validatePublicProfile(prof); err != nil { - return domain.ExecutionProfile{}, err - } - return prof, nil -} - -func validatePublicProfile(prof domain.ExecutionProfile) error { - if strings.TrimSpace(prof.ID) == "" { - return errors.New("id is required") - } - if strings.TrimSpace(prof.Endpoint) == "" { - return errors.New("endpoint is required") - } - if strings.TrimSpace(prof.Model) == "" { - return errors.New("model is required") - } - if prof.Temperature < 0 || prof.Temperature > 2 { - return errors.New("temperature must be between 0 and 2") - } - if prof.MaxTokens < 0 { - return errors.New("max_tokens must be greater than or equal to 0") - } - if prof.TopP < 0 || prof.TopP > 1 { - return errors.New("top_p must be between 0 and 1") - } - if prof.TimeoutSeconds < 0 { - return errors.New("timeout_seconds must be greater than or equal to 0") - } - return nil -} diff --git a/testdata/framework/fixtures/glossary.yml b/testdata/framework/fixtures/glossary.yml deleted file mode 100644 index 503c677..0000000 --- a/testdata/framework/fixtures/glossary.yml +++ /dev/null @@ -1,2 +0,0 @@ -archive: A catalogued collection of written records. -marker: A small label used to classify an entry. diff --git a/testdata/framework/fixtures/transcript.md b/testdata/framework/fixtures/transcript.md deleted file mode 100644 index b74a773..0000000 --- a/testdata/framework/fixtures/transcript.md +++ /dev/null @@ -1,2 +0,0 @@ -Nia labels the archive. -The archive receives a blue marker. diff --git a/testdata/framework/profiles/contract-fast.yaml b/testdata/framework/profiles/contract-fast.yaml deleted file mode 100644 index 5ac326d..0000000 --- a/testdata/framework/profiles/contract-fast.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: contract-fast -endpoint: http://localhost:8000/v1 -model: contract-fast-model -temperature: 0.2 -max_tokens: 500 -top_p: 1 -timeout_seconds: 90 diff --git a/testdata/framework/profiles/contract-quality.yaml b/testdata/framework/profiles/contract-quality.yaml deleted file mode 100644 index 4477c74..0000000 --- a/testdata/framework/profiles/contract-quality.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: contract-quality -endpoint: http://localhost:8000/v1 -model: contract-quality-model -temperature: 0.1 -max_tokens: 1000 -top_p: 0.9 -timeout_seconds: 120 diff --git a/testdata/framework/prompts/contract.markdown_summary.system.md b/testdata/framework/prompts/contract.markdown_summary.system.md deleted file mode 100644 index 3e649a2..0000000 --- a/testdata/framework/prompts/contract.markdown_summary.system.md +++ /dev/null @@ -1 +0,0 @@ -You summarize synthetic archive notes in clear Markdown. diff --git a/testdata/framework/prompts/contract.markdown_summary.user.md b/testdata/framework/prompts/contract.markdown_summary.user.md deleted file mode 100644 index aa47620..0000000 --- a/testdata/framework/prompts/contract.markdown_summary.user.md +++ /dev/null @@ -1,7 +0,0 @@ -Summarize this transcript: - -{{input "transcript"}} - -Optional glossary: - -{{input "glossary"}} diff --git a/testdata/framework/prompts/contract.markdown_summary.yaml b/testdata/framework/prompts/contract.markdown_summary.yaml deleted file mode 100644 index 3242ff5..0000000 --- a/testdata/framework/prompts/contract.markdown_summary.yaml +++ /dev/null @@ -1,20 +0,0 @@ -id: contract.markdown_summary -version: "1.0.0" -default_profile: contract-fast -description: Summarize a synthetic transcript in Markdown. -inputs: - - name: transcript - required: true - content_type: text/markdown - - name: glossary - required: false - content_type: text/yaml -messages: - - role: system - content_file: ./contract.markdown_summary.system.md - - role: user - content_file: ./contract.markdown_summary.user.md -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/testdata/framework/prompts/contract.structured_events.system.md b/testdata/framework/prompts/contract.structured_events.system.md deleted file mode 100644 index eba5946..0000000 --- a/testdata/framework/prompts/contract.structured_events.system.md +++ /dev/null @@ -1 +0,0 @@ -Return only JSON that satisfies the requested event schema. diff --git a/testdata/framework/prompts/contract.structured_events.user.md b/testdata/framework/prompts/contract.structured_events.user.md deleted file mode 100644 index 9bcc0cc..0000000 --- a/testdata/framework/prompts/contract.structured_events.user.md +++ /dev/null @@ -1,7 +0,0 @@ -Extract events from this transcript: - -{{input "transcript"}} - -Optional glossary: - -{{input "glossary"}} diff --git a/testdata/framework/prompts/contract.structured_events.yaml b/testdata/framework/prompts/contract.structured_events.yaml deleted file mode 100644 index 607fad8..0000000 --- a/testdata/framework/prompts/contract.structured_events.yaml +++ /dev/null @@ -1,21 +0,0 @@ -id: contract.structured_events -version: "1.0.0" -default_profile: contract-quality -description: Extract synthetic events as structured JSON. -inputs: - - name: transcript - required: true - content_type: text/markdown - - name: glossary - required: false - content_type: text/yaml -messages: - - role: system - content_file: ./contract.structured_events.system.md - - role: user - content_file: ./contract.structured_events.user.md -output: - format: json - validation_mode: json_schema - schema_path: structured_events.schema.json - repair_attempts: 0 diff --git a/testdata/framework/schemas/structured_events.schema.json b/testdata/framework/schemas/structured_events.schema.json deleted file mode 100644 index 4791fe6..0000000 --- a/testdata/framework/schemas/structured_events.schema.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": ["events"], - "properties": { - "events": { - "type": "array", - "items": { - "type": "object", - "required": ["title"], - "properties": { - "title": {"type": "string"} - }, - "additionalProperties": false - } - } - }, - "additionalProperties": false -} diff --git a/types.go b/types.go deleted file mode 100644 index c16cafc..0000000 --- a/types.go +++ /dev/null @@ -1,306 +0,0 @@ -package scriptorium - -import ( - "context" - "time" -) - -// ArtifactRefType defines how an artifact is referenced. -type ArtifactRefType string - -const ( - ArtifactRefInline ArtifactRefType = "inline" - ArtifactRefFile ArtifactRefType = "file" -) - -// OutputFormat defines the desired output format. -type OutputFormat string - -const ( - FormatText OutputFormat = "text" - FormatMarkdown OutputFormat = "markdown" - FormatJSON OutputFormat = "json" -) - -// ValidationMode defines the output validation strategy. -type ValidationMode string - -const ( - ValidationNone ValidationMode = "none" - ValidationBasic ValidationMode = "basic" - ValidationJSON ValidationMode = "json" - ValidationJSONSchema ValidationMode = "json_schema" -) - -// ValidationStatus defines the result of a validation check. -type ValidationStatus string - -const ( - ValidationPassed ValidationStatus = "passed" - ValidationFailed ValidationStatus = "failed" - ValidationSkipped ValidationStatus = "skipped" -) - -// CacheControlType defines provider cache behavior for prompt content. -type CacheControlType string - -const ( - CacheControlEphemeral CacheControlType = "ephemeral" -) - -// StructuredOutputType identifies provider-level structured output modes. -type StructuredOutputType string - -const ( - StructuredOutputJSONSchema StructuredOutputType = "json_schema" -) - -// RunRequest represents a request to prepare or run a single prompt. -type RunRequest struct { - PromptID string - PromptVersion string - ProfileID string - APIKey string `json:"-"` - Inputs map[string]ArtifactRef - Vars map[string]string - Execution *ExecutionTargetOverride - Validation *OutputContract - Metadata map[string]string -} - -// PreparedRun contains prepared prompt execution state. It does not include -// resolved API key values, model output, validation results, or internal target -// presence metadata. -type PreparedRun struct { - PromptID string `json:"prompt_id"` - PromptVersion string `json:"prompt_version,omitempty"` - PromptHash string `json:"prompt_hash,omitempty"` - SelectedProfileID string `json:"selected_profile_id"` - EffectiveModelParams ExecutionTarget `json:"effective_model_params"` - OutputContract OutputContract `json:"output_contract"` - StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` - InputHashes map[string]string `json:"input_hashes,omitempty"` - SessionID string `json:"session_id,omitempty"` - RenderedPromptHash string `json:"rendered_prompt_hash"` - Messages []RenderedMessage `json:"messages"` - StartTime time.Time `json:"start_time,omitempty"` - EndTime time.Time `json:"end_time,omitempty"` - DurationMS int64 `json:"duration_ms,omitempty"` -} - -// RunResult contains generated output, validation state, and run metadata. -type RunResult struct { - RunID string `json:"run_id"` - Artifact Artifact `json:"artifact"` - RawOutput string `json:"raw_output"` - Validation ValidationResult `json:"validation"` - PromptID string `json:"prompt_id"` - PromptVersion string `json:"prompt_version,omitempty"` - PromptHash string `json:"prompt_hash,omitempty"` - RenderedPromptHash string `json:"rendered_prompt_hash"` - SelectedProfileID string `json:"selected_profile_id"` - ModelName string `json:"model_name"` - Endpoint string `json:"endpoint"` - EffectiveModelParams ExecutionTarget `json:"effective_model_params"` - InputHashes map[string]string `json:"input_hashes,omitempty"` - Usage TokenUsage `json:"usage"` - StartTime time.Time `json:"start_time,omitempty"` - EndTime time.Time `json:"end_time,omitempty"` - Duration time.Duration `json:"duration,omitempty"` -} - -// ArtifactRef represents a reference to prompt input content. -type ArtifactRef struct { - Type ArtifactRefType - URI string - Body string -} - -// Artifact represents loaded artifact content. -type Artifact struct { - Name string - ContentType string - Body []byte - URI string - Size int64 - Hash string -} - -// ArtifactReader resolves a prompt input reference into its content. -// -// Readers are responsible for supplying artifact metadata. The engine assigns -// an input-map name only when the returned artifact name is empty. -type ArtifactReader interface { - Read(context.Context, ArtifactRef) (*Artifact, error) -} - -// ExecutionTarget represents effective model runtime settings. -type ExecutionTarget struct { - Endpoint string `json:"endpoint"` - Model string `json:"model"` - Temperature float64 `json:"temperature"` - MaxTokens int `json:"max_tokens"` - TopP float64 `json:"top_p"` - TimeoutSeconds int `json:"timeout_seconds"` - ServiceTier string `json:"service_tier"` - ReasoningEffort string `json:"reasoning_effort"` - APIKeyEnv string `json:"api_key_env"` - ExtraParams map[string]any `json:"extra_params"` -} - -// ExecutionTargetOverride represents per-request runtime setting overrides. -type ExecutionTargetOverride struct { - Endpoint string - Model string - Temperature *float64 - MaxTokens *int - TopP *float64 - TimeoutSeconds *int - ServiceTier string - ReasoningEffort string - APIKeyEnv string - ExtraParams map[string]any -} - -// Profile is an in-memory execution profile for library consumers. -// -// It is equivalent to a loaded profile file after validation. Raw API keys do -// not belong in profiles; use APIKeyRequired to require callers to provide -// RunRequest.APIKey for each request, or use profile YAML api_key_env with file -// and FS profile sources. -type Profile struct { - ID string - Endpoint string - Model string - Temperature float64 - MaxTokens int - TopP float64 - TimeoutSeconds int - ServiceTier string - ReasoningEffort string - APIKeyRequired bool - ExtraParams map[string]any -} - -// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory -// profile. -// -// It contains ordinary profile fields for OpenAI-compatible chat-completions -// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do -// not belong in this config. -type OpenAICompatibleProfileConfig struct { - ID string - Endpoint string - Model string - APIKeyRequired bool - Temperature float64 - MaxTokens int - TopP float64 - TimeoutSeconds int - ServiceTier string - ReasoningEffort string - ExtraParams map[string]any -} - -// ExecutionTargetPresence tracks which numeric runtime settings were explicit -// request overrides. -type ExecutionTargetPresence struct { - Temperature bool - MaxTokens bool - TopP bool - TimeoutSeconds bool -} - -// OutputContract defines output and validation requirements. -type OutputContract struct { - Format OutputFormat `json:"format"` - ValidationMode ValidationMode `json:"validation_mode"` - SchemaPath string `json:"schema_path"` - RepairAttempts int `json:"repair_attempts"` -} - -// ValidationResult represents output validation state. -type ValidationResult struct { - Status ValidationStatus `json:"status"` - Mode ValidationMode `json:"mode"` - Errors []string `json:"errors,omitempty"` - SchemaPath string `json:"schema_path,omitempty"` - RepairAttempts int `json:"repair_attempts"` - IsValid bool `json:"is_valid"` -} - -// TokenUsage tracks token consumption. -type TokenUsage struct { - PromptTokens int `json:"prompt_tokens"` - CompletionTokens int `json:"completion_tokens"` - TotalTokens int `json:"total_tokens"` - CachedTokens int `json:"cached_tokens"` - CacheWriteTokens int `json:"cache_write_tokens"` -} - -// RenderedPrompt is the fully rendered prompt passed to an LLM client. -type RenderedPrompt struct { - SessionID string `json:"session_id,omitempty"` - Messages []RenderedMessage `json:"messages"` -} - -// RenderedMessage is a rendered chat message. -type RenderedMessage struct { - Role string `json:"role"` - Content string `json:"content"` - CacheControl *CacheControl `json:"cache_control,omitempty"` -} - -// CacheControl describes provider cache metadata attached to prompt content. -type CacheControl struct { - Type CacheControlType `json:"type"` - TTL string `json:"ttl,omitempty"` -} - -// StructuredOutputSpec describes provider-level structured output. -type StructuredOutputSpec struct { - Type StructuredOutputType `json:"type"` - JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"` -} - -// StructuredOutputJSONSpec contains JSON Schema output constraints. -type StructuredOutputJSONSpec struct { - Name string `json:"name"` - Strict bool `json:"strict"` - Schema any `json:"schema"` -} - -// LLMClient executes rendered prompts for Engine.Run. -type LLMClient interface { - Generate(context.Context, GenerateRequest) (*GenerateResponse, error) -} - -// GenerateRequest is passed to an injected LLM client. -type GenerateRequest struct { - Prompt RenderedPrompt `json:"prompt"` - Target ExecutionTarget `json:"target"` - TargetPresence ExecutionTargetPresence `json:"target_presence"` - StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` - APIKey string `json:"-"` -} - -// GenerateResponse is returned by an injected LLM client. -type GenerateResponse struct { - Content string `json:"content"` - Usage TokenUsage `json:"usage"` -} - -// File returns a file-backed artifact reference. -func File(path string) ArtifactRef { - return ArtifactRef{Type: ArtifactRefFile, URI: path} -} - -// Inline returns an inline artifact reference. -func Inline(body string) ArtifactRef { - return ArtifactRef{Type: ArtifactRefInline, Body: body} -} - -// InlineWithURI returns an inline artifact reference with URI metadata. -func InlineWithURI(uri string, body string) ArtifactRef { - return ArtifactRef{Type: ArtifactRefInline, URI: uri, Body: body} -}