diff --git a/README.md b/README.md index b16b228..ea0f46e 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,21 @@ # Promptkit -Promptkit is the reusable Go prompt-execution framework being separated from -Scriptorium. Its module path is: +Promptkit is a reusable Go library for preparing and executing prompt-defined +LLM workflows. Its module path is: ```text gitea.maximumdirect.net/eric/promptkit ``` -Framework extraction is in progress. The repository now contains the -`internal/domain` model, application-neutral `internal/defaults`, and -`internal/filecatalog` helpers that form the implementation foundation. It also -contains internal prompt-definition and profile repositories, the embedded -built-in profile catalog, artifact loading, Go-template rendering, output -validation, model generation, and orchestration. These internal packages are -not a consumer API, and the root package does not yet provide a usable public -framework API, so there is no installation or usage example at this time. +The root `promptkit` package provides the supported public engine. Consumers +can configure filesystem or in-memory prompt, profile, and schema sources, +prepare requests without generation, run requests with the built-in +OpenAI-compatible client, or inject their own model client and artifact reader. +See the [Go package consumer guide](docs/consumers/pkg-promptkit.md) for the +public workflow and contract. Contributors should start with the [development guide](docs/development.md). The [architecture policy](docs/policy/architecture.md) defines the library -boundary and constraints that future framework work must preserve. +boundary and constraints that framework work must preserve. Promptkit is licensed under the [GNU General Public License version 3](LICENSE). diff --git a/architecture_test.go b/architecture_test.go new file mode 100644 index 0000000..c680b9f --- /dev/null +++ b/architecture_test.go @@ -0,0 +1,89 @@ +package promptkit_test + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +const formerModulePath = "gitea.maximumdirect.net/eric/" + "scrip" + "torium" + +func TestRepositoryDoesNotImportFormerModule(t *testing.T) { + violations, err := findFormerModuleImports(".") + if err != nil { + t.Fatalf("inspect repository imports: %v", err) + } + if len(violations) > 0 { + t.Fatalf("repository imports the former module:\n%s", strings.Join(violations, "\n")) + } +} + +func TestFormerModuleGuardFindsNestedImport(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "nested", "package") + if err := os.MkdirAll(nested, 0o755); err != nil { + t.Fatalf("create nested package: %v", err) + } + + sourcePath := filepath.Join(nested, "violation.go") + source := "package nested\n\nimport _ " + strconv.Quote(formerModulePath+"/internal/domain") + "\n" + if err := os.WriteFile(sourcePath, []byte(source), 0o600); err != nil { + t.Fatalf("write nested source: %v", err) + } + + violations, err := findFormerModuleImports(root) + if err != nil { + t.Fatalf("inspect nested imports: %v", err) + } + if len(violations) != 1 { + t.Fatalf("violations = %v, want one nested import", violations) + } + if !strings.Contains(violations[0], "violation.go") || + !strings.Contains(violations[0], formerModulePath+"/internal/domain") { + t.Fatalf("violation = %q, want file and import path", violations[0]) + } +} + +func findFormerModuleImports(root string) ([]string, error) { + var violations []string + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + switch entry.Name() { + case ".git", "generated", "vendor": + return filepath.SkipDir + } + return nil + } + if filepath.Ext(path) != ".go" { + return nil + } + + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly|parser.ParseComments) + if err != nil { + return err + } + if ast.IsGenerated(file) { + return nil + } + for _, spec := range file.Imports { + importPath, err := strconv.Unquote(spec.Path.Value) + if err != nil { + return err + } + if importPath == formerModulePath || strings.HasPrefix(importPath, formerModulePath+"/") { + violations = append(violations, path+": "+importPath) + } + } + return nil + }) + return violations, err +} diff --git a/artifact_reader.go b/artifact_reader.go new file mode 100644 index 0000000..ea470dc --- /dev/null +++ b/artifact_reader.go @@ -0,0 +1,39 @@ +package promptkit + +import ( + "context" + "errors" + + artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact" + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +var errNilArtifactReaderResponse = errors.New("artifact reader returned nil artifact without error") + +type publicArtifactReaderAdapter struct { + reader ArtifactReader +} + +var _ artifactadapter.Reader = publicArtifactReaderAdapter{} + +func (a publicArtifactReaderAdapter) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { + artifact, err := a.reader.Read(ctx, ArtifactRef{ + Type: ArtifactRefType(ref.Type), + URI: ref.URI, + Body: ref.Body, + }) + if err != nil { + return nil, err + } + if artifact == nil { + return nil, errNilArtifactReaderResponse + } + return &domain.Artifact{ + Name: artifact.Name, + ContentType: artifact.ContentType, + Body: copyBytes(artifact.Body), + URI: artifact.URI, + Size: artifact.Size, + Hash: artifact.Hash, + }, nil +} diff --git a/artifact_reader_internal_test.go b/artifact_reader_internal_test.go new file mode 100644 index 0000000..fb764fe --- /dev/null +++ b/artifact_reader_internal_test.go @@ -0,0 +1,36 @@ +package promptkit + +import ( + "context" + "testing" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +func TestPublicArtifactReaderAdapterCopiesBody(t *testing.T) { + reader := internalArtifactReaderFake{ + artifact: &Artifact{Body: []byte("original")}, + } + adapter := publicArtifactReaderAdapter{reader: &reader} + + artifact, err := adapter.Read(context.Background(), domain.ArtifactRef{ + Type: domain.ArtifactRefInline, + URI: "memory://input", + Body: "input", + }) + if err != nil { + t.Fatalf("read artifact: %v", err) + } + artifact.Body[0] = 'X' + if got := string(reader.artifact.Body); got != "original" { + t.Fatalf("reader artifact body was mutated: %q", got) + } +} + +type internalArtifactReaderFake struct { + artifact *Artifact +} + +func (r *internalArtifactReaderFake) Read(context.Context, ArtifactRef) (*Artifact, error) { + return r.artifact, nil +} diff --git a/convert.go b/convert.go new file mode 100644 index 0000000..933ba15 --- /dev/null +++ b/convert.go @@ -0,0 +1,406 @@ +package promptkit + +import ( + "reflect" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) { + execution, err := toDomainExecutionTargetOverride(req.Execution) + if err != nil { + return domain.RunRequest{}, err + } + return domain.RunRequest{ + PromptID: req.PromptID, + PromptVersion: req.PromptVersion, + ProfileID: req.ProfileID, + APIKey: req.APIKey, + Inputs: toDomainArtifactRefMap(req.Inputs), + Vars: copyStringMap(req.Vars), + Execution: execution, + Validation: toDomainOutputContractPtr(req.Validation), + Metadata: copyStringMap(req.Metadata), + }, nil +} + +func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun { + if prepared == nil { + return nil + } + return &PreparedRun{ + PromptID: prepared.PromptID, + PromptVersion: prepared.PromptVersion, + PromptHash: prepared.PromptHash, + SelectedProfileID: prepared.SelectedProfileID, + EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams), + OutputContract: fromDomainOutputContract(prepared.OutputContract), + StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput), + InputHashes: copyStringMap(prepared.InputHashes), + SessionID: prepared.SessionID, + RenderedPromptHash: prepared.RenderedPromptHash, + Messages: fromDomainRenderedMessages(prepared.Messages), + StartTime: prepared.StartTime, + EndTime: prepared.EndTime, + DurationMS: prepared.DurationMS, + } +} + +func fromDomainRunResult(result *domain.RunResult) *RunResult { + if result == nil { + return nil + } + return &RunResult{ + RunID: result.RunID, + Artifact: fromDomainArtifact(result.Artifact), + RawOutput: result.RawOutput, + Validation: fromDomainValidationResult(result.Validation), + PromptID: result.PromptID, + PromptVersion: result.PromptVersion, + PromptHash: result.PromptHash, + RenderedPromptHash: result.RenderedPromptHash, + SelectedProfileID: result.SelectedProfileID, + ModelName: result.ModelName, + Endpoint: result.Endpoint, + EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams), + InputHashes: copyStringMap(result.InputHashes), + Usage: fromDomainTokenUsage(result.Usage), + StartTime: result.StartTime, + EndTime: result.EndTime, + Duration: result.Duration, + } +} + +func fromDomainGenerateRequest(req domain.GenerateRequest) GenerateRequest { + return GenerateRequest{ + Prompt: fromDomainRenderedPrompt(req.Prompt), + Target: fromDomainExecutionTarget(req.Target), + TargetPresence: fromDomainExecutionTargetPresence(req.TargetPresence), + StructuredOutput: fromDomainStructuredOutputSpec(req.StructuredOutput), + APIKey: req.Target.APIKey, + } +} + +func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse { + if resp == nil { + return nil + } + return &domain.GenerateResponse{ + Content: resp.Content, + Usage: toDomainTokenUsage(resp.Usage), + } +} + +func fromDomainRenderedPrompt(prompt domain.RenderedPrompt) RenderedPrompt { + return RenderedPrompt{ + SessionID: prompt.SessionID, + Messages: fromDomainRenderedMessages(prompt.Messages), + } +} + +func toDomainArtifactRefMap(src map[string]ArtifactRef) map[string]domain.ArtifactRef { + if src == nil { + return nil + } + out := make(map[string]domain.ArtifactRef, len(src)) + for k, v := range src { + out[k] = toDomainArtifactRef(v) + } + return out +} + +func toDomainArtifactRef(ref ArtifactRef) domain.ArtifactRef { + return domain.ArtifactRef{ + Type: domain.ArtifactRefType(ref.Type), + URI: ref.URI, + Body: ref.Body, + } +} + +func fromDomainArtifact(artifact domain.Artifact) Artifact { + return Artifact{ + Name: artifact.Name, + ContentType: artifact.ContentType, + Body: copyBytes(artifact.Body), + URI: artifact.URI, + Size: artifact.Size, + Hash: artifact.Hash, + } +} + +func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain.ExecutionTargetOverride, error) { + if override == nil { + return nil, nil + } + extraParams, err := copyPublicJSONMap(override.ExtraParams) + if err != nil { + return nil, err + } + return &domain.ExecutionTargetOverride{ + Endpoint: override.Endpoint, + Model: override.Model, + Temperature: copyFloat64Ptr(override.Temperature), + MaxTokens: copyIntPtr(override.MaxTokens), + TopP: copyFloat64Ptr(override.TopP), + TimeoutSeconds: copyIntPtr(override.TimeoutSeconds), + ServiceTier: override.ServiceTier, + ReasoningEffort: override.ReasoningEffort, + APIKeyEnv: override.APIKeyEnv, + ExtraParams: extraParams, + }, nil +} + +func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget { + return ExecutionTarget{ + Endpoint: target.Endpoint, + Model: target.Model, + Temperature: target.Temperature, + MaxTokens: target.MaxTokens, + TopP: target.TopP, + TimeoutSeconds: target.TimeoutSeconds, + ServiceTier: target.ServiceTier, + ReasoningEffort: target.ReasoningEffort, + APIKeyEnv: target.APIKeyEnv, + ExtraParams: copyAnyMap(target.ExtraParams), + } +} + +func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence { + return ExecutionTargetPresence{ + Temperature: presence.Temperature, + MaxTokens: presence.MaxTokens, + TopP: presence.TopP, + TimeoutSeconds: presence.TimeoutSeconds, + } +} + +func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract { + if contract == nil { + return nil + } + out := toDomainOutputContract(*contract) + return &out +} + +func toDomainOutputContract(contract OutputContract) domain.OutputContract { + return domain.OutputContract{ + Format: domain.OutputFormat(contract.Format), + ValidationMode: domain.ValidationMode(contract.ValidationMode), + SchemaPath: contract.SchemaPath, + RepairAttempts: contract.RepairAttempts, + } +} + +func fromDomainOutputContract(contract domain.OutputContract) OutputContract { + return OutputContract{ + Format: OutputFormat(contract.Format), + ValidationMode: ValidationMode(contract.ValidationMode), + SchemaPath: contract.SchemaPath, + RepairAttempts: contract.RepairAttempts, + } +} + +func fromDomainValidationResult(result domain.ValidationResult) ValidationResult { + return ValidationResult{ + Status: ValidationStatus(result.Status), + Mode: ValidationMode(result.Mode), + Errors: copyStringSlice(result.Errors), + SchemaPath: result.SchemaPath, + RepairAttempts: result.RepairAttempts, + IsValid: result.IsValid, + } +} + +func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage { + return TokenUsage{ + PromptTokens: usage.PromptTokens, + CompletionTokens: usage.CompletionTokens, + TotalTokens: usage.TotalTokens, + CachedTokens: usage.CachedTokens, + CacheWriteTokens: usage.CacheWriteTokens, + } +} + +func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage { + return domain.TokenUsage{ + PromptTokens: usage.PromptTokens, + CompletionTokens: usage.CompletionTokens, + TotalTokens: usage.TotalTokens, + CachedTokens: usage.CachedTokens, + CacheWriteTokens: usage.CacheWriteTokens, + } +} + +func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMessage { + if messages == nil { + return nil + } + out := make([]RenderedMessage, len(messages)) + for i, msg := range messages { + out[i] = RenderedMessage{ + Role: msg.Role, + Content: msg.Content, + CacheControl: fromDomainCacheControl(msg.CacheControl), + } + } + return out +} + +func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl { + if cacheControl == nil { + return nil + } + return &CacheControl{ + Type: CacheControlType(cacheControl.Type), + TTL: cacheControl.TTL, + } +} + +func fromDomainStructuredOutputSpec(spec *domain.StructuredOutputSpec) *StructuredOutputSpec { + if spec == nil { + return nil + } + out := &StructuredOutputSpec{ + Type: StructuredOutputType(spec.Type), + } + if spec.JSONSchema != nil { + out.JSONSchema = &StructuredOutputJSONSpec{ + Name: spec.JSONSchema.Name, + Strict: spec.JSONSchema.Strict, + Schema: copyAny(spec.JSONSchema.Schema), + } + } + return out +} + +func copyStringMap(src map[string]string) map[string]string { + if src == nil { + return nil + } + out := make(map[string]string, len(src)) + for k, v := range src { + out[k] = v + } + return out +} + +func copyAnyMap(src map[string]any) map[string]any { + if src == nil { + return nil + } + out := make(map[string]any, len(src)) + for k, v := range src { + out[k] = copyAny(v) + } + return out +} + +func copyAny(value any) any { + if value == nil { + return nil + } + switch v := value.(type) { + case map[string]any: + return copyAnyMap(v) + case []any: + out := make([]any, len(v)) + for i, item := range v { + out[i] = copyAny(item) + } + return out + case []string: + return copyStringSlice(v) + case []byte: + return copyBytes(v) + default: + return copyReflectValue(reflect.ValueOf(value)).Interface() + } +} + +func copyReflectValue(value reflect.Value) reflect.Value { + if !value.IsValid() { + return value + } + + switch value.Kind() { + case reflect.Interface: + if value.IsNil() { + return reflect.Zero(value.Type()) + } + copied := copyReflectValue(value.Elem()) + if copied.IsValid() && copied.Type().AssignableTo(value.Type()) { + return copied + } + out := reflect.New(value.Type()).Elem() + out.Set(copied) + return out + case reflect.Pointer: + if value.IsNil() { + return reflect.Zero(value.Type()) + } + out := reflect.New(value.Type().Elem()) + out.Elem().Set(copyReflectValue(value.Elem())) + return out + case reflect.Map: + if value.IsNil() { + return reflect.Zero(value.Type()) + } + out := reflect.MakeMapWithSize(value.Type(), value.Len()) + iter := value.MapRange() + for iter.Next() { + out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value())) + } + return out + case reflect.Slice: + if value.IsNil() { + return reflect.Zero(value.Type()) + } + out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap()) + for i := 0; i < value.Len(); i++ { + out.Index(i).Set(copyReflectValue(value.Index(i))) + } + return out + case reflect.Array: + out := reflect.New(value.Type()).Elem() + for i := 0; i < value.Len(); i++ { + out.Index(i).Set(copyReflectValue(value.Index(i))) + } + return out + default: + return value + } +} + +func copyStringSlice(src []string) []string { + if src == nil { + return nil + } + out := make([]string, len(src)) + copy(out, src) + return out +} + +func copyBytes(src []byte) []byte { + if src == nil { + return nil + } + out := make([]byte, len(src)) + copy(out, src) + return out +} + +func copyFloat64Ptr(src *float64) *float64 { + if src == nil { + return nil + } + v := *src + return &v +} + +func copyIntPtr(src *int) *int { + if src == nil { + return nil + } + v := *src + return &v +} diff --git a/doc.go b/doc.go index ca5bda0..42400e6 100644 --- a/doc.go +++ b/doc.go @@ -1,2 +1,8 @@ -// Package promptkit defines the public package boundary for the Promptkit Go module. +// Package promptkit provides an embeddable engine for preparing and executing +// prompt-defined LLM workflows. +// +// Applications construct an Engine with NewEngine, select filesystem or +// in-memory definition sources with options, and use Prepare or Run to execute +// requests. Concrete repositories, validators, and outbound clients remain +// internal implementation details. package promptkit diff --git a/docs/consumers/pkg-promptkit.md b/docs/consumers/pkg-promptkit.md new file mode 100644 index 0000000..c867f3e --- /dev/null +++ b/docs/consumers/pkg-promptkit.md @@ -0,0 +1,165 @@ +# Package `promptkit` + +Import path: + +```go +import "gitea.maximumdirect.net/eric/promptkit" +``` + +Package `promptkit` is the supported Go contract for in-process prompt +preparation and execution. The declarations and their GoDoc in the +[root package](../../doc.go) own the exact API; this guide explains how the +pieces are used together. + +## Engine Construction And Sources + +Construct an engine with [`NewEngine`, `Config`, and +`Option`](../../engine.go). `PromptDir` is required unless a prompt source +option is supplied. `ProfileDir` optionally overlays built-in profiles, and an +empty `SchemaDir` uses the current directory. `Timeout` is the transport-wide +safety cap for the built-in OpenAI-compatible client. An optional `HTTPClient` +is cloned; its positive timeout takes precedence. + +Nil options are ignored. Invalid construction, including a nil injected client +or artifact reader, returns an error matching `ErrInvalidConfig`. + +The [source options](../../engine.go) replace their matching directory source: + +- `WithPromptFS` and `WithPromptFile` select prompt definitions; +- `WithProfileFS` and `WithProfileFile` overlay built-in profiles; +- `WithProfiles` adds in-memory profiles ahead of file and built-in profiles; +- `WithSchemaFS` and `WithSchemaFile` select JSON Schema documents; +- `WithLLMClient` replaces the built-in model client; and +- `WithArtifactReader` replaces the default reader for every input. + +Prompt-content and schema paths from an `fs.FS` stay within the configured +root. Single-file prompt and profile sources select definitions by YAML ID. +Relative prompt content resolves from its prompt file, while a single schema +is addressed by its base name. + +Per-generation timeout values from profiles or requests are independent of +the transport cap and caller context. An explicit request value of zero +disables only the per-generation deadline. The +[outbound integration contract](../integrations/openai-compatible-chat.md#timeout-and-cancellation) +defines the complete timeout layering. + +## Preparation And Execution + +[`Engine.Prepare` and `Engine.Run`](../../engine.go) accept the public +[`RunRequest`](../../types.go). `Prepare` resolves the prompt, profile, input +artifacts, validation contract, and rendered messages without calling an LLM. +`Run` performs the same preparation, calls the configured client, and validates +the generated content. + +```go +engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: "./prompts", + ProfileDir: "./profiles", +}) +if err != nil { + return err +} + +prepared, err := engine.Prepare(ctx, promptkit.RunRequest{ + PromptID: "meeting.summary", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.File("./transcript.md"), + }, +}) +if err != nil { + return err +} +_ = prepared.Messages +``` + +[`PreparedRun` and `RunResult`](../../types.go) expose copied public values. +Preparation returns effective settings, hashes, rendered messages, selected +profile, structured-output information, and timing without resolved secrets or +model output. Execution adds the generated artifact and raw output, validation +state, model metadata, usage, run ID, and duration. + +A generated-content validation failure returns a result with +`Validation.Status == ValidationFailed`. An inability to perform validation +returns an error matching `ErrValidation`. + +## Requests, Inputs, And Overrides + +The [request and value declarations](../../types.go) own the available fields, +serialized constants, and result shapes. Use `File`, `Inline`, or +`InlineWithURI` to construct artifact references. Required declared inputs and +every input referenced by a template must be supplied. + +`ExecutionTargetOverride` uses pointers for numeric settings so an explicit +zero remains distinct from no override. `ExtraParams` accepts JSON-compatible +strings, booleans, finite numbers, string-keyed objects, arrays or slices, and +nil. Unsupported values, non-string map keys, non-finite numbers, and cycles +match `ErrInvalidConfig` in profiles or `ErrInvalidRequest` in request +overrides. + +Returned requests, profiles, prepared values, results, artifacts, maps, and +slices are isolated from internal engine state. Consumers and injected +extensions should not retain or mutate values owned by another caller. + +## Profiles And Credentials + +[`OpenAICompatibleProfile`](../../profiles.go) constructs an ordinary +in-memory profile for an OpenAI-compatible chat-completions endpoint. +`WithProfiles` rejects duplicate IDs in one call and gives in-memory profiles +precedence over explicit file sources and built-ins. + +Raw API keys do not belong in profiles. File-backed profiles may name an +environment variable, while an in-memory profile can require a request key. +A direct `RunRequest.APIKey` is request-scoped and takes precedence over an +environment lookup for the built-in client. + +API keys are excluded from JSON, prepared values, and results. The public +`String` and `GoString` methods report only whether a direct key is present. +Avoid reflection-based dumps of request structs, which can bypass that +redaction. + +## Extension Interfaces + +The [`LLMClient`, `GenerateRequest`, and +`GenerateResponse`](../../types.go) boundary lets a consumer replace model +generation. Injected clients receive copied rendered messages, effective +settings, explicit numeric-setting presence, structured-output constraints, +and the request-scoped key. They return generated content and token usage. + +The [`ArtifactReader`](../../types.go) boundary replaces the default inline and +file reader for every input. Readers provide artifact content and metadata; the +engine fills an empty artifact name from the input-map key. A reader error +matches `ErrArtifactLoad` while preserving the original identity for +`errors.Is`. A nil artifact with a nil error is also an artifact-load failure. + +Extensions should honor context cancellation and avoid logging raw prompts, +artifacts, or credentials. + +## Errors + +The [public error declarations](../../engine.go) and +[mapping](../../errors.go) preserve these sentinel checks through `errors.Is`: + +- `ErrInvalidConfig` +- `ErrInvalidRequest` +- `ErrPromptNotFound` +- `ErrProfileNotFound` +- `ErrProfileRequired` +- `ErrPromptLoad` +- `ErrProfileLoad` +- `ErrAPIKeyEnvMissing` +- `ErrArtifactLoad` +- `ErrPromptRender` +- `ErrLLMGenerate` +- `ErrValidation` + +`ErrProfileRequired` and `ErrAPIKeyEnvMissing` also match +`ErrInvalidRequest`, allowing either broad request handling or a specific +condition. Wrapped collaborator errors retain their identity where the public +contract promises it. + +## Consumer Boundary + +Promptkit is an importable library. It does not own a command, inbound HTTP +API, process configuration, or deployment policy. Scriptorium is one +downstream application that maps this root package contract into those +application concerns. diff --git a/docs/integrations/openai-compatible-chat.md b/docs/integrations/openai-compatible-chat.md index a6985bd..039f69f 100644 --- a/docs/integrations/openai-compatible-chat.md +++ b/docs/integrations/openai-compatible-chat.md @@ -5,8 +5,8 @@ This document defines the outbound HTTP behavior implemented by Promptkit's internal OpenAI-compatible model client. The [internal model-client document](../internal/llm.md) owns implementation flow, -errors, and test ownership. The client is not yet available through a usable -public Promptkit engine. +errors, and test ownership. The root Promptkit engine uses this client by +default unless a consumer injects another implementation. ## Endpoint And Method diff --git a/docs/internal/llm.md b/docs/internal/llm.md index 49b36cc..dd6899f 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -8,8 +8,8 @@ and the [OpenAI-compatible chat integration](../integrations/openai-compatible-chat.md) owns the observable outbound HTTP contract. -The client is implemented only under `internal/llm`. The root package does not -yet assemble it into a usable public engine. +The concrete client remains under `internal/llm`. The root engine assembles it +as the default implementation behind Promptkit's public client boundary. ## Components And Flow @@ -40,8 +40,8 @@ non-success provider statuses, and malformed successful responses. Provider response bodies are not included in non-success errors. Caller cancellation and deadline failures during the outbound request are -reported as request execution failures. The future runner can classify these -identities without depending on HTTP status mapping. +reported as request execution failures. The runner classifies these identities +without depending on HTTP status mapping. ## Test Ownership diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 637057f..2f349de 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -11,7 +11,7 @@ contributor workflow and validation. | Component | Implemented responsibility | References | | --- | --- | --- | -| Root `promptkit` package | Establishes the public package boundary for the Go module. It does not yet provide migrated framework behavior or exported APIs. | [Package declaration](../../doc.go) | +| Root `promptkit` package | Provides the supported engine facade, source and injection options, public request and result values, built-in profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [engine assembly](../../engine.go) | | `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) | | `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) | | `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) | @@ -24,9 +24,8 @@ contributor workflow and validation. | `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests, response decoding, authentication, and deadline handling. | [Internal model client](llm.md) | | `internal/usecase` | Coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) | -These packages provide the internal model, source, rendering, validation, and -model-client workflow. A usable public engine is not implemented in Promptkit -yet. +The root package assembles these internal components without exposing their +representations. Consumers depend only on the root facade. ## Maintenance diff --git a/docs/internal/runner.md b/docs/internal/runner.md index 8036b77..6096d5a 100644 --- a/docs/internal/runner.md +++ b/docs/internal/runner.md @@ -9,8 +9,8 @@ artifact, rendering, and validation behavior, while the [model-client document](llm.md) owns generation behavior and failure categories. -The runner remains under `internal/usecase`. The root package does not yet -assemble it into a usable public engine. +The runner remains under `internal/usecase` and is assembled by the root +Promptkit engine. Its concrete type is not part of the public API. ## Collaborators @@ -65,8 +65,8 @@ input hashes, token usage, a generated run identifier, and UTC timing. Package errors distinguish invalid requests, required profile selection, credential failures, and prompt, profile, artifact, rendering, generation, and -validation failures. Wrapping preserves the package identities needed by the -future facade and retains collaborator identities where they are part of the +validation failures. Wrapping preserves the package identities mapped by the +public facade and retains collaborator identities where they are part of the internal contract. Context cancellation propagates through the invoked collaborator and is classified by the owning operation. diff --git a/docs/internal/sources.md b/docs/internal/sources.md index 338334f..afe03db 100644 --- a/docs/internal/sources.md +++ b/docs/internal/sources.md @@ -6,7 +6,7 @@ This document describes Promptkit's implemented internal source, artifact, rendering, and output-validation behavior. The [architecture policy](../policy/architecture.md) owns the library boundary and dependency rules. None of these internal packages is a supported consumer API, -and the root package does not yet assemble them into a usable engine. +and the root engine assembles them behind its public source options and values. ## Prompt Definitions diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index fe0704b..f322c2e 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -13,8 +13,8 @@ Promptkit is an importable Go library. It does not provide a runnable command, an HTTP service, or another application process. The module root contains package `promptkit`, which is the public facade. It -declares the module's public package boundary but does not yet provide a usable -exported framework API. +provides the supported engine, configuration and source options, requests, +results, public values, extension interfaces, profiles, and error sentinels. The implemented internal components consist of: @@ -40,17 +40,19 @@ The implemented internal components consist of: - `internal/usecase`, which coordinates preparation and execution across the internal framework components. -The defaults and renderer depend on the domain model. Prompt-definition and -profile repositories use the domain model, file catalog, and YAML decoder. The -built-in profile repository supplies an embedded `fs.FS` to the profile -package. Artifact reading uses the domain model and application-neutral -defaults. Validation uses the domain model, file catalog, and JSON Schema -implementation. The model client uses the domain model, application-neutral -defaults, and an injected or standard-library HTTP client. The use-case runner -depends on the narrow interfaces owned by each internal component. The public -engine has not yet been extracted. +The root facade assembles the internal repositories, renderer, validator, +outbound client, and use-case runner while translating public values and +errors at the library boundary. The defaults and renderer depend on the domain +model. Prompt-definition and profile repositories use the domain model, file +catalog, and YAML decoder. The built-in profile repository supplies an +embedded `fs.FS` to the profile package. Artifact reading uses the domain model +and application-neutral defaults. Validation uses the domain model, file +catalog, and JSON Schema implementation. The model client uses the domain +model, application-neutral defaults, and an injected or standard-library HTTP +client. The use-case runner depends on the narrow interfaces owned by each +internal component. -Future framework extraction must follow this dependency direction: +The current implementation follows this dependency direction: ```text downstream consumers, including Scriptorium @@ -65,16 +67,13 @@ downstream consumers, including Scriptorium narrow injected abstractions ``` -The facade may coordinate internal components once the public engine is -extracted. Internal components must depend on narrow abstractions for behavior -supplied from outside the library; they must not depend on consumers or on -Scriptorium. This diagram is the target dependency direction for later -extraction and does not assert that the public facade already assembles the -implemented foundation. +The facade coordinates internal components and adapts the supported public +extension interfaces to narrow internal abstractions. Internal components must +not depend on consumers or on Scriptorium. ## Repository And Consumer Boundary -Scriptorium is a downstream application that will consume Promptkit through +Scriptorium is a downstream application that consumes Promptkit through the supported public facade. It is not a Promptkit package and must not become an internal dependency. @@ -147,7 +146,6 @@ state. ## Current-State Maintenance -This policy distinguishes present implementation from constraints on future -framework extraction. Do not list planned packages as implemented components. -When extraction introduces a package, update the internal inventory and the -owning contract or subsystem document in the same change. +Do not list planned packages as implemented components. When implementation +introduces a package, update the internal inventory and the owning contract or +subsystem document in the same change. diff --git a/engine.go b/engine.go new file mode 100644 index 0000000..8a14155 --- /dev/null +++ b/engine.go @@ -0,0 +1,342 @@ +package promptkit + +import ( + "context" + "errors" + "fmt" + "io/fs" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact" + "gitea.maximumdirect.net/eric/promptkit/internal/defaults" + "gitea.maximumdirect.net/eric/promptkit/internal/llm" + "gitea.maximumdirect.net/eric/promptkit/internal/profile" + "gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin" + "gitea.maximumdirect.net/eric/promptkit/internal/prompt" + "gitea.maximumdirect.net/eric/promptkit/internal/promptdef" + "gitea.maximumdirect.net/eric/promptkit/internal/usecase" + "gitea.maximumdirect.net/eric/promptkit/internal/validate" +) + +// ErrInvalidConfig indicates invalid public engine configuration. +var ErrInvalidConfig = errors.New("invalid engine configuration") + +var ( + ErrInvalidRequest = errors.New("invalid run request") + ErrPromptNotFound = errors.New("prompt not found") + ErrProfileNotFound = errors.New("profile not found") + ErrProfileRequired = errors.New("profile selection is required") + ErrPromptLoad = errors.New("failed to load prompt definition") + ErrProfileLoad = errors.New("failed to load execution profile") + ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable") + ErrArtifactLoad = errors.New("failed to load artifact") + ErrPromptRender = errors.New("failed to render prompt") + ErrLLMGenerate = errors.New("failed to generate output") + ErrValidation = errors.New("failed to validate output") +) + +// Engine prepares and runs Promptkit prompt requests. +type Engine struct { + runner *usecase.Runner +} + +// Config configures a public Promptkit engine. +type Config struct { + PromptDir string + ProfileDir string + SchemaDir string + // Timeout is the transport-wide safety cap for the built-in LLM client + // when HTTPClient is absent or has a non-positive timeout. + Timeout time.Duration + // HTTPClient is cloned for the built-in LLM client. Its positive Timeout + // takes precedence over Config.Timeout as the transport-wide safety cap. + HTTPClient *http.Client +} + +// Option customizes engine construction. +type Option interface { + apply(*engineOptions) error +} + +type optionFunc func(*engineOptions) error + +func (f optionFunc) apply(options *engineOptions) error { + return f(options) +} + +type engineOptions struct { + llmClient llm.Client + artifactReader artifactadapter.Reader + promptDefs promptdef.Repository + profiles profile.Repository + memoryProfiles profile.Repository + validator validate.Validator + promptSource bool + profileSource bool + memorySource bool + validatorSource bool + artifactSource bool +} + +// WithLLMClient injects a custom LLM client for execution. +func WithLLMClient(client LLMClient) Option { + return optionFunc(func(options *engineOptions) error { + if client == nil { + return ErrInvalidConfig + } + options.llmClient = publicLLMClientAdapter{client: client} + return nil + }) +} + +// WithArtifactReader injects a reader for every input artifact reference. +func WithArtifactReader(reader ArtifactReader) Option { + return optionFunc(func(options *engineOptions) error { + if reader == nil { + return ErrInvalidConfig + } + options.artifactReader = publicArtifactReaderAdapter{reader: reader} + options.artifactSource = true + return nil + }) +} + +// WithPromptFS loads prompt definitions from fsys under root. +// +// The source uses the same strict prompt YAML rules as configured prompt +// directories, and prompt content_file paths resolve within this source. +func WithPromptFS(fsys fs.FS, root string) Option { + return optionFunc(func(options *engineOptions) error { + if fsys == nil { + return ErrInvalidConfig + } + if strings.TrimSpace(root) == "" { + return ErrInvalidConfig + } + options.promptDefs = promptdef.NewFSRepository(fsys, root) + options.promptSource = true + return nil + }) +} + +// WithPromptFile loads prompt definitions from the single prompt file at path. +// +// Relative prompt content_file paths resolve from the file's directory. +func WithPromptFile(path string) Option { + return optionFunc(func(options *engineOptions) error { + fsys, root, err := fileSource(path) + if err != nil { + return err + } + options.promptDefs = promptdef.NewFSRepository(fsys, root) + options.promptSource = true + return nil + }) +} + +// WithProfileFS loads execution profiles from fsys under root. +// +// Profiles from this source overlay built-in profiles. Profile YAML must use +// api_key_env for environment-based credentials; raw API keys are rejected. +func WithProfileFS(fsys fs.FS, root string) Option { + return optionFunc(func(options *engineOptions) error { + if fsys == nil { + return ErrInvalidConfig + } + if strings.TrimSpace(root) == "" { + return ErrInvalidConfig + } + options.profiles = profile.NewFSRepository(fsys, root) + options.profileSource = true + return nil + }) +} + +// WithProfileFile loads execution profiles from the single profile file at path. +// +// The profile overlays built-in profiles. Profile YAML must use api_key_env for +// environment-based credentials; raw API keys are rejected. +func WithProfileFile(path string) Option { + return optionFunc(func(options *engineOptions) error { + fsys, root, err := fileSource(path) + if err != nil { + return err + } + options.profiles = profile.NewFSRepository(fsys, root) + options.profileSource = true + return nil + }) +} + +// WithProfiles configures in-memory profiles that take precedence over +// configured profile files and built-in profiles. +func WithProfiles(profiles ...Profile) Option { + return optionFunc(func(options *engineOptions) error { + repo, err := newMemoryProfileRepository(profiles) + if err != nil { + return err + } + options.memoryProfiles = repo + options.memorySource = true + return nil + }) +} + +// WithSchemaFS loads JSON Schema documents from fsys under root. +// +// Prompt schema_path values resolve within this source when schema validation +// or structured output is requested. +func WithSchemaFS(fsys fs.FS, root string) Option { + return optionFunc(func(options *engineOptions) error { + if fsys == nil { + return ErrInvalidConfig + } + if strings.TrimSpace(root) == "" { + return ErrInvalidConfig + } + options.validator = validate.NewFSValidator(fsys, root) + options.validatorSource = true + return nil + }) +} + +// WithSchemaFile loads JSON Schema documents from the single schema file at path. +// +// Prompt schema_path values refer to the file's base name. +func WithSchemaFile(path string) Option { + return optionFunc(func(options *engineOptions) error { + fsys, root, err := fileSource(path) + if err != nil { + return err + } + options.validator = validate.NewFSValidator(fsys, root) + options.validatorSource = true + return nil + }) +} + +// NewEngine constructs an Engine from configuration and options. +func NewEngine(cfg Config, opts ...Option) (*Engine, error) { + var options engineOptions + for _, opt := range opts { + if opt == nil { + continue + } + if err := opt.apply(&options); err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) + } + } + + promptDefs := options.promptDefs + if !options.promptSource { + if strings.TrimSpace(cfg.PromptDir) == "" { + return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig) + } + promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir) + } + + profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir) + if options.profileSource { + profiles = builtin.NewRepositoryWithPrimary(options.profiles) + } + if options.memorySource { + profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles) + } + + validator := options.validator + if !options.validatorSource { + schemaDir := cfg.SchemaDir + if strings.TrimSpace(schemaDir) == "" { + schemaDir = defaults.SchemaDirDefault + } + validator = validate.NewStandardValidator(schemaDir) + } + + llmClient := options.llmClient + if llmClient == nil { + var err error + llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ + Timeout: cfg.Timeout, + HTTPClient: cfg.HTTPClient, + }) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) + } + } + + artifacts := options.artifactReader + if !options.artifactSource { + artifacts = artifactadapter.NewCompositeReader() + } + + return &Engine{ + runner: usecase.NewRunner( + promptDefs, + profiles, + artifacts, + prompt.NewGoRenderer(), + llmClient, + validator, + ), + }, nil +} + +func fileSource(name string) (fs.FS, string, error) { + cleanName := strings.TrimSpace(name) + if cleanName == "" { + return nil, "", ErrInvalidConfig + } + dir := filepath.Dir(cleanName) + base := filepath.Base(cleanName) + if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" { + return nil, "", ErrInvalidConfig + } + info, err := os.Stat(cleanName) + if err != nil { + return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err) + } + if info.IsDir() { + return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName) + } + return os.DirFS(dir), filepath.ToSlash(base), nil +} + +// Prepare resolves a prompt request without calling an LLM. +func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) { + if e == nil || e.runner == nil { + return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) + } + + domainReq, err := toDomainRunRequest(req) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + + prepared, err := e.runner.Prepare(ctx, domainReq) + if err != nil { + return nil, mapPublicError(err) + } + return fromDomainPreparedRun(prepared), nil +} + +// Run executes a prompt request and returns the generated artifact and metadata. +func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { + if e == nil || e.runner == nil { + return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) + } + + domainReq, err := toDomainRunRequest(req) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + + result, err := e.runner.Run(ctx, domainReq) + if err != nil { + return nil, mapPublicError(err) + } + return fromDomainRunResult(result), nil +} diff --git a/engine_test.go b/engine_test.go new file mode 100644 index 0000000..220f72f --- /dev/null +++ b/engine_test.go @@ -0,0 +1,2559 @@ +package promptkit_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "testing/fstest" + "time" + + "gitea.maximumdirect.net/eric/promptkit" +) + +const ( + frameworkContractRoot = "./testdata/framework" + frameworkPromptDir = frameworkContractRoot + "/prompts" + frameworkProfileDir = frameworkContractRoot + "/profiles" + frameworkSchemaDir = frameworkContractRoot + "/schemas" + + frameworkMarkdownSummaryPromptID = "contract.markdown_summary" + frameworkStructuredEventsPromptID = "contract.structured_events" + frameworkFastProfileID = "contract-fast" + frameworkQualityProfileID = "contract-quality" + + frameworkTranscriptPath = frameworkContractRoot + "/fixtures/transcript.md" + frameworkGlossaryPath = frameworkContractRoot + "/fixtures/glossary.yml" +) + +func TestNewEngineRejectsMissingPromptDir(t *testing.T) { + _, err := promptkit.NewEngine(promptkit.Config{ProfileDir: frameworkProfileDir}) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) + } +} + +func TestNewEngineAcceptsMissingProfileDir(t *testing.T) { + _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}) + if err != nil { + t.Fatalf("expected missing profile dir to use built-ins, got %v", err) + } +} + +func TestPrepareWorksWithFrameworkContractCorpus(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("construct engine from framework contract corpus: %v", err) + } + + tests := []struct { + name string + promptID string + profileID string + model string + structured bool + }{ + { + name: "markdown summary", + promptID: frameworkMarkdownSummaryPromptID, + profileID: frameworkFastProfileID, + model: "contract-fast-model", + }, + { + name: "structured events", + promptID: frameworkStructuredEventsPromptID, + profileID: frameworkQualityProfileID, + model: "contract-quality-model", + structured: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: tt.promptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.File(frameworkTranscriptPath), + "glossary": promptkit.File(frameworkGlossaryPath), + }, + }) + if err != nil { + t.Fatalf("prepare framework contract prompt: %v", err) + } + if prepared.PromptID != tt.promptID { + t.Fatalf("unexpected prompt id: got %q, want %q", prepared.PromptID, tt.promptID) + } + if prepared.SelectedProfileID != tt.profileID { + t.Fatalf("unexpected selected profile: got %q, want %q", prepared.SelectedProfileID, tt.profileID) + } + if prepared.EffectiveModelParams.Model != tt.model { + t.Fatalf("unexpected effective model: got %q, want %q", prepared.EffectiveModelParams.Model, tt.model) + } + if len(prepared.Messages) != 2 { + t.Fatalf("expected rendered messages, got %d", len(prepared.Messages)) + } + if !strings.Contains(prepared.Messages[1].Content, "Nia labels the archive.") { + t.Fatalf("expected relative prompt content to render the transcript, got %q", prepared.Messages[1].Content) + } + if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" { + t.Fatalf("expected input hashes, got %#v", prepared.InputHashes) + } + + if !tt.structured { + if prepared.StructuredOutput != nil { + t.Fatalf("expected no structured output specification, got %#v", prepared.StructuredOutput) + } + return + } + + if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil { + t.Fatalf("expected loaded JSON Schema structured output, got %#v", prepared.StructuredOutput) + } + schema, ok := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any) + if !ok || schema["type"] != "object" { + t.Fatalf("expected loaded object JSON Schema, got %#v", prepared.StructuredOutput.JSONSchema.Schema) + } + properties, ok := schema["properties"].(map[string]any) + if !ok || properties["events"] == nil { + t.Fatalf("expected loaded events schema property, got %#v", schema) + } + }) + } +} + +func TestPrepareWorksWithInlineInputs(t *testing.T) { + engine := newContractEngine(t) + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin scouts the tower.\nKara lights a lantern."), + "glossary": promptkit.InlineWithURI("memory://glossary.yml", "party:\n - Rin\n - Kara\n"), + }, + }) + if err != nil { + t.Fatalf("expected prepare to succeed, got %v", err) + } + if len(prepared.Messages) != 2 { + t.Fatalf("expected rendered messages, got %d", len(prepared.Messages)) + } + rendered := prepared.Messages[1].Content + if !strings.Contains(rendered, "Rin scouts the tower.") || !strings.Contains(rendered, "party:") { + t.Fatalf("expected inline inputs in rendered prompt, got %q", rendered) + } +} + +func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) { + const envName = "PROMPTKIT_API_KEY" + const secret = "public-api-test-secret" + t.Setenv(envName, secret) + + profileDir := t.TempDir() + writePublicProfileFileWithAPIKeyEnv(t, profileDir, "prepared-secret", "http://localhost:8000/v1", "prepared-secret-model", envName) + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: profileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkStructuredEventsPromptID, + ProfileID: "prepared-secret", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.File(frameworkTranscriptPath), + "glossary": promptkit.File(frameworkGlossaryPath), + }, + }) + if err != nil { + t.Fatalf("expected prepare to succeed, got %v", err) + } + + payload, err := json.Marshal(prepared) + if err != nil { + t.Fatalf("expected prepared run to marshal, got %v", err) + } + out := string(payload) + if strings.Contains(out, secret) { + t.Fatalf("prepared run JSON leaked raw API key value: %s", out) + } + if !strings.Contains(out, envName) { + t.Fatalf("prepared run JSON should retain api_key_env name, got %s", out) + } + for _, forbidden := range []string{"TargetPresence", "target_presence"} { + if strings.Contains(out, forbidden) { + t.Fatalf("prepared run JSON exposed internal target presence metadata %q: %s", forbidden, out) + } + } +} + +func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) { + const secret = "run-request-secret" + req := promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: frameworkFastProfileID, + APIKey: secret, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + } + + for _, formatted := range []string{ + fmt.Sprint(req), + fmt.Sprintf("%+v", req), + fmt.Sprintf("%#v", req), + } { + if strings.Contains(formatted, secret) { + t.Fatalf("formatted RunRequest leaked API key: %s", formatted) + } + if !strings.Contains(formatted, "APIKeySet:true") { + t.Fatalf("formatted RunRequest should indicate an API key is set, got %s", formatted) + } + } + + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("expected RunRequest to marshal, got %v", err) + } + if strings.Contains(string(payload), secret) { + t.Fatalf("RunRequest JSON leaked API key: %s", payload) + } +} + +func TestGenerateRequestFormattingRedactsDirectAPIKey(t *testing.T) { + const secret = "generate-request-secret" + req := promptkit.GenerateRequest{ + Prompt: promptkit.RenderedPrompt{Messages: []promptkit.RenderedMessage{ + {Role: "user", Content: "secret prompt content"}, + }}, + Target: promptkit.ExecutionTarget{ + Model: "test-model", + ExtraParams: map[string]any{ + "provider_option": "on", + }, + }, + APIKey: secret, + } + + for _, formatted := range []string{ + fmt.Sprint(req), + fmt.Sprintf("%+v", req), + fmt.Sprintf("%#v", req), + } { + if strings.Contains(formatted, secret) { + t.Fatalf("formatted GenerateRequest leaked API key: %s", formatted) + } + if strings.Contains(formatted, "secret prompt content") { + t.Fatalf("formatted GenerateRequest leaked prompt content: %s", formatted) + } + if !strings.Contains(formatted, "APIKeySet:true") { + t.Fatalf("formatted GenerateRequest should indicate an API key is set, got %s", formatted) + } + } + + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("expected GenerateRequest to marshal, got %v", err) + } + if strings.Contains(string(payload), secret) { + t.Fatalf("GenerateRequest JSON leaked API key: %s", payload) + } +} + +func TestEngineExecutionSettingPrecedence(t *testing.T) { + floatPointer := func(value float64) *float64 { + return &value + } + intPointer := func(value int) *int { + return &value + } + + defaultsProfile := executionProfileFixture{ + id: "settings-defaults", + endpoint: "http://profile-defaults.test/v1", + model: "profile-defaults-model", + serviceTier: "profile-defaults-tier", + reasoningEffort: "profile-defaults-reasoning", + apiKeyEnv: "PROMPTKIT_PRECEDENCE_DEFAULTS", + extraParamSource: "profile-defaults", + } + profileSettings := executionProfileFixture{ + id: "settings-profile", + endpoint: "http://profile-settings.test/v1", + model: "profile-settings-model", + temperature: 0.31, + maxTokens: 311, + topP: 0.61, + timeoutSeconds: 71, + serviceTier: "profile-settings-tier", + reasoningEffort: "profile-settings-reasoning", + apiKeyEnv: "PROMPTKIT_PRECEDENCE_PROFILE", + extraParamSource: "profile-settings", + } + requestProfile := executionProfileFixture{ + id: "settings-request", + endpoint: "http://profile-request.test/v1", + model: "profile-request-model", + temperature: 0.29, + maxTokens: 299, + topP: 0.59, + timeoutSeconds: 79, + serviceTier: "profile-request-tier", + reasoningEffort: "profile-request-reasoning", + apiKeyEnv: "PROMPTKIT_PRECEDENCE_REQUEST_PROFILE", + extraParamSource: "profile-request", + } + zeroOverrideProfile := executionProfileFixture{ + id: "settings-zero", + endpoint: "http://profile-zero.test/v1", + model: "profile-zero-model", + temperature: 0.43, + maxTokens: 433, + topP: 0.73, + timeoutSeconds: 83, + serviceTier: "profile-zero-tier", + reasoningEffort: "profile-zero-reasoning", + apiKeyEnv: "PROMPTKIT_PRECEDENCE_ZERO", + extraParamSource: "profile-zero", + } + + requestTarget := promptkit.ExecutionTarget{ + Endpoint: "http://request-settings.test/v1", + Model: "request-settings-model", + Temperature: 0.87, + MaxTokens: 877, + TopP: 0.97, + TimeoutSeconds: 177, + ServiceTier: "request-settings-tier", + ReasoningEffort: "request-settings-reasoning", + APIKeyEnv: "PROMPTKIT_PRECEDENCE_REQUEST", + ExtraParams: map[string]any{"source": "request-settings"}, + } + zeroOverrideTarget := executionTargetFromProfileFixture(zeroOverrideProfile) + zeroOverrideTarget.Temperature = 0 + zeroOverrideTarget.MaxTokens = 0 + zeroOverrideTarget.TopP = 0 + zeroOverrideTarget.TimeoutSeconds = 0 + + tests := []struct { + name string + profile executionProfileFixture + override *promptkit.ExecutionTargetOverride + want promptkit.ExecutionTarget + wantPresence promptkit.ExecutionTargetPresence + }{ + { + name: "framework defaults fill zero-valued profile settings", + profile: defaultsProfile, + want: promptkit.ExecutionTarget{ + Endpoint: defaultsProfile.endpoint, + Model: defaultsProfile.model, + Temperature: 0, + MaxTokens: 0, + TopP: 1, + TimeoutSeconds: 600, + ServiceTier: defaultsProfile.serviceTier, + ReasoningEffort: defaultsProfile.reasoningEffort, + APIKeyEnv: defaultsProfile.apiKeyEnv, + ExtraParams: map[string]any{"source": defaultsProfile.extraParamSource}, + }, + }, + { + name: "profile settings replace framework defaults", + profile: profileSettings, + want: executionTargetFromProfileFixture(profileSettings), + }, + { + name: "request settings replace profile settings", + profile: requestProfile, + override: &promptkit.ExecutionTargetOverride{ + Endpoint: requestTarget.Endpoint, + Model: requestTarget.Model, + Temperature: floatPointer(requestTarget.Temperature), + MaxTokens: intPointer(requestTarget.MaxTokens), + TopP: floatPointer(requestTarget.TopP), + TimeoutSeconds: intPointer(requestTarget.TimeoutSeconds), + ServiceTier: requestTarget.ServiceTier, + ReasoningEffort: requestTarget.ReasoningEffort, + APIKeyEnv: requestTarget.APIKeyEnv, + ExtraParams: requestTarget.ExtraParams, + }, + want: requestTarget, + wantPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}, + }, + { + name: "explicit request zero replaces profile settings", + profile: zeroOverrideProfile, + override: &promptkit.ExecutionTargetOverride{ + Temperature: floatPointer(0), + MaxTokens: intPointer(0), + TopP: floatPointer(0), + TimeoutSeconds: intPointer(0), + }, + want: zeroOverrideTarget, + wantPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(tt.profile.apiKeyEnv, "set") + if tt.override != nil && tt.override.APIKeyEnv != "" { + t.Setenv(tt.override.APIKeyEnv, "set") + } + + profileDir := t.TempDir() + writeExecutionProfileFixture(t, profileDir, tt.profile) + fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: profileDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithLLMClient(fake)) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + + _, err = engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: tt.profile.id, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Nia labels the archive."), + "glossary": promptkit.Inline("archive: A catalogued collection."), + }, + Execution: tt.override, + }) + if err != nil { + t.Fatalf("run engine: %v", err) + } + if len(fake.requests) != 1 { + t.Fatalf("expected one generation request, got %d", len(fake.requests)) + } + got := fake.requests[0] + if !reflect.DeepEqual(got.Target, tt.want) { + t.Fatalf("unexpected effective target:\ngot=%#v\nwant=%#v", got.Target, tt.want) + } + if got.TargetPresence != tt.wantPresence { + t.Fatalf("unexpected target presence: got=%+v want=%+v", got.TargetPresence, tt.wantPresence) + } + }) + } +} + +func TestRunSucceedsWithInjectedLLMClient(t *testing.T) { + const envName = "PROMPTKIT_API_KEY" + const secret = "run-secret-value" + t.Setenv(envName, secret) + + fake := &fakeLLMClient{ + response: &promptkit.GenerateResponse{ + Content: "# Summary\n\nDone.", + Usage: promptkit.TokenUsage{ + PromptTokens: 10, + CompletionTokens: 5, + TotalTokens: 15, + CachedTokens: 3, + CacheWriteTokens: 2, + }, + }, + } + engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) + + result, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + Execution: &promptkit.ExecutionTargetOverride{ + APIKeyEnv: envName, + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if result.RunID == "" { + t.Fatalf("expected run id") + } + if result.RawOutput != fake.response.Content { + t.Fatalf("unexpected raw output: %q", result.RawOutput) + } + if string(result.Artifact.Body) != fake.response.Content { + t.Fatalf("unexpected artifact body: %q", string(result.Artifact.Body)) + } + if result.Artifact.ContentType != "text/markdown" { + t.Fatalf("unexpected artifact content type: %q", result.Artifact.ContentType) + } + if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid { + t.Fatalf("expected passed validation, got %+v", result.Validation) + } + if result.PromptID != frameworkMarkdownSummaryPromptID || result.SelectedProfileID != frameworkFastProfileID || result.ModelName != "contract-fast-model" { + t.Fatalf("unexpected run metadata: %+v", result) + } + if result.Usage.TotalTokens != 15 || result.Usage.CachedTokens != 3 || result.Usage.CacheWriteTokens != 2 { + t.Fatalf("unexpected usage: %+v", result.Usage) + } + + payload, err := json.Marshal(result) + if err != nil { + t.Fatalf("expected run result to marshal, got %v", err) + } + if strings.Contains(string(payload), secret) { + t.Fatalf("run result JSON leaked raw API key value: %s", payload) + } +} + +func TestEngineRunWithDirectorySourcesAndFileInputs(t *testing.T) { + fake := &fakeLLMClient{ + response: &promptkit.GenerateResponse{ + Content: `{"events":[{"title":"Archive labelled"}]}`, + Usage: promptkit.TokenUsage{ + PromptTokens: 42, + CompletionTokens: 36, + TotalTokens: 78, + }, + }, + } + engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) + + result, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkStructuredEventsPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.File(frameworkTranscriptPath), + "glossary": promptkit.File(frameworkGlossaryPath), + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if result.PromptID != frameworkStructuredEventsPromptID || result.SelectedProfileID != frameworkQualityProfileID { + t.Fatalf("unexpected run metadata: %+v", result) + } + if result.RunID == "" || result.PromptHash == "" || result.RenderedPromptHash == "" { + t.Fatalf("expected run and prompt hashes, got %+v", result) + } + if result.InputHashes["transcript"] == "" || result.InputHashes["glossary"] == "" { + t.Fatalf("expected both input hashes, got %#v", result.InputHashes) + } + if len(fake.requests) != 1 || fake.requests[0].StructuredOutput == nil || + fake.requests[0].StructuredOutput.Type != promptkit.StructuredOutputJSONSchema || + fake.requests[0].StructuredOutput.JSONSchema == nil || + fake.requests[0].StructuredOutput.JSONSchema.Schema == nil { + t.Fatalf("expected provider JSON Schema structured output, got %+v", fake.requests) + } + if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid || result.Validation.Mode != promptkit.ValidationJSONSchema { + t.Fatalf("expected passed JSON Schema validation, got %+v", result.Validation) + } + if result.Artifact.ContentType != "application/json" { + t.Fatalf("expected JSON artifact, got %q", result.Artifact.ContentType) + } + if result.RawOutput != fake.response.Content || result.Usage != fake.response.Usage { + t.Fatalf("expected preserved output and usage, got output=%q usage=%+v", result.RawOutput, result.Usage) + } + if result.StartTime.IsZero() || result.EndTime.IsZero() || result.EndTime.Before(result.StartTime) || result.Duration < 0 { + t.Fatalf("expected ordered non-zero timestamps and non-negative duration, got start=%v end=%v duration=%v", result.StartTime, result.EndTime, result.Duration) + } +} + +func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) { + const directKey = "direct-injected-key" + fake := &fakeLLMClient{ + response: &promptkit.GenerateResponse{Content: `{"events":[{"title":"Archive labelled"}]}`}, + } + engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) + + _, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkStructuredEventsPromptID, + APIKey: directKey, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if len(fake.requests) != 1 { + t.Fatalf("expected one generate request, got %d", len(fake.requests)) + } + req := fake.requests[0] + if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") { + t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt) + } + if req.StructuredOutput == nil || req.StructuredOutput.Type != promptkit.StructuredOutputJSONSchema || req.StructuredOutput.JSONSchema == nil { + t.Fatalf("expected structured output handoff, got %+v", req.StructuredOutput) + } + if req.APIKey != directKey { + t.Fatalf("expected direct key on injected generate request") + } + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("expected generate request to marshal, got %v", err) + } + if strings.Contains(string(payload), directKey) { + t.Fatalf("generate request JSON leaked direct API key: %s", payload) + } +} + +func TestEngineRunPropagatesCallerCancellation(t *testing.T) { + const synchronizationTimeout = 5 * time.Second + + started := make(chan struct{}) + transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { + close(started) + <-req.Context().Done() + return nil, req.Context().Err() + }) + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, + HTTPClient: &http.Client{Transport: transport}, + }) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + result := make(chan error, 1) + go func() { + _, err := engine.Run(ctx, promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Nia labels the archive."), + "glossary": promptkit.Inline("archive: A catalogued collection."), + }, + }) + result <- err + }() + + watchdog := time.NewTimer(synchronizationTimeout) + defer watchdog.Stop() + select { + case <-started: + case err := <-result: + t.Fatalf("Engine.Run returned before the transport started: %v", err) + case <-watchdog.C: + t.Fatal("timed out waiting for the transport to start") + } + + cancel() + select { + case err := <-result: + if !errors.Is(err, promptkit.ErrLLMGenerate) { + t.Fatalf("expected ErrLLMGenerate after caller cancellation, got %v", err) + } + case <-watchdog.C: + t.Fatal("timed out waiting for Engine.Run to return after cancellation") + } +} + +func TestRunRejectsReservedExtraParamsBeforeProviderCall(t *testing.T) { + called := false + transport := roundTripFunc(func(*http.Request) (*http.Response, error) { + called = true + return nil, errors.New("provider should not be called") + }) + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, + HTTPClient: &http.Client{Transport: transport}, + }) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + + _, err = engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Nia labels the archive."), + "glossary": promptkit.Inline("archive: A catalogued collection."), + }, + Execution: &promptkit.ExecutionTargetOverride{ + ExtraParams: map[string]any{"model": "collision"}, + }, + }) + if !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("expected ErrInvalidRequest, got %v", err) + } + if called { + t.Fatal("expected reserved provider parameter to fail before the provider call") + } +} + +func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) { + const directKey = "direct-public-key" + const missingEnv = "PROMPTKIT_PUBLIC_DIRECT_MISSING" + t.Setenv(missingEnv, "") + + var gotAuth string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + if r.URL.Path != "/v1/chat/completions" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7} +}`)) + })) + defer server.Close() + + profileDir := t.TempDir() + writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-auth", server.URL+"/v1", "test-model", missingEnv) + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: profileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + result, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "direct-auth", + APIKey: directKey, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected run with direct API key to succeed, got %v", err) + } + if gotAuth != "Bearer "+directKey { + t.Fatalf("unexpected Authorization header: %q", gotAuth) + } + if result.Usage.TotalTokens != 7 { + t.Fatalf("unexpected usage: %+v", result.Usage) + } + + payload, err := json.Marshal(result) + if err != nil { + t.Fatalf("expected run result to marshal, got %v", err) + } + if strings.Contains(string(payload), directKey) { + t.Fatalf("run result JSON leaked direct API key: %s", payload) + } +} + +func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) { + const missingEnv = "PROMPTKIT_PUBLIC_PREPARE_MISSING" + const firstKey = "first-direct-key" + const secondKey = "second-direct-key" + t.Setenv(missingEnv, "") + + profileDir := t.TempDir() + writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-prepare", "http://localhost:8000/v1", "test-model", missingEnv) + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: profileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + baseReq := promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "direct-prepare", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + } + firstReq := baseReq + firstReq.APIKey = firstKey + firstPrepared, err := engine.Prepare(context.Background(), firstReq) + if err != nil { + t.Fatalf("expected prepare with direct API key to succeed, got %v", err) + } + secondReq := baseReq + secondReq.APIKey = secondKey + secondPrepared, err := engine.Prepare(context.Background(), secondReq) + if err != nil { + t.Fatalf("expected prepare with alternate direct API key to succeed, got %v", err) + } + + if firstPrepared.PromptHash != secondPrepared.PromptHash { + t.Fatalf("direct API keys changed prompt hash: %q vs %q", firstPrepared.PromptHash, secondPrepared.PromptHash) + } + if firstPrepared.RenderedPromptHash != secondPrepared.RenderedPromptHash { + t.Fatalf("direct API keys changed rendered prompt hash: %q vs %q", firstPrepared.RenderedPromptHash, secondPrepared.RenderedPromptHash) + } + + payload, err := json.Marshal(firstPrepared) + if err != nil { + t.Fatalf("expected prepared run to marshal, got %v", err) + } + if strings.Contains(string(payload), firstKey) { + t.Fatalf("prepared run JSON leaked direct API key: %s", payload) + } +} + +func TestMissingCredentialsFailClearlyWhenProfileRequiresAuth(t *testing.T) { + const missingEnv = "PROMPTKIT_PUBLIC_AUTH_MISSING" + t.Setenv(missingEnv, "") + + profileDir := t.TempDir() + writePublicProfileFileWithAPIKeyEnv(t, profileDir, "requires-auth", "http://localhost:8000/v1", "test-model", missingEnv) + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: profileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "requires-auth", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("expected invalid request for missing credentials, got %v", err) + } + if !errors.Is(err, promptkit.ErrAPIKeyEnvMissing) { + t.Fatalf("expected missing credential environment error, got %v", err) + } + if err == nil || !strings.Contains(err.Error(), missingEnv) { + t.Fatalf("expected missing env name in error, got %v", err) + } +} + +func TestWithArtifactReaderRejectsNilReader(t *testing.T) { + _, err := promptkit.NewEngine(contractConfig(frameworkSchemaDir), promptkit.WithArtifactReader(nil)) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) + } +} + +func TestArtifactReaderReceivesPublicReferenceAndPreparesArtifact(t *testing.T) { + reader := &recordingArtifactReader{ + artifact: &promptkit.Artifact{ + ContentType: "text/plain", + Body: []byte("Reader-supplied transcript."), + URI: "reader://transcript", + Size: int64(len("Reader-supplied transcript.")), + Hash: "reader-transcript-hash", + }, + } + engine := newArtifactReaderEngine(t, reader) + + ref := promptkit.ArtifactRef{ + Type: promptkit.ArtifactRefInline, + URI: "reader://transcript", + Body: "request body", + } + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: "artifact-reader", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": ref, + }, + }) + if err != nil { + t.Fatalf("prepare with artifact reader: %v", err) + } + if len(reader.refs) != 1 || !reflect.DeepEqual(reader.refs[0], ref) { + t.Fatalf("reader received %#v, want %#v", reader.refs, ref) + } + if prepared.InputHashes["transcript"] != "reader-transcript-hash" { + t.Fatalf("unexpected input hash: %#v", prepared.InputHashes) + } + if len(prepared.Messages) != 1 || !strings.Contains(prepared.Messages[0].Content, "Reader-supplied transcript.") { + t.Fatalf("prepared prompt omitted reader artifact: %#v", prepared.Messages) + } +} + +func TestArtifactReaderFailuresPreserveArtifactLoadErrors(t *testing.T) { + readerErr := errors.New("artifact reader failed") + + tests := []struct { + name string + ctx context.Context + reader *recordingArtifactReader + wantNested error + }{ + { + name: "reader error", + ctx: context.Background(), + reader: &recordingArtifactReader{err: readerErr}, + wantNested: readerErr, + }, + { + name: "nil artifact", + ctx: context.Background(), + reader: &recordingArtifactReader{}, + }, + { + name: "reader cancellation", + ctx: context.Background(), + reader: &recordingArtifactReader{read: func(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { + return nil, context.Canceled + }}, + wantNested: context.Canceled, + }, + { + name: "public reader error", + ctx: context.Background(), + reader: &recordingArtifactReader{err: promptkit.ErrInvalidRequest}, + wantNested: promptkit.ErrInvalidRequest, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + engine := newArtifactReaderEngine(t, tc.reader) + _, err := engine.Prepare(tc.ctx, promptkit.RunRequest{ + PromptID: "artifact-reader", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("input"), + }, + }) + if !errors.Is(err, promptkit.ErrArtifactLoad) { + t.Fatalf("expected ErrArtifactLoad, got %v", err) + } + if tc.wantNested != nil && !errors.Is(err, tc.wantNested) { + t.Fatalf("expected nested %v, got %v", tc.wantNested, err) + } + }) + } +} + +func TestRunAddsLLMGenerateToCollaboratorPublicError(t *testing.T) { + engine := newContractEngineWithOptions(t, frameworkSchemaDir, + promptkit.WithLLMClient(&fakeLLMClient{err: promptkit.ErrArtifactLoad}), + ) + + _, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if !errors.Is(err, promptkit.ErrLLMGenerate) { + t.Fatalf("expected ErrLLMGenerate, got %v", err) + } + if !errors.Is(err, promptkit.ErrArtifactLoad) { + t.Fatalf("expected preserved ErrArtifactLoad, got %v", err) + } +} + +func TestPrepareWithoutProfileMatchesSpecificPublicError(t *testing.T) { + promptDir := t.TempDir() + writePublicPromptFile(t, promptDir, "profile-required", "") + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: promptDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: "profile-required", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("input"), + }, + }) + if !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("expected ErrInvalidRequest, got %v", err) + } + if !errors.Is(err, promptkit.ErrProfileRequired) { + t.Fatalf("expected ErrProfileRequired, got %v", err) + } +} + +func TestRunValidationFailureReturnsResult(t *testing.T) { + fake := &fakeLLMClient{ + response: &promptkit.GenerateResponse{Content: ""}, + } + engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) + + result, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected validation failure as successful result, got %v", err) + } + if result.Validation.Status != promptkit.ValidationFailed || result.Validation.IsValid { + t.Fatalf("expected failed validation result, got %+v", result.Validation) + } + if len(result.Validation.Errors) == 0 { + t.Fatalf("expected validation errors") + } +} + +func TestPublicErrorsSupportErrorsIs(t *testing.T) { + llmErr := errors.New("llm failed") + + tests := []struct { + name string + req promptkit.RunRequest + client promptkit.LLMClient + schemaDir string + want error + }{ + { + name: "invalid request", + req: promptkit.RunRequest{}, + client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}, + want: promptkit.ErrInvalidRequest, + }, + { + name: "prompt not found", + req: promptkit.RunRequest{PromptID: "missing.prompt"}, + client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}, + want: promptkit.ErrPromptNotFound, + }, + { + name: "profile not found", + req: promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "missing-profile", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + }, + client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}, + want: promptkit.ErrProfileNotFound, + }, + { + name: "artifact load", + req: promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.File(filepath.Join(t.TempDir(), "does-not-exist.md")), + }, + }, + client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}, + want: promptkit.ErrArtifactLoad, + }, + { + name: "prompt render", + req: promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + }, + client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}, + want: promptkit.ErrPromptRender, + }, + { + name: "llm failure", + req: promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }, + client: &fakeLLMClient{err: llmErr}, + want: promptkit.ErrLLMGenerate, + }, + { + name: "nil llm response", + req: promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }, + client: &fakeLLMClient{}, + want: promptkit.ErrLLMGenerate, + }, + { + name: "validation runtime failure", + req: promptkit.RunRequest{ + PromptID: frameworkStructuredEventsPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + }, + client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"events":[]}`}}, + schemaDir: t.TempDir(), + want: promptkit.ErrValidation, + }, + } + + t.Setenv("PROMPTKIT_API_KEY", "test-secret") + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + schemaDir := tc.schemaDir + if schemaDir == "" { + schemaDir = frameworkSchemaDir + } + engine := newContractEngineWithOptions(t, schemaDir, promptkit.WithLLMClient(tc.client)) + _, err := engine.Run(context.Background(), tc.req) + if !errors.Is(err, tc.want) { + t.Fatalf("expected errors.Is(%v), got %v", tc.want, err) + } + }) + } +} + +func TestSelectedProfileRawAPIKeyMapsToProfileLoad(t *testing.T) { + profileDir := t.TempDir() + if err := os.WriteFile(filepath.Join(profileDir, "raw.yaml"), []byte(` +id: raw-profile +endpoint: http://localhost:8000/v1 +model: model +api_key: secret +`), 0644); err != nil { + t.Fatal(err) + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: profileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "raw-profile", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if !errors.Is(err, promptkit.ErrProfileLoad) { + t.Fatalf("expected ErrProfileLoad, got %v", err) + } + if errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("did not expect ErrPromptLoad, got %v", err) + } +} + +func TestSelectedProfileInvalidYAMLMapsToProfileLoad(t *testing.T) { + profileDir := t.TempDir() + if err := os.WriteFile(filepath.Join(profileDir, "broken.yaml"), []byte(` +id: broken-profile +unknown_field: true +`), 0644); err != nil { + t.Fatal(err) + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: profileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "broken-profile", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if !errors.Is(err, promptkit.ErrProfileLoad) { + t.Fatalf("expected ErrProfileLoad, got %v", err) + } + if errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("did not expect ErrPromptLoad, got %v", err) + } +} + +func TestPromptRepositoryReadFailureMapsToPromptLoad(t *testing.T) { + missingPromptDir := filepath.Join(t.TempDir(), "missing-prompts") + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: missingPromptDir, + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if !errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("expected ErrPromptLoad, got %v", err) + } + if errors.Is(err, promptkit.ErrProfileLoad) { + t.Fatalf("did not expect ErrProfileLoad, got %v", err) + } +} + +func TestSelectedProfileRepositoryReadFailureMapsToProfileLoad(t *testing.T) { + missingProfileDir := filepath.Join(t.TempDir(), "missing-profiles") + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: missingProfileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: frameworkFastProfileID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if !errors.Is(err, promptkit.ErrProfileLoad) { + t.Fatalf("expected ErrProfileLoad, got %v", err) + } + if errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("did not expect ErrPromptLoad, got %v", err) + } +} + +func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "test-key") + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "mistral-small-3", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected built-in profile prepare to succeed, got %v", err) + } + if prepared.SelectedProfileID != "mistral-small-3" { + t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) + } + if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" { + t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model) + } +} + +func TestPromptDefaultProfileCanUseBuiltInProfile(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "test-key") + promptDir := t.TempDir() + writePublicPromptFile(t, promptDir, "prompt.builtin.default", "mistral-small-3") + + engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: promptDir}) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: "prompt.builtin.default", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + }) + if err != nil { + t.Fatalf("expected built-in default profile prepare to succeed, got %v", err) + } + if prepared.SelectedProfileID != "mistral-small-3" { + t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) + } +} + +func TestCustomProfileOverridesBuiltInProfile(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "test-key") + profileDir := t.TempDir() + writePublicProfileFile(t, profileDir, "mistral-small-3", "http://localhost:8000/v1", "custom-model") + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: profileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "mistral-small-3", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected custom profile prepare to succeed, got %v", err) + } + if prepared.EffectiveModelParams.Model != "custom-model" { + t.Fatalf("expected custom profile to override built-in, got %q", prepared.EffectiveModelParams.Model) + } +} + +func TestMalformedCustomProfileDoesNotFallbackToBuiltIn(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "test-key") + profileDir := t.TempDir() + if err := os.WriteFile(filepath.Join(profileDir, "mistral-small-3.yml"), []byte(` +id: mistral-small-3 +endpoint: http://localhost:8000/v1 +model: custom-model +unexpected: true +`), 0o644); err != nil { + t.Fatal(err) + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: profileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "mistral-small-3", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if !errors.Is(err, promptkit.ErrProfileLoad) { + t.Fatalf("expected custom profile load error, got %v", err) + } +} + +func TestPrepareWorksWithPromptFSAndRelativeContentFile(t *testing.T) { + promptFS := fstest.MapFS{ + "assets/prompts/fs-summary.yaml": &fstest.MapFile{Data: []byte(` +id: fs.summary +version: "1.0.0" +default_profile: contract-fast +inputs: + - name: transcript + required: true +messages: + - role: user + content_file: ./messages/summary.tmpl +output: + format: text + validation_mode: none + repair_attempts: 0 +`)}, + "assets/prompts/messages/summary.tmpl": &fstest.MapFile{Data: []byte(`Summarize {{input "transcript"}} from prompt fs.`)}, + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: t.TempDir(), + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithPromptFS(promptFS, "assets/prompts")) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: "fs.summary", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + }) + if err != nil { + t.Fatalf("expected prepare to succeed, got %v", err) + } + if len(prepared.Messages) != 1 || !strings.Contains(prepared.Messages[0].Content, "prompt fs") { + t.Fatalf("expected content_file body from prompt fs, got %+v", prepared.Messages) + } +} + +func TestPrepareWithPromptFSRejectsEscapedContentFile(t *testing.T) { + promptFS := fstest.MapFS{ + "assets/prompts/fs-escape.yaml": &fstest.MapFile{Data: []byte(` +id: fs.escape +version: "1.0.0" +default_profile: contract-fast +messages: + - role: user + content_file: ../outside.tmpl +output: + format: text + validation_mode: none + repair_attempts: 0 +`)}, + "assets/outside.tmpl": &fstest.MapFile{Data: []byte(`Outside root.`)}, + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: t.TempDir(), + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithPromptFS(promptFS, "assets/prompts")) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "fs.escape"}) + if !errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("expected ErrPromptLoad, got %v", err) + } +} + +func TestPrepareWorksWithPromptFile(t *testing.T) { + promptDir := t.TempDir() + promptPath := filepath.Join(promptDir, "single.yaml") + if err := os.WriteFile(promptPath, []byte(` +id: single.file.prompt +version: "1.0.0" +default_profile: contract-fast +inputs: + - name: transcript + required: true +messages: + - role: user + content_file: ./single.tmpl +output: + format: text + validation_mode: none + repair_attempts: 0 +`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(promptDir, "single.tmpl"), []byte(`Summarize {{input "transcript"}} from file.`), 0o644); err != nil { + t.Fatal(err) + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithPromptFile(promptPath)) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: "single.file.prompt", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + }) + if err != nil { + t.Fatalf("expected prepare to succeed, got %v", err) + } + if prepared.PromptID != "single.file.prompt" { + t.Fatalf("unexpected prompt id: %q", prepared.PromptID) + } + if len(prepared.Messages) != 1 || !strings.Contains(prepared.Messages[0].Content, "from file") { + t.Fatalf("expected content_file body from prompt file, got %+v", prepared.Messages) + } +} + +func TestPrepareWorksWithProfileFSOverBuiltIns(t *testing.T) { + profileFS := fstest.MapFS{ + "profiles/mistral-small-3.yaml": &fstest.MapFile{Data: []byte(` +id: mistral-small-3 +endpoint: http://profile-fs/v1 +model: profile-fs-model +`)}, + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithProfileFS(profileFS, "profiles")) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "mistral-small-3", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected prepare to succeed, got %v", err) + } + if prepared.EffectiveModelParams.Model != "profile-fs-model" { + t.Fatalf("expected profile fs to override built-in, got %q", prepared.EffectiveModelParams.Model) + } +} + +func TestPrepareWorksWithProfileFileOverBuiltIns(t *testing.T) { + profileDir := t.TempDir() + profilePath := filepath.Join(profileDir, "mistral-small-3.yaml") + if err := os.WriteFile(profilePath, []byte(` +id: mistral-small-3 +endpoint: http://profile-file/v1 +model: profile-file-model +`), 0o644); err != nil { + t.Fatal(err) + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithProfileFile(profilePath)) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "mistral-small-3", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected prepare to succeed, got %v", err) + } + if prepared.EffectiveModelParams.Model != "profile-file-model" { + t.Fatalf("expected profile file to override built-in, got %q", prepared.EffectiveModelParams.Model) + } +} + +func TestPrepareWorksWithInMemoryProfilesWithoutProfileFiles(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithProfiles(promptkit.Profile{ + ID: "memory-profile", + Endpoint: "http://memory-profile/v1", + Model: "memory-model", + })) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "memory-profile", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected prepare to succeed, got %v", err) + } + if prepared.EffectiveModelParams.Model != "memory-model" { + t.Fatalf("expected in-memory profile model, got %q", prepared.EffectiveModelParams.Model) + } +} + +func TestInMemoryProfilesOverrideBuiltInsAndProfileSources(t *testing.T) { + profileFS := fstest.MapFS{ + "profiles/mistral-small-3.yaml": &fstest.MapFile{Data: []byte(` +id: mistral-small-3 +endpoint: http://profile-fs/v1 +model: profile-fs-model +`)}, + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, + promptkit.WithProfileFS(profileFS, "profiles"), + promptkit.WithProfiles(promptkit.Profile{ + ID: "mistral-small-3", + Endpoint: "http://memory-profile/v1", + Model: "memory-profile-model", + }), + ) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "mistral-small-3", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected prepare to succeed, got %v", err) + } + if prepared.EffectiveModelParams.Model != "memory-profile-model" { + t.Fatalf("expected in-memory profile to have highest precedence, got %q", prepared.EffectiveModelParams.Model) + } +} + +func TestWithProfilesRejectsDuplicateIDs(t *testing.T) { + _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, + promptkit.WithProfiles( + promptkit.Profile{ID: "duplicate", Endpoint: "http://one/v1", Model: "one"}, + promptkit.Profile{ID: "duplicate", Endpoint: "http://two/v1", Model: "two"}, + ), + ) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) + } +} + +func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) { + fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} + prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + ID: "template-profile", + Endpoint: "http://template/v1", + Model: "template-model", + APIKeyRequired: true, + ExtraParams: map[string]any{ + "provider": "template", + }, + }) + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithProfiles(prof), promptkit.WithLLMClient(fake)) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + _, err = engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "template-profile", + APIKey: "template-key", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if len(fake.requests) != 1 { + t.Fatalf("expected one request, got %d", len(fake.requests)) + } + if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" { + t.Fatalf("unexpected generated request: %+v", fake.requests[0]) + } + if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) { + t.Fatalf("unexpected extra params: %#v", fake.requests[0].Target.ExtraParams) + } +} + +func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) { + intPointer := func(value int) *int { + return &value + } + + tests := []struct { + name string + configTimeout time.Duration + suppliedClientTimeout time.Duration + profileTimeoutSeconds int + requestTimeoutSeconds *int + callerTimeout time.Duration + wantRemainingAtRequest time.Duration + }{ + { + name: "positive supplied client cap takes precedence over config", + configTimeout: 2 * time.Second, + suppliedClientTimeout: 6 * time.Second, + wantRemainingAtRequest: 6 * time.Second, + }, + { + name: "zero supplied client timeout inherits config cap", + configTimeout: 5 * time.Second, + wantRemainingAtRequest: 5 * time.Second, + }, + { + name: "profile deadline is shorter than transport cap", + suppliedClientTimeout: 6 * time.Second, + profileTimeoutSeconds: 4, + wantRemainingAtRequest: 4 * time.Second, + }, + { + name: "request deadline is shorter than profile and transport limits", + suppliedClientTimeout: 6 * time.Second, + profileTimeoutSeconds: 4, + requestTimeoutSeconds: intPointer(2), + wantRemainingAtRequest: 2 * time.Second, + }, + { + name: "explicit zero removes generation deadline but retains transport cap", + suppliedClientTimeout: 5 * time.Second, + profileTimeoutSeconds: 2, + requestTimeoutSeconds: intPointer(0), + wantRemainingAtRequest: 5 * time.Second, + }, + { + name: "framework default remains layered with shorter transport cap", + configTimeout: 7 * time.Second, + suppliedClientTimeout: 3 * time.Second, + wantRemainingAtRequest: 3 * time.Second, + }, + { + name: "caller deadline remains layered with other limits", + suppliedClientTimeout: 6 * time.Second, + profileTimeoutSeconds: 4, + callerTimeout: 2 * time.Second, + wantRemainingAtRequest: 2 * time.Second, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var ( + sawDeadline bool + remaining time.Duration + ) + transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { + deadline, ok := req.Context().Deadline() + sawDeadline = ok + if ok { + remaining = time.Until(deadline) + } + return &http.Response{ + StatusCode: http.StatusOK, + Status: "200 OK", + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader( + `{"choices":[{"message":{"content":"ok"}}]}`, + )), + Request: req, + }, nil + }) + httpClient := &http.Client{ + Timeout: tc.suppliedClientTimeout, + Transport: transport, + } + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + Timeout: tc.configTimeout, + HTTPClient: httpClient, + }, promptkit.WithProfiles(promptkit.Profile{ + ID: "layered-timeout", + Endpoint: "http://timeout.test/v1", + Model: "timeout-model", + TimeoutSeconds: tc.profileTimeoutSeconds, + })) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + ctx := context.Background() + cancel := func() {} + if tc.callerTimeout > 0 { + ctx, cancel = context.WithTimeout(ctx, tc.callerTimeout) + } + defer cancel() + + _, err = engine.Run(ctx, promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "layered-timeout", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + Execution: &promptkit.ExecutionTargetOverride{ + TimeoutSeconds: tc.requestTimeoutSeconds, + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if !sawDeadline { + t.Fatal("expected outbound request context to have a deadline") + } + + const deadlineTolerance = 750 * time.Millisecond + if remaining < tc.wantRemainingAtRequest-deadlineTolerance || + remaining > tc.wantRemainingAtRequest+50*time.Millisecond { + t.Fatalf( + "unexpected request deadline: remaining=%v want approximately %v", + remaining, + tc.wantRemainingAtRequest, + ) + } + }) + } +} + +func TestOpenAICompatibleProfileDefersExtraParamsValidation(t *testing.T) { + cyclic := map[string]any{} + cyclic["self"] = cyclic + + prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + ID: "cyclic-template-profile", + Endpoint: "http://cyclic-template/v1", + Model: "cyclic-template-model", + ExtraParams: cyclic, + }) + + _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, + promptkit.WithProfiles(prof), + ) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) + } +} + +func TestOpenAICompatibleProfileNestedExtraParamsRunThroughWithProfiles(t *testing.T) { + fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} + nested := map[string]any{ + "labels": map[string]string{"route": "primary"}, + "ids": []int{1, 2, 3}, + } + extraParams := map[string]any{ + "nested": nested, + } + prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + ID: "nested-template-profile", + Endpoint: "http://nested-template/v1", + Model: "nested-template-model", + ExtraParams: extraParams, + }) + extraParams["added"] = "mutated-after-construction" + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithProfiles(prof), promptkit.WithLLMClient(fake)) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + nested["added"] = "mutated-after-construction" + + _, err = engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "nested-template-profile", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + want := map[string]any{ + "nested": map[string]any{ + "labels": map[string]string{"route": "primary"}, + "ids": []int{1, 2, 3}, + }, + } + if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, want) { + t.Fatalf("unexpected extra params:\ngot=%#v\nwant=%#v", fake.requests[0].Target.ExtraParams, want) + } +} + +func TestInMemoryProfileAPIKeyRequiredBehavior(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithProfiles(promptkit.Profile{ + ID: "requires-key", + Endpoint: "http://requires-key/v1", + Model: "requires-key-model", + APIKeyRequired: true, + })) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + req := promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "requires-key", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + } + _, err = engine.Prepare(context.Background(), req) + if !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("expected ErrInvalidRequest without API key, got %v", err) + } + req.APIKey = "direct-required-key" + if _, err := engine.Prepare(context.Background(), req); err != nil { + t.Fatalf("expected direct API key to satisfy APIKeyRequired, got %v", err) + } +} + +func TestInMemoryProfileWithoutAPIKeyRequiredWorksWithoutKey(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithProfiles(promptkit.Profile{ + ID: "no-key-required", + Endpoint: "http://no-key/v1", + Model: "no-key-model", + })) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "no-key-required", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected prepare without API key to succeed, got %v", err) + } +} + +func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) { + fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} + labels := map[string]string{"route": "primary"} + ids := []int{1, 2, 3} + extraParams := map[string]any{ + "labels": labels, + "ids": ids, + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, + promptkit.WithProfiles(promptkit.Profile{ + ID: "copy-profile", + Endpoint: "http://copy/v1", + Model: "copy-model", + ExtraParams: extraParams, + }), + promptkit.WithLLMClient(fake), + ) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + labels["route"] = "mutated-before-run" + ids[0] = 99 + extraParams["added"] = "mutated" + + _, err = engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "copy-profile", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + want := map[string]any{ + "labels": map[string]string{"route": "primary"}, + "ids": []int{1, 2, 3}, + } + if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, want) { + t.Fatalf("captured extra params changed after mutation:\ngot=%#v\nwant=%#v", fake.requests[0].Target.ExtraParams, want) + } +} + +func TestWithProfilesRejectsInvalidExtraParams(t *testing.T) { + tests := []struct { + name string + extraParams map[string]any + }{ + {name: "function", extraParams: map[string]any{"bad": func() {}}}, + {name: "channel", extraParams: map[string]any{"bad": make(chan struct{})}}, + {name: "struct", extraParams: map[string]any{"bad": struct{ Name string }{Name: "bad"}}}, + {name: "non string map key", extraParams: map[string]any{"bad": map[int]string{1: "one"}}}, + {name: "nan", extraParams: map[string]any{"bad": math.NaN()}}, + {name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}}, + {name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, + promptkit.WithProfiles(promptkit.Profile{ + ID: "invalid-extra-params", + Endpoint: "http://invalid/v1", + Model: "invalid-model", + ExtraParams: tc.extraParams, + }), + ) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) + } + }) + } +} + +func TestWithProfilesRejectsCyclicExtraParams(t *testing.T) { + cyclicMap := map[string]any{} + cyclicMap["self"] = cyclicMap + cyclicSlice := []any{nil} + cyclicSlice[0] = cyclicSlice + + tests := []struct { + name string + extraParams map[string]any + }{ + {name: "map", extraParams: cyclicMap}, + {name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, + promptkit.WithProfiles(promptkit.Profile{ + ID: "cyclic-extra-params", + Endpoint: "http://cyclic/v1", + Model: "cyclic-model", + ExtraParams: tc.extraParams, + }), + ) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) + } + }) + } +} + +func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) { + fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"events":[]}`}} + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: t.TempDir(), + ProfileDir: frameworkProfileDir, + SchemaDir: t.TempDir(), + }, + promptkit.WithPromptFS(publicStructuredPromptFS("schema.fs.prompt", "events.schema.json"), "prompts"), + promptkit.WithSchemaFS(publicSchemaFS(), "schemas"), + promptkit.WithLLMClient(fake), + ) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + result, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: "schema.fs.prompt", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid { + t.Fatalf("expected schema validation to pass, got %+v", result.Validation) + } + if len(fake.requests) != 1 || fake.requests[0].StructuredOutput == nil { + t.Fatalf("expected structured output request, got %+v", fake.requests) + } +} + +func TestRunStructuredOutputWorksWithSchemaFile(t *testing.T) { + schemaDir := t.TempDir() + schemaPath := filepath.Join(schemaDir, "events.schema.json") + if err := os.WriteFile(schemaPath, []byte(publicSchemaJSON()), 0o644); err != nil { + t.Fatal(err) + } + + fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"events":[]}`}} + engine, err := promptkit.NewEngine(promptkit.Config{ + ProfileDir: frameworkProfileDir, + }, + promptkit.WithPromptFS(publicStructuredPromptFS("schema.file.prompt", "events.schema.json"), "prompts"), + promptkit.WithSchemaFile(schemaPath), + promptkit.WithLLMClient(fake), + ) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + result, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: "schema.file.prompt", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid { + t.Fatalf("expected schema validation to pass, got %+v", result.Validation) + } +} + +func TestSourceOptionsRejectInvalidInputs(t *testing.T) { + missingFile := filepath.Join(t.TempDir(), "missing.yaml") + directoryPath := t.TempDir() + + tests := []struct { + name string + opt promptkit.Option + }{ + {name: "prompt fs nil", opt: promptkit.WithPromptFS(nil, "prompts")}, + {name: "prompt fs empty root", opt: promptkit.WithPromptFS(fstest.MapFS{}, "")}, + {name: "prompt file empty", opt: promptkit.WithPromptFile("")}, + {name: "prompt file missing", opt: promptkit.WithPromptFile(missingFile)}, + {name: "prompt file directory", opt: promptkit.WithPromptFile(directoryPath)}, + {name: "profile fs nil", opt: promptkit.WithProfileFS(nil, "profiles")}, + {name: "profile fs empty root", opt: promptkit.WithProfileFS(fstest.MapFS{}, "")}, + {name: "profile file empty", opt: promptkit.WithProfileFile("")}, + {name: "schema fs nil", opt: promptkit.WithSchemaFS(nil, "schemas")}, + {name: "schema fs empty root", opt: promptkit.WithSchemaFS(fstest.MapFS{}, "")}, + {name: "schema file empty", opt: promptkit.WithSchemaFile("")}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, tc.opt) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) + } + }) + } +} + +func TestPackageOptionsComposeFromSlice(t *testing.T) { + fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} + options := []promptkit.Option{ + nil, + promptkit.WithProfiles(promptkit.Profile{ + ID: "slice-profile", + Endpoint: "http://slice/v1", + Model: "slice-model", + }), + promptkit.WithLLMClient(fake), + } + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, options...) + if err != nil { + t.Fatalf("expected package-provided options to compose, got %v", err) + } + + _, err = engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "slice-profile", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected run with composed options to succeed, got %v", err) + } + if len(fake.requests) != 1 { + t.Fatalf("expected one generate request, got %d", len(fake.requests)) + } + if fake.requests[0].Target.Model != "slice-model" { + t.Fatalf("expected profile from composed options, got %q", fake.requests[0].Target.Model) + } +} + +func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T) { + fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} + engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) + + labels := map[string]string{"route": "primary"} + counts := map[string]int{"retry_budget": 2} + weights := []float64{0.25, 0.75} + ids := []int{1, 2, 3} + nested := map[string]any{ + "labels": labels, + "counts": counts, + "weights": weights, + "ids": ids, + } + extraParams := map[string]any{ + "labels": labels, + "counts": counts, + "nested": nested, + } + + _, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + Execution: &promptkit.ExecutionTargetOverride{ExtraParams: extraParams}, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if len(fake.requests) != 1 { + t.Fatalf("expected one generate request, got %d", len(fake.requests)) + } + + captured := fake.requests[0].Target.ExtraParams + labels["route"] = "mutated" + counts["retry_budget"] = 99 + weights[0] = 9.9 + ids[0] = 99 + nested["added"] = "mutated" + extraParams["new_top_level"] = "mutated" + + want := map[string]any{ + "labels": map[string]string{"route": "primary"}, + "counts": map[string]int{"retry_budget": 2}, + "nested": map[string]any{ + "labels": map[string]string{"route": "primary"}, + "counts": map[string]int{"retry_budget": 2}, + "weights": []float64{0.25, 0.75}, + "ids": []int{1, 2, 3}, + }, + } + if !reflect.DeepEqual(captured, want) { + t.Fatalf("captured extra_params changed after mutating source:\ngot=%#v\nwant=%#v", captured, want) + } +} + +func TestRunRejectsInvalidExtraParams(t *testing.T) { + tests := []struct { + name string + extraParams map[string]any + }{ + {name: "function", extraParams: map[string]any{"bad": func() {}}}, + {name: "channel", extraParams: map[string]any{"bad": make(chan struct{})}}, + {name: "struct", extraParams: map[string]any{"bad": struct{ Name string }{Name: "bad"}}}, + {name: "non string map key", extraParams: map[string]any{"bad": map[int]string{1: "one"}}}, + {name: "nan", extraParams: map[string]any{"bad": math.NaN()}}, + {name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}}, + {name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} + engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) + + _, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + Execution: &promptkit.ExecutionTargetOverride{ExtraParams: tc.extraParams}, + }) + if !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("expected ErrInvalidRequest, got %v", err) + } + if len(fake.requests) != 0 { + t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests)) + } + }) + } +} + +func TestRunRejectsCyclicExtraParams(t *testing.T) { + cyclicMap := map[string]any{} + cyclicMap["self"] = cyclicMap + cyclicSlice := []any{nil} + cyclicSlice[0] = cyclicSlice + + tests := []struct { + name string + extraParams map[string]any + }{ + {name: "map", extraParams: cyclicMap}, + {name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} + engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) + + _, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + Execution: &promptkit.ExecutionTargetOverride{ExtraParams: tc.extraParams}, + }) + if !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("expected ErrInvalidRequest, got %v", err) + } + if len(fake.requests) != 0 { + t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests)) + } + }) + } +} + +func TestWithLLMClientRejectsNilClient(t *testing.T) { + _, err := promptkit.NewEngine(contractConfig(frameworkSchemaDir), promptkit.WithLLMClient(nil)) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) + } +} + +func TestNewEngineConstructsDefaultLLMClientWithoutCredentials(t *testing.T) { + if _, err := promptkit.NewEngine(contractConfig(frameworkSchemaDir)); err != nil { + t.Fatalf("expected default engine construction without credentials to succeed, got %v", err) + } +} + +func newContractEngine(t *testing.T) *promptkit.Engine { + t.Helper() + + for _, path := range []string{ + frameworkPromptDir, + frameworkProfileDir, + frameworkSchemaDir, + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected framework contract path %s to exist: %v", path, err) + } + } + + engine, err := promptkit.NewEngine(contractConfig(frameworkSchemaDir)) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + return engine +} + +func newContractEngineWithOptions(t *testing.T, schemaDir string, opts ...promptkit.Option) *promptkit.Engine { + t.Helper() + + engine, err := promptkit.NewEngine(contractConfig(schemaDir), opts...) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + return engine +} + +func newArtifactReaderEngine(t *testing.T, reader promptkit.ArtifactReader) *promptkit.Engine { + t.Helper() + + promptDir := t.TempDir() + writePublicPromptFile(t, promptDir, "artifact-reader", frameworkFastProfileID) + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: promptDir, + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithArtifactReader(reader)) + if err != nil { + t.Fatalf("construct engine with artifact reader: %v", err) + } + return engine +} + +func contractConfig(schemaDir string) promptkit.Config { + return promptkit.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: frameworkProfileDir, + SchemaDir: schemaDir, + } +} + +func writePublicPromptFile(t *testing.T, dir, id, defaultProfile string) { + t.Helper() + data := `id: ` + id + ` +version: "1.0.0" +default_profile: ` + defaultProfile + ` +inputs: + - name: transcript + required: true +messages: + - role: user + content: "Summarize: {{input \"transcript\"}}" +output: + format: text + validation_mode: none + repair_attempts: 0 +` + if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { + t.Fatalf("failed to write prompt fixture: %v", err) + } +} + +func writePublicProfileFile(t *testing.T, dir, id, endpoint, model string) { + t.Helper() + data := `id: ` + id + ` +endpoint: ` + endpoint + ` +model: ` + model + ` +` + if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { + t.Fatalf("failed to write profile fixture: %v", err) + } +} + +func writePublicProfileFileWithAPIKeyEnv(t *testing.T, dir, id, endpoint, model, apiKeyEnv string) { + t.Helper() + data := `id: ` + id + ` +endpoint: ` + endpoint + ` +model: ` + model + ` +api_key_env: ` + apiKeyEnv + ` +` + if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { + t.Fatalf("failed to write profile fixture: %v", err) + } +} + +func publicStructuredPromptFS(id string, schemaPath string) fstest.MapFS { + return fstest.MapFS{ + "prompts/prompt.yaml": &fstest.MapFile{Data: []byte(`id: ` + id + ` +version: "1.0.0" +default_profile: contract-fast +inputs: + - name: transcript + required: true +messages: + - role: user + content: "Extract events from {{input \"transcript\"}}." +output: + format: json + validation_mode: json_schema + schema_path: ` + schemaPath + ` + repair_attempts: 0 +`)}, + } +} + +func publicSchemaFS() fstest.MapFS { + return fstest.MapFS{ + "schemas/events.schema.json": &fstest.MapFile{Data: []byte(publicSchemaJSON())}, + } +} + +func publicSchemaJSON() string { + return `{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["events"], + "properties": { + "events": {"type": "array"} + } +}` +} + +type executionProfileFixture struct { + id string + endpoint string + model string + temperature float64 + maxTokens int + topP float64 + timeoutSeconds int + serviceTier string + reasoningEffort string + apiKeyEnv string + extraParamSource string +} + +func executionTargetFromProfileFixture(profile executionProfileFixture) promptkit.ExecutionTarget { + return promptkit.ExecutionTarget{ + Endpoint: profile.endpoint, + Model: profile.model, + Temperature: profile.temperature, + MaxTokens: profile.maxTokens, + TopP: profile.topP, + TimeoutSeconds: profile.timeoutSeconds, + ServiceTier: profile.serviceTier, + ReasoningEffort: profile.reasoningEffort, + APIKeyEnv: profile.apiKeyEnv, + ExtraParams: map[string]any{"source": profile.extraParamSource}, + } +} + +func writeExecutionProfileFixture(t *testing.T, dir string, profile executionProfileFixture) { + t.Helper() + data := fmt.Sprintf(`id: %s +endpoint: %s +model: %s +temperature: %g +max_tokens: %d +top_p: %g +timeout_seconds: %d +service_tier: %s +reasoning_effort: %s +api_key_env: %s +extra_params: + source: %q +`, + profile.id, + profile.endpoint, + profile.model, + profile.temperature, + profile.maxTokens, + profile.topP, + profile.timeoutSeconds, + profile.serviceTier, + profile.reasoningEffort, + profile.apiKeyEnv, + profile.extraParamSource, + ) + if err := os.WriteFile(filepath.Join(dir, profile.id+".yaml"), []byte(data), 0o644); err != nil { + t.Fatalf("write execution profile fixture: %v", err) + } +} + +type fakeLLMClient struct { + response *promptkit.GenerateResponse + err error + requests []promptkit.GenerateRequest +} + +type recordingArtifactReader struct { + artifact *promptkit.Artifact + err error + refs []promptkit.ArtifactRef + read func(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) +} + +func (r *recordingArtifactReader) Read(ctx context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) { + r.refs = append(r.refs, ref) + if r.read != nil { + return r.read(ctx, ref) + } + if r.err != nil { + return nil, r.err + } + return r.artifact, nil +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func (f *fakeLLMClient) Generate(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + f.requests = append(f.requests, req) + if f.err != nil { + return nil, f.err + } + return f.response, nil +} diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..763b59c --- /dev/null +++ b/errors.go @@ -0,0 +1,60 @@ +package promptkit + +import ( + "errors" + "fmt" + + "gitea.maximumdirect.net/eric/promptkit/internal/profile" + "gitea.maximumdirect.net/eric/promptkit/internal/promptdef" + "gitea.maximumdirect.net/eric/promptkit/internal/usecase" +) + +func mapPublicError(err error) error { + if err == nil { + return nil + } + publicErr := publicErrorFor(err) + if publicErr == nil { + return err + } + return fmt.Errorf("%w: %w", publicErr, err) +} + +func publicErrorFor(err error) error { + switch { + case errors.Is(err, promptdef.ErrPromptDefinitionNotFound): + return ErrPromptNotFound + case errors.Is(err, profile.ErrProfileNotFound): + return ErrProfileNotFound + case errors.Is(err, usecase.ErrProfileRequired): + return errors.Join(ErrInvalidRequest, ErrProfileRequired) + case errors.Is(err, usecase.ErrPromptLoad): + return ErrPromptLoad + case errors.Is(err, usecase.ErrProfileLoad): + return ErrProfileLoad + case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition): + return ErrPromptLoad + case isProfileLoadCause(err): + return ErrProfileLoad + case errors.Is(err, usecase.ErrAPIKeyEnvMissing): + return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing) + case errors.Is(err, usecase.ErrArtifactLoad): + return ErrArtifactLoad + case errors.Is(err, usecase.ErrPromptRender): + return ErrPromptRender + case errors.Is(err, usecase.ErrLLMGenerate): + return ErrLLMGenerate + case errors.Is(err, usecase.ErrValidation): + return ErrValidation + case errors.Is(err, usecase.ErrInvalidRequest): + return ErrInvalidRequest + default: + return nil + } +} + +func isProfileLoadCause(err error) bool { + return errors.Is(err, profile.ErrInvalidYAML) || + errors.Is(err, profile.ErrInvalidProfile) || + errors.Is(err, profile.ErrRawAPIKeyNotAllowed) +} diff --git a/formatting.go b/formatting.go new file mode 100644 index 0000000..4d55a25 --- /dev/null +++ b/formatting.go @@ -0,0 +1,51 @@ +package promptkit + +import "fmt" + +// String returns a concise request summary without exposing direct API keys. +func (r RunRequest) String() string { + return r.redactedString() +} + +// GoString returns a concise request summary without exposing direct API keys. +func (r RunRequest) GoString() string { + return r.redactedString() +} + +func (r RunRequest) redactedString() string { + return fmt.Sprintf( + "promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t Metadata:%d}", + r.PromptID, + r.PromptVersion, + r.ProfileID, + r.APIKey != "", + len(r.Inputs), + len(r.Vars), + r.Execution != nil, + r.Validation != nil, + len(r.Metadata), + ) +} + +// String returns a concise request summary without exposing direct API keys or +// rendered prompt content. +func (r GenerateRequest) String() string { + return r.redactedString() +} + +// GoString returns a concise request summary without exposing direct API keys or +// rendered prompt content. +func (r GenerateRequest) GoString() string { + return r.redactedString() +} + +func (r GenerateRequest) redactedString() string { + return fmt.Sprintf( + "promptkit.GenerateRequest{Messages:%d Model:%q APIKeySet:%t StructuredOutputSet:%t ExtraParams:%d}", + len(r.Prompt.Messages), + r.Target.Model, + r.APIKey != "", + r.StructuredOutput != nil, + len(r.Target.ExtraParams), + ) +} diff --git a/internal/domain/prepared_run_test.go b/internal/domain/prepared_run_test.go index cf84c1d..da6d231 100644 --- a/internal/domain/prepared_run_test.go +++ b/internal/domain/prepared_run_test.go @@ -7,7 +7,7 @@ import ( ) func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) { - const envName = "SCRIPTORIUM_TEST_API_KEY" + const envName = "PROMPTKIT_TEST_API_KEY" const secret = "super-secret-value" t.Setenv(envName, secret) diff --git a/internal/llm/openai_compatible_client_test.go b/internal/llm/openai_compatible_client_test.go index d363837..54ce938 100644 --- a/internal/llm/openai_compatible_client_test.go +++ b/internal/llm/openai_compatible_client_test.go @@ -176,7 +176,7 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) { if err != nil { t.Fatalf("unexpected constructor error: %v", err) } - t.Setenv("SCRIPTORIUM_TEST_API_KEY", "secret-key") + t.Setenv("PROMPTKIT_TEST_API_KEY", "secret-key") resp, err := client.Generate(context.Background(), domain.GenerateRequest{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{ @@ -189,7 +189,7 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) { MaxTokens: 123, TopP: 0.7, ServiceTier: "priority", - APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY", + APIKeyEnv: "PROMPTKIT_TEST_API_KEY", }, StructuredOutput: &domain.StructuredOutputSpec{ Type: domain.StructuredOutputJSONSchema, @@ -276,7 +276,7 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) { func TestOpenAICompatibleClientDirectAPIKeyPreferredOverEnv(t *testing.T) { const directKey = "direct-llm-key" - t.Setenv("SCRIPTORIUM_TEST_API_KEY", "env-key") + t.Setenv("PROMPTKIT_TEST_API_KEY", "env-key") var gotAuth string ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -294,7 +294,7 @@ func TestOpenAICompatibleClientDirectAPIKeyPreferredOverEnv(t *testing.T) { Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Target: domain.ExecutionTarget{ Model: "model", - APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY", + APIKeyEnv: "PROMPTKIT_TEST_API_KEY", APIKey: directKey, }, }) @@ -870,7 +870,7 @@ func TestOpenAICompatibleClientAPIKeyEnvMissing(t *testing.T) { _, err = client.Generate(context.Background(), domain.GenerateRequest{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ExecutionTarget{APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"}, + Target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_MISSING_KEY"}, }) if err == nil { t.Fatal("expected missing API key env error") diff --git a/internal/profile/repository_test.go b/internal/profile/repository_test.go index 7de08a3..bb36151 100644 --- a/internal/profile/repository_test.go +++ b/internal/profile/repository_test.go @@ -57,7 +57,7 @@ func TestFilesystemRepository_GetProfile(t *testing.T) { if err != nil { t.Fatalf("expected no error, got %v", err) } - if p.APIKeyEnv != "SCRIPTORIUM_API_KEY" { + if p.APIKeyEnv != "PROMPTKIT_API_KEY" { t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv) } if p.ReasoningEffort != "medium" { diff --git a/internal/profile/testdata/valid_with_api_key_env.yaml b/internal/profile/testdata/valid_with_api_key_env.yaml index 8059ca0..3bfb7d8 100644 --- a/internal/profile/testdata/valid_with_api_key_env.yaml +++ b/internal/profile/testdata/valid_with_api_key_env.yaml @@ -1,7 +1,7 @@ id: local-secure endpoint: http://localhost:8000/v1 model: gpt-4o-mini -api_key_env: SCRIPTORIUM_API_KEY +api_key_env: PROMPTKIT_API_KEY service_tier: priority reasoning_effort: medium extra_params: diff --git a/json_copy.go b/json_copy.go new file mode 100644 index 0000000..35e9b83 --- /dev/null +++ b/json_copy.go @@ -0,0 +1,218 @@ +package promptkit + +import ( + "encoding/json" + "fmt" + "math" + "reflect" + "strconv" +) + +const maxSafeJSONInteger = 1<<53 - 1 + +type jsonVisit struct { + typ reflect.Type + ptr uintptr +} + +func copyPublicJSONMap(src map[string]any) (map[string]any, error) { + if src == nil { + return nil, nil + } + copied, err := copyPublicJSONValue(reflect.ValueOf(src), "extra_params", make(map[jsonVisit]struct{})) + if err != nil { + return nil, err + } + out, ok := copied.(map[string]any) + if !ok { + return nil, fmt.Errorf("extra_params: expected object") + } + return out, nil +} + +func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) { + if !value.IsValid() { + return nil, nil + } + if value.Kind() == reflect.Interface { + if value.IsNil() { + return nil, nil + } + return copyPublicJSONValue(value.Elem(), path, seen) + } + if !value.CanInterface() { + return nil, fmt.Errorf("%s: value cannot be copied", path) + } + if number, ok := value.Interface().(json.Number); ok { + f, err := strconv.ParseFloat(number.String(), 64) + if err != nil || math.IsNaN(f) || math.IsInf(f, 0) { + return nil, fmt.Errorf("%s: invalid JSON number", path) + } + return number, nil + } + + switch value.Kind() { + case reflect.Bool, reflect.String: + return value.Interface(), nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if value.Int() < -maxSafeJSONInteger || value.Int() > maxSafeJSONInteger { + return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path) + } + return value.Interface(), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + if value.Uint() > maxSafeJSONInteger { + return nil, fmt.Errorf("%s: integer is outside the JSON-safe range", path) + } + return value.Interface(), nil + case reflect.Float32, reflect.Float64: + f := value.Convert(reflect.TypeOf(float64(0))).Float() + if math.IsNaN(f) || math.IsInf(f, 0) { + return nil, fmt.Errorf("%s: floating-point value must be finite", path) + } + return value.Interface(), nil + case reflect.Pointer: + if value.IsNil() { + return nil, nil + } + visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()} + if _, ok := seen[visit]; ok { + return nil, fmt.Errorf("%s: cyclic value is not supported", path) + } + seen[visit] = struct{}{} + defer delete(seen, visit) + return copyPublicJSONValue(value.Elem(), path, seen) + case reflect.Map: + return copyPublicJSONMapValue(value, path, seen) + case reflect.Slice: + if value.IsNil() { + return nil, nil + } + return copyPublicJSONSequenceValue(value, path, seen) + case reflect.Array: + return copyPublicJSONSequenceValue(value, path, seen) + default: + return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type()) + } +} + +func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) { + if value.IsNil() { + return nil, nil + } + if value.Type().Key().Kind() != reflect.String { + return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key()) + } + + visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()} + if _, ok := seen[visit]; ok { + return nil, fmt.Errorf("%s: cyclic value is not supported", path) + } + seen[visit] = struct{}{} + defer delete(seen, visit) + + type entry struct { + key reflect.Value + name string + value any + } + entries := make([]entry, 0, value.Len()) + preserveType := true + elemType := value.Type().Elem() + iter := value.MapRange() + for iter.Next() { + key := iter.Key() + name := key.String() + copied, err := copyPublicJSONValue(iter.Value(), path+"."+name, seen) + if err != nil { + return nil, err + } + entries = append(entries, entry{key: key, name: name, value: copied}) + if copied == nil { + if !canAssignNil(elemType) { + preserveType = false + } + continue + } + if !reflect.TypeOf(copied).AssignableTo(elemType) { + preserveType = false + } + } + + if preserveType { + out := reflect.MakeMapWithSize(value.Type(), len(entries)) + for _, entry := range entries { + if entry.value == nil { + out.SetMapIndex(entry.key, reflect.Zero(elemType)) + continue + } + out.SetMapIndex(entry.key, reflect.ValueOf(entry.value)) + } + return out.Interface(), nil + } + + out := make(map[string]any, len(entries)) + for _, entry := range entries { + out[entry.name] = entry.value + } + return out, nil +} + +func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) { + var visit jsonVisit + if value.Kind() == reflect.Slice { + visit = jsonVisit{typ: value.Type(), ptr: value.Pointer()} + if _, ok := seen[visit]; ok { + return nil, fmt.Errorf("%s: cyclic value is not supported", path) + } + seen[visit] = struct{}{} + defer delete(seen, visit) + } + + values := make([]any, value.Len()) + preserveType := true + elemType := value.Type().Elem() + for i := 0; i < value.Len(); i++ { + copied, err := copyPublicJSONValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen) + if err != nil { + return nil, err + } + values[i] = copied + if copied == nil { + if !canAssignNil(elemType) { + preserveType = false + } + continue + } + if !reflect.TypeOf(copied).AssignableTo(elemType) { + preserveType = false + } + } + + if preserveType { + out := reflect.New(value.Type()).Elem() + if value.Kind() == reflect.Slice { + out = reflect.MakeSlice(value.Type(), value.Len(), value.Len()) + } + for i, copied := range values { + if copied == nil { + out.Index(i).Set(reflect.Zero(elemType)) + continue + } + out.Index(i).Set(reflect.ValueOf(copied)) + } + return out.Interface(), nil + } + + out := make([]any, len(values)) + copy(out, values) + return out, nil +} + +func canAssignNil(typ reflect.Type) bool { + switch typ.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return true + default: + return false + } +} diff --git a/llm_adapter.go b/llm_adapter.go new file mode 100644 index 0000000..9efbbcb --- /dev/null +++ b/llm_adapter.go @@ -0,0 +1,23 @@ +package promptkit + +import ( + "context" + "fmt" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +type publicLLMClientAdapter struct { + client LLMClient +} + +func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { + resp, err := a.client.Generate(ctx, fromDomainGenerateRequest(req)) + if err != nil { + return nil, err + } + if resp == nil { + return nil, fmt.Errorf("%w: llm client returned nil response", ErrLLMGenerate) + } + return toDomainGenerateResponse(resp), nil +} diff --git a/profiles.go b/profiles.go new file mode 100644 index 0000000..44f5158 --- /dev/null +++ b/profiles.go @@ -0,0 +1,124 @@ +package promptkit + +import ( + "context" + "errors" + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" + "gitea.maximumdirect.net/eric/promptkit/internal/profile" +) + +// OpenAICompatibleProfile returns an ordinary in-memory Profile for an +// OpenAI-compatible chat-completions endpoint. +// +// It does not register global state, maintain a model catalog, or resolve +// credentials. If APIKeyRequired is true, callers satisfy it with +// RunRequest.APIKey. Raw API keys do not belong in profiles. +func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile { + return Profile{ + ID: cfg.ID, + Endpoint: cfg.Endpoint, + Model: cfg.Model, + Temperature: cfg.Temperature, + MaxTokens: cfg.MaxTokens, + TopP: cfg.TopP, + TimeoutSeconds: cfg.TimeoutSeconds, + ServiceTier: cfg.ServiceTier, + ReasoningEffort: cfg.ReasoningEffort, + APIKeyRequired: cfg.APIKeyRequired, + ExtraParams: copyShallowAnyMap(cfg.ExtraParams), + } +} + +func copyShallowAnyMap(src map[string]any) map[string]any { + if src == nil { + return nil + } + out := make(map[string]any, len(src)) + for k, v := range src { + out[k] = v + } + return out +} + +type memoryProfileRepository struct { + profiles map[string]domain.ExecutionProfile +} + +func newMemoryProfileRepository(profiles []Profile) (*memoryProfileRepository, error) { + repo := &memoryProfileRepository{profiles: make(map[string]domain.ExecutionProfile, len(profiles))} + for _, publicProfile := range profiles { + prof, err := toDomainProfile(publicProfile) + if err != nil { + return nil, err + } + if _, exists := repo.profiles[prof.ID]; exists { + return nil, fmt.Errorf("duplicate profile id %q", prof.ID) + } + repo.profiles[prof.ID] = prof + } + return repo, nil +} + +func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) { + if r == nil { + return nil, profile.ErrProfileNotFound + } + prof, ok := r.profiles[id] + if !ok { + return nil, profile.ErrProfileNotFound + } + prof.ExtraParams = copyAnyMap(prof.ExtraParams) + return &prof, nil +} + +func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) { + extraParams, err := copyPublicJSONMap(publicProfile.ExtraParams) + if err != nil { + return domain.ExecutionProfile{}, err + } + prof := domain.ExecutionProfile{ + ID: strings.TrimSpace(publicProfile.ID), + Endpoint: publicProfile.Endpoint, + Model: publicProfile.Model, + Temperature: publicProfile.Temperature, + MaxTokens: publicProfile.MaxTokens, + TopP: publicProfile.TopP, + TimeoutSeconds: publicProfile.TimeoutSeconds, + ServiceTier: publicProfile.ServiceTier, + ReasoningEffort: publicProfile.ReasoningEffort, + APIKeyRequired: publicProfile.APIKeyRequired, + ExtraParams: extraParams, + } + if err := validatePublicProfile(prof); err != nil { + return domain.ExecutionProfile{}, err + } + return prof, nil +} + +func validatePublicProfile(prof domain.ExecutionProfile) error { + if strings.TrimSpace(prof.ID) == "" { + return errors.New("id is required") + } + if strings.TrimSpace(prof.Endpoint) == "" { + return errors.New("endpoint is required") + } + if strings.TrimSpace(prof.Model) == "" { + return errors.New("model is required") + } + if prof.Temperature < 0 || prof.Temperature > 2 { + return errors.New("temperature must be between 0 and 2") + } + if prof.MaxTokens < 0 { + return errors.New("max_tokens must be greater than or equal to 0") + } + if prof.TopP < 0 || prof.TopP > 1 { + return errors.New("top_p must be between 0 and 1") + } + if prof.TimeoutSeconds < 0 { + return errors.New("timeout_seconds must be greater than or equal to 0") + } + return nil +} diff --git a/testdata/framework/fixtures/glossary.yml b/testdata/framework/fixtures/glossary.yml new file mode 100644 index 0000000..503c677 --- /dev/null +++ b/testdata/framework/fixtures/glossary.yml @@ -0,0 +1,2 @@ +archive: A catalogued collection of written records. +marker: A small label used to classify an entry. diff --git a/testdata/framework/fixtures/transcript.md b/testdata/framework/fixtures/transcript.md new file mode 100644 index 0000000..b74a773 --- /dev/null +++ b/testdata/framework/fixtures/transcript.md @@ -0,0 +1,2 @@ +Nia labels the archive. +The archive receives a blue marker. diff --git a/testdata/framework/profiles/contract-fast.yaml b/testdata/framework/profiles/contract-fast.yaml new file mode 100644 index 0000000..5ac326d --- /dev/null +++ b/testdata/framework/profiles/contract-fast.yaml @@ -0,0 +1,7 @@ +id: contract-fast +endpoint: http://localhost:8000/v1 +model: contract-fast-model +temperature: 0.2 +max_tokens: 500 +top_p: 1 +timeout_seconds: 90 diff --git a/testdata/framework/profiles/contract-quality.yaml b/testdata/framework/profiles/contract-quality.yaml new file mode 100644 index 0000000..4477c74 --- /dev/null +++ b/testdata/framework/profiles/contract-quality.yaml @@ -0,0 +1,7 @@ +id: contract-quality +endpoint: http://localhost:8000/v1 +model: contract-quality-model +temperature: 0.1 +max_tokens: 1000 +top_p: 0.9 +timeout_seconds: 120 diff --git a/testdata/framework/prompts/contract.markdown_summary.system.md b/testdata/framework/prompts/contract.markdown_summary.system.md new file mode 100644 index 0000000..3e649a2 --- /dev/null +++ b/testdata/framework/prompts/contract.markdown_summary.system.md @@ -0,0 +1 @@ +You summarize synthetic archive notes in clear Markdown. diff --git a/testdata/framework/prompts/contract.markdown_summary.user.md b/testdata/framework/prompts/contract.markdown_summary.user.md new file mode 100644 index 0000000..aa47620 --- /dev/null +++ b/testdata/framework/prompts/contract.markdown_summary.user.md @@ -0,0 +1,7 @@ +Summarize this transcript: + +{{input "transcript"}} + +Optional glossary: + +{{input "glossary"}} diff --git a/testdata/framework/prompts/contract.markdown_summary.yaml b/testdata/framework/prompts/contract.markdown_summary.yaml new file mode 100644 index 0000000..3242ff5 --- /dev/null +++ b/testdata/framework/prompts/contract.markdown_summary.yaml @@ -0,0 +1,20 @@ +id: contract.markdown_summary +version: "1.0.0" +default_profile: contract-fast +description: Summarize a synthetic transcript in Markdown. +inputs: + - name: transcript + required: true + content_type: text/markdown + - name: glossary + required: false + content_type: text/yaml +messages: + - role: system + content_file: ./contract.markdown_summary.system.md + - role: user + content_file: ./contract.markdown_summary.user.md +output: + format: markdown + validation_mode: basic + repair_attempts: 0 diff --git a/testdata/framework/prompts/contract.structured_events.system.md b/testdata/framework/prompts/contract.structured_events.system.md new file mode 100644 index 0000000..eba5946 --- /dev/null +++ b/testdata/framework/prompts/contract.structured_events.system.md @@ -0,0 +1 @@ +Return only JSON that satisfies the requested event schema. diff --git a/testdata/framework/prompts/contract.structured_events.user.md b/testdata/framework/prompts/contract.structured_events.user.md new file mode 100644 index 0000000..9bcc0cc --- /dev/null +++ b/testdata/framework/prompts/contract.structured_events.user.md @@ -0,0 +1,7 @@ +Extract events from this transcript: + +{{input "transcript"}} + +Optional glossary: + +{{input "glossary"}} diff --git a/testdata/framework/prompts/contract.structured_events.yaml b/testdata/framework/prompts/contract.structured_events.yaml new file mode 100644 index 0000000..607fad8 --- /dev/null +++ b/testdata/framework/prompts/contract.structured_events.yaml @@ -0,0 +1,21 @@ +id: contract.structured_events +version: "1.0.0" +default_profile: contract-quality +description: Extract synthetic events as structured JSON. +inputs: + - name: transcript + required: true + content_type: text/markdown + - name: glossary + required: false + content_type: text/yaml +messages: + - role: system + content_file: ./contract.structured_events.system.md + - role: user + content_file: ./contract.structured_events.user.md +output: + format: json + validation_mode: json_schema + schema_path: structured_events.schema.json + repair_attempts: 0 diff --git a/testdata/framework/schemas/structured_events.schema.json b/testdata/framework/schemas/structured_events.schema.json new file mode 100644 index 0000000..4791fe6 --- /dev/null +++ b/testdata/framework/schemas/structured_events.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["events"], + "properties": { + "events": { + "type": "array", + "items": { + "type": "object", + "required": ["title"], + "properties": { + "title": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/types.go b/types.go new file mode 100644 index 0000000..3abc3df --- /dev/null +++ b/types.go @@ -0,0 +1,306 @@ +package promptkit + +import ( + "context" + "time" +) + +// ArtifactRefType defines how an artifact is referenced. +type ArtifactRefType string + +const ( + ArtifactRefInline ArtifactRefType = "inline" + ArtifactRefFile ArtifactRefType = "file" +) + +// OutputFormat defines the desired output format. +type OutputFormat string + +const ( + FormatText OutputFormat = "text" + FormatMarkdown OutputFormat = "markdown" + FormatJSON OutputFormat = "json" +) + +// ValidationMode defines the output validation strategy. +type ValidationMode string + +const ( + ValidationNone ValidationMode = "none" + ValidationBasic ValidationMode = "basic" + ValidationJSON ValidationMode = "json" + ValidationJSONSchema ValidationMode = "json_schema" +) + +// ValidationStatus defines the result of a validation check. +type ValidationStatus string + +const ( + ValidationPassed ValidationStatus = "passed" + ValidationFailed ValidationStatus = "failed" + ValidationSkipped ValidationStatus = "skipped" +) + +// CacheControlType defines provider cache behavior for prompt content. +type CacheControlType string + +const ( + CacheControlEphemeral CacheControlType = "ephemeral" +) + +// StructuredOutputType identifies provider-level structured output modes. +type StructuredOutputType string + +const ( + StructuredOutputJSONSchema StructuredOutputType = "json_schema" +) + +// RunRequest represents a request to prepare or run a single prompt. +type RunRequest struct { + PromptID string + PromptVersion string + ProfileID string + APIKey string `json:"-"` + Inputs map[string]ArtifactRef + Vars map[string]string + Execution *ExecutionTargetOverride + Validation *OutputContract + Metadata map[string]string +} + +// PreparedRun contains prepared prompt execution state. It does not include +// resolved API key values, model output, validation results, or internal target +// presence metadata. +type PreparedRun struct { + PromptID string `json:"prompt_id"` + PromptVersion string `json:"prompt_version,omitempty"` + PromptHash string `json:"prompt_hash,omitempty"` + SelectedProfileID string `json:"selected_profile_id"` + EffectiveModelParams ExecutionTarget `json:"effective_model_params"` + OutputContract OutputContract `json:"output_contract"` + StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` + InputHashes map[string]string `json:"input_hashes,omitempty"` + SessionID string `json:"session_id,omitempty"` + RenderedPromptHash string `json:"rendered_prompt_hash"` + Messages []RenderedMessage `json:"messages"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + DurationMS int64 `json:"duration_ms,omitempty"` +} + +// RunResult contains generated output, validation state, and run metadata. +type RunResult struct { + RunID string `json:"run_id"` + Artifact Artifact `json:"artifact"` + RawOutput string `json:"raw_output"` + Validation ValidationResult `json:"validation"` + PromptID string `json:"prompt_id"` + PromptVersion string `json:"prompt_version,omitempty"` + PromptHash string `json:"prompt_hash,omitempty"` + RenderedPromptHash string `json:"rendered_prompt_hash"` + SelectedProfileID string `json:"selected_profile_id"` + ModelName string `json:"model_name"` + Endpoint string `json:"endpoint"` + EffectiveModelParams ExecutionTarget `json:"effective_model_params"` + InputHashes map[string]string `json:"input_hashes,omitempty"` + Usage TokenUsage `json:"usage"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + Duration time.Duration `json:"duration,omitempty"` +} + +// ArtifactRef represents a reference to prompt input content. +type ArtifactRef struct { + Type ArtifactRefType + URI string + Body string +} + +// Artifact represents loaded artifact content. +type Artifact struct { + Name string + ContentType string + Body []byte + URI string + Size int64 + Hash string +} + +// ArtifactReader resolves a prompt input reference into its content. +// +// Readers are responsible for supplying artifact metadata. The engine assigns +// an input-map name only when the returned artifact name is empty. +type ArtifactReader interface { + Read(context.Context, ArtifactRef) (*Artifact, error) +} + +// ExecutionTarget represents effective model runtime settings. +type ExecutionTarget struct { + Endpoint string `json:"endpoint"` + Model string `json:"model"` + Temperature float64 `json:"temperature"` + MaxTokens int `json:"max_tokens"` + TopP float64 `json:"top_p"` + TimeoutSeconds int `json:"timeout_seconds"` + ServiceTier string `json:"service_tier"` + ReasoningEffort string `json:"reasoning_effort"` + APIKeyEnv string `json:"api_key_env"` + ExtraParams map[string]any `json:"extra_params"` +} + +// ExecutionTargetOverride represents per-request runtime setting overrides. +type ExecutionTargetOverride struct { + Endpoint string + Model string + Temperature *float64 + MaxTokens *int + TopP *float64 + TimeoutSeconds *int + ServiceTier string + ReasoningEffort string + APIKeyEnv string + ExtraParams map[string]any +} + +// Profile is an in-memory execution profile for library consumers. +// +// It is equivalent to a loaded profile file after validation. Raw API keys do +// not belong in profiles; use APIKeyRequired to require callers to provide +// RunRequest.APIKey for each request, or use profile YAML api_key_env with file +// and FS profile sources. +type Profile struct { + ID string + Endpoint string + Model string + Temperature float64 + MaxTokens int + TopP float64 + TimeoutSeconds int + ServiceTier string + ReasoningEffort string + APIKeyRequired bool + ExtraParams map[string]any +} + +// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory +// profile. +// +// It contains ordinary profile fields for OpenAI-compatible chat-completions +// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do +// not belong in this config. +type OpenAICompatibleProfileConfig struct { + ID string + Endpoint string + Model string + APIKeyRequired bool + Temperature float64 + MaxTokens int + TopP float64 + TimeoutSeconds int + ServiceTier string + ReasoningEffort string + ExtraParams map[string]any +} + +// ExecutionTargetPresence tracks which numeric runtime settings were explicit +// request overrides. +type ExecutionTargetPresence struct { + Temperature bool + MaxTokens bool + TopP bool + TimeoutSeconds bool +} + +// OutputContract defines output and validation requirements. +type OutputContract struct { + Format OutputFormat `json:"format"` + ValidationMode ValidationMode `json:"validation_mode"` + SchemaPath string `json:"schema_path"` + RepairAttempts int `json:"repair_attempts"` +} + +// ValidationResult represents output validation state. +type ValidationResult struct { + Status ValidationStatus `json:"status"` + Mode ValidationMode `json:"mode"` + Errors []string `json:"errors,omitempty"` + SchemaPath string `json:"schema_path,omitempty"` + RepairAttempts int `json:"repair_attempts"` + IsValid bool `json:"is_valid"` +} + +// TokenUsage tracks token consumption. +type TokenUsage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + CachedTokens int `json:"cached_tokens"` + CacheWriteTokens int `json:"cache_write_tokens"` +} + +// RenderedPrompt is the fully rendered prompt passed to an LLM client. +type RenderedPrompt struct { + SessionID string `json:"session_id,omitempty"` + Messages []RenderedMessage `json:"messages"` +} + +// RenderedMessage is a rendered chat message. +type RenderedMessage struct { + Role string `json:"role"` + Content string `json:"content"` + CacheControl *CacheControl `json:"cache_control,omitempty"` +} + +// CacheControl describes provider cache metadata attached to prompt content. +type CacheControl struct { + Type CacheControlType `json:"type"` + TTL string `json:"ttl,omitempty"` +} + +// StructuredOutputSpec describes provider-level structured output. +type StructuredOutputSpec struct { + Type StructuredOutputType `json:"type"` + JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"` +} + +// StructuredOutputJSONSpec contains JSON Schema output constraints. +type StructuredOutputJSONSpec struct { + Name string `json:"name"` + Strict bool `json:"strict"` + Schema any `json:"schema"` +} + +// LLMClient executes rendered prompts for Engine.Run. +type LLMClient interface { + Generate(context.Context, GenerateRequest) (*GenerateResponse, error) +} + +// GenerateRequest is passed to an injected LLM client. +type GenerateRequest struct { + Prompt RenderedPrompt `json:"prompt"` + Target ExecutionTarget `json:"target"` + TargetPresence ExecutionTargetPresence `json:"target_presence"` + StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` + APIKey string `json:"-"` +} + +// GenerateResponse is returned by an injected LLM client. +type GenerateResponse struct { + Content string `json:"content"` + Usage TokenUsage `json:"usage"` +} + +// File returns a file-backed artifact reference. +func File(path string) ArtifactRef { + return ArtifactRef{Type: ArtifactRefFile, URI: path} +} + +// Inline returns an inline artifact reference. +func Inline(body string) ArtifactRef { + return ArtifactRef{Type: ArtifactRefInline, Body: body} +} + +// InlineWithURI returns an inline artifact reference with URI metadata. +func InlineWithURI(uri string, body string) ArtifactRef { + return ArtifactRef{Type: ArtifactRefInline, URI: uri, Body: body} +}