Architectural improvements in the http adapter
This commit is contained in:
6
.gitignore
vendored
6
.gitignore
vendored
@@ -16,7 +16,7 @@
|
|||||||
*.out
|
*.out
|
||||||
|
|
||||||
# Dependency directories (remove the comment below to include it)
|
# Dependency directories (remove the comment below to include it)
|
||||||
# vendor/
|
vendor/
|
||||||
|
|
||||||
# Go workspace file
|
# Go workspace file
|
||||||
go.work
|
go.work
|
||||||
@@ -25,6 +25,10 @@ go.work.sum
|
|||||||
# env file
|
# env file
|
||||||
.env
|
.env
|
||||||
|
|
||||||
|
# Ignore local test directory and complied binary
|
||||||
|
scriptorium
|
||||||
|
local-test/
|
||||||
|
|
||||||
# ---> VisualStudio
|
# ---> VisualStudio
|
||||||
## Ignore Visual Studio temporary files, build results, and
|
## Ignore Visual Studio temporary files, build results, and
|
||||||
## files generated by popular Visual Studio add-ons.
|
## files generated by popular Visual Studio add-ons.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# scriptorium
|
# scriptorium
|
||||||
|
|
||||||
Scriptorium (Analyzer) is a generic prompt-profile execution engine written in Go.
|
Scriptorium is a generic prompt-profile execution engine written in Go.
|
||||||
|
|
||||||
It loads a prompt profile, resolves named input artifacts, renders a prompt, calls an OpenAI-compatible LLM endpoint, validates output, and returns an artifact plus metadata.
|
It loads a prompt profile, resolves named input artifacts, renders a prompt, calls an OpenAI-compatible LLM endpoint, validates output, and returns an artifact plus metadata.
|
||||||
|
|
||||||
|
|||||||
@@ -1,14 +1,14 @@
|
|||||||
# Analyzer Architecture
|
# Scriptorium Architecture
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
Analyzer is a general-purpose prompt-profile execution service.
|
Scriptorium is a general-purpose prompt-profile execution service.
|
||||||
|
|
||||||
Its job is to take one or more named input artifacts, render a configured prompt profile, execute that prompt against an LLM endpoint, optionally validate the output, and return a generated artifact with useful metadata.
|
Its job is to take one or more named input artifacts, render a configured prompt profile, execute that prompt against an LLM endpoint, optionally validate the output, and return a generated artifact with useful metadata.
|
||||||
|
|
||||||
The initial concrete use case is generating artifacts from cleaned Dungeons & Dragons session transcripts, such as session recaps, player analysis, structured event extraction, and glossary update suggestions.
|
The initial concrete use case is generating artifacts from cleaned Dungeons & Dragons session transcripts, such as session recaps, player analysis, structured event extraction, and glossary update suggestions.
|
||||||
|
|
||||||
However, Analyzer must not be D&D-specific. D&D behavior belongs in prompt profiles, schemas, and caller-provided inputs. The Go application should remain a generic engine for prompt execution and output validation.
|
However, Scriptorium must not be D&D-specific. D&D behavior belongs in prompt profiles, schemas, and caller-provided inputs. The Go application should remain a generic engine for prompt execution and output validation.
|
||||||
|
|
||||||
## Intended Audience
|
## Intended Audience
|
||||||
|
|
||||||
@@ -28,7 +28,7 @@ The desired implementation style is:
|
|||||||
|
|
||||||
## Core Concept
|
## Core Concept
|
||||||
|
|
||||||
Analyzer transforms:
|
Scriptorium transforms:
|
||||||
|
|
||||||
- Prompt profile
|
- Prompt profile
|
||||||
- Named input artifacts
|
- Named input artifacts
|
||||||
@@ -44,19 +44,19 @@ Into:
|
|||||||
- Raw model output
|
- Raw model output
|
||||||
- Structured error details, if applicable
|
- Structured error details, if applicable
|
||||||
|
|
||||||
Analyzer should be thought of as a deterministic wrapper around a nondeterministic model call.
|
Scriptorium should be thought of as a deterministic wrapper around a nondeterministic model call.
|
||||||
|
|
||||||
The system should make the model call as auditable and reproducible as possible, even though LLM output itself may not be exactly reproducible.
|
The system should make the model call as auditable and reproducible as possible, even though LLM output itself may not be exactly reproducible.
|
||||||
|
|
||||||
## Application Boundary
|
## Application Boundary
|
||||||
|
|
||||||
Analyzer is not an orchestrator.
|
Scriptorium is not an orchestrator.
|
||||||
|
|
||||||
The broader workflow may include audio transcription, transcript merging, transcript polishing, artifact persistence, and notifications. Those responsibilities belong to the external orchestrator, currently expected to be Narratio.
|
The broader workflow may include audio transcription, transcript merging, transcript polishing, artifact persistence, and notifications. Those responsibilities belong to the external orchestrator, currently expected to be Narratio.
|
||||||
|
|
||||||
Analyzer should not know about WhisperX, Seriatim, Audita, or any other pipeline stage.
|
Scriptorium should not know about WhisperX, Seriatim, Audita, or any other pipeline stage.
|
||||||
|
|
||||||
Analyzer only knows how to:
|
Scriptorium only knows how to:
|
||||||
|
|
||||||
1. Load a prompt profile.
|
1. Load a prompt profile.
|
||||||
2. Load or receive named input artifacts.
|
2. Load or receive named input artifacts.
|
||||||
@@ -75,19 +75,19 @@ The initial D&D workflow is expected to look like this:
|
|||||||
4. Narratio saves the merged transcript.
|
4. Narratio saves the merged transcript.
|
||||||
5. Narratio calls Audita to polish the transcript.
|
5. Narratio calls Audita to polish the transcript.
|
||||||
6. Narratio saves the processed transcript.
|
6. Narratio saves the processed transcript.
|
||||||
7. Narratio calls Analyzer one or more times to generate output artifacts.
|
7. Narratio calls Scriptorium one or more times to generate output artifacts.
|
||||||
8. Narratio saves each generated artifact.
|
8. Narratio saves each generated artifact.
|
||||||
9. Narratio optionally sends a completion notification.
|
9. Narratio optionally sends a completion notification.
|
||||||
|
|
||||||
Analyzer only owns step 7.
|
Scriptorium only owns step 7.
|
||||||
|
|
||||||
Each Analyzer request should initially produce one artifact. If multiple artifacts are needed, the orchestrator should call Analyzer multiple times.
|
Each Scriptorium request should initially produce one artifact. If multiple artifacts are needed, the orchestrator should call Scriptorium multiple times.
|
||||||
|
|
||||||
Batch execution can be added later, but should not be part of the core v1 design unless there is an immediate need.
|
Batch execution can be added later, but should not be part of the core v1 design unless there is an immediate need.
|
||||||
|
|
||||||
## Primary Use Cases
|
## Primary Use Cases
|
||||||
|
|
||||||
Analyzer should support the following v1 use cases:
|
Scriptorium should support the following v1 use cases:
|
||||||
|
|
||||||
1. Generate a freeform Markdown artifact from a transcript and prompt profile.
|
1. Generate a freeform Markdown artifact from a transcript and prompt profile.
|
||||||
2. Generate a structured JSON artifact from a transcript and prompt profile.
|
2. Generate a structured JSON artifact from a transcript and prompt profile.
|
||||||
@@ -129,7 +129,7 @@ The central use case should be easy to test with fake prompt repositories, fake
|
|||||||
|
|
||||||
Recommended high-level structure:
|
Recommended high-level structure:
|
||||||
|
|
||||||
- cmd/analyzer: application entrypoint
|
- cmd/scriptorium: application entrypoint
|
||||||
- internal/domain: core domain types
|
- internal/domain: core domain types
|
||||||
- internal/usecase: application use cases
|
- internal/usecase: application use cases
|
||||||
- internal/profile: prompt profile loading and parsing
|
- internal/profile: prompt profile loading and parsing
|
||||||
@@ -425,7 +425,7 @@ If token counting is not implemented initially, use byte-size limits or leave to
|
|||||||
|
|
||||||
## Output Formats
|
## Output Formats
|
||||||
|
|
||||||
Analyzer should support at least these output formats:
|
Scriptorium should support at least these output formats:
|
||||||
|
|
||||||
- markdown
|
- markdown
|
||||||
- text
|
- text
|
||||||
@@ -454,7 +454,7 @@ Validation modes:
|
|||||||
- json: parse as JSON
|
- json: parse as JSON
|
||||||
- json_schema: parse as JSON and validate against schema
|
- json_schema: parse as JSON and validate against schema
|
||||||
|
|
||||||
For invalid structured output, Analyzer should return:
|
For invalid structured output, Scriptorium should return:
|
||||||
|
|
||||||
- Validation status
|
- Validation status
|
||||||
- Validation errors
|
- Validation errors
|
||||||
@@ -540,9 +540,9 @@ It should support local development and pipeline usage.
|
|||||||
|
|
||||||
Suggested commands:
|
Suggested commands:
|
||||||
|
|
||||||
- analyzer run
|
- scriptorium run
|
||||||
- analyzer profiles list
|
- scriptorium profiles list
|
||||||
- analyzer profiles inspect
|
- scriptorium profiles inspect
|
||||||
|
|
||||||
The run command should accept:
|
The run command should accept:
|
||||||
|
|
||||||
@@ -575,7 +575,7 @@ Avoid hardcoding D&D-specific defaults.
|
|||||||
|
|
||||||
## Artifact Storage
|
## Artifact Storage
|
||||||
|
|
||||||
Analyzer does not need to own artifact persistence in v1.
|
Scriptorium does not need to own artifact persistence in v1.
|
||||||
|
|
||||||
The default behavior should be:
|
The default behavior should be:
|
||||||
|
|
||||||
@@ -584,7 +584,7 @@ The default behavior should be:
|
|||||||
|
|
||||||
Narratio or another orchestrator can save the result to S3.
|
Narratio or another orchestrator can save the result to S3.
|
||||||
|
|
||||||
However, Analyzer should be designed so that artifact readers and writers can be added later.
|
However, Scriptorium should be designed so that artifact readers and writers can be added later.
|
||||||
|
|
||||||
If an ArtifactWriter is added, it should be optional and should not change the core use case.
|
If an ArtifactWriter is added, it should be optional and should not change the core use case.
|
||||||
|
|
||||||
@@ -606,7 +606,7 @@ Important error categories:
|
|||||||
|
|
||||||
Validation failure is not necessarily the same as application failure.
|
Validation failure is not necessarily the same as application failure.
|
||||||
|
|
||||||
If the model returns output but the output fails validation, Analyzer should return a structured RunResult with failed validation status when possible.
|
If the model returns output but the output fails validation, Scriptorium should return a structured RunResult with failed validation status when possible.
|
||||||
|
|
||||||
Transport-level errors, missing inputs, invalid profiles, and failed model calls should be returned as application errors.
|
Transport-level errors, missing inputs, invalid profiles, and failed model calls should be returned as application errors.
|
||||||
|
|
||||||
@@ -656,7 +656,7 @@ This metadata is important for auditing and regeneration.
|
|||||||
|
|
||||||
## Security and Safety Considerations
|
## Security and Safety Considerations
|
||||||
|
|
||||||
Analyzer will often handle private transcripts or documents.
|
Scriptorium will often handle private transcripts or documents.
|
||||||
|
|
||||||
Default behavior should avoid accidental disclosure.
|
Default behavior should avoid accidental disclosure.
|
||||||
|
|
||||||
@@ -740,11 +740,11 @@ Do not discard invalid model output.
|
|||||||
|
|
||||||
Do not hide validation errors.
|
Do not hide validation errors.
|
||||||
|
|
||||||
Do not implement an orchestrator inside Analyzer.
|
Do not implement an orchestrator inside Scriptorium.
|
||||||
|
|
||||||
## Summary
|
## Summary
|
||||||
|
|
||||||
Analyzer is a reusable prompt-profile execution engine.
|
Scriptorium is a reusable prompt-profile execution engine.
|
||||||
|
|
||||||
It should provide this core transformation:
|
It should provide this core transformation:
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,6 @@ package httpadapter
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type runRequestDTO struct {
|
type runRequestDTO struct {
|
||||||
@@ -30,10 +28,10 @@ type modelOverrideRequestDTO struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type runResponseDTO struct {
|
type runResponseDTO struct {
|
||||||
Artifact artifactDTO `json:"artifact"`
|
Artifact artifactDTO `json:"artifact"`
|
||||||
Validation domain.ValidationResult `json:"validation"`
|
Validation validationDTO `json:"validation"`
|
||||||
Metadata metadataDTO `json:"metadata"`
|
Metadata metadataDTO `json:"metadata"`
|
||||||
RawModelOutput string `json:"raw_model_output"`
|
RawModelOutput string `json:"raw_model_output"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type artifactDTO struct {
|
type artifactDTO struct {
|
||||||
@@ -52,11 +50,26 @@ type metadataDTO struct {
|
|||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
InputHashes map[string]string `json:"input_hashes"`
|
InputHashes map[string]string `json:"input_hashes"`
|
||||||
PromptHash string `json:"prompt_hash"`
|
PromptHash string `json:"prompt_hash"`
|
||||||
Usage domain.TokenUsage `json:"usage"`
|
Usage tokenUsageDTO `json:"usage"`
|
||||||
StartTime time.Time `json:"start_time"`
|
StartTime time.Time `json:"start_time"`
|
||||||
EndTime time.Time `json:"end_time"`
|
EndTime time.Time `json:"end_time"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type tokenUsageDTO struct {
|
||||||
|
PromptTokens int `json:"prompt_tokens"`
|
||||||
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
|
TotalTokens int `json:"total_tokens"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type validationDTO struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
Errors []string `json:"errors,omitempty"`
|
||||||
|
SchemaPath string `json:"schema_path,omitempty"`
|
||||||
|
RepairAttempts int `json:"repair_attempts"`
|
||||||
|
IsValid bool `json:"is_valid"`
|
||||||
|
}
|
||||||
|
|
||||||
type errorResponse struct {
|
type errorResponse struct {
|
||||||
Error errorBody `json:"error"`
|
Error errorBody `json:"error"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
Size: res.Artifact.Size,
|
Size: res.Artifact.Size,
|
||||||
Hash: res.Artifact.Hash,
|
Hash: res.Artifact.Hash,
|
||||||
},
|
},
|
||||||
Validation: res.Validation,
|
Validation: mapValidation(res.Validation),
|
||||||
Metadata: metadataDTO{
|
Metadata: metadataDTO{
|
||||||
ProfileID: res.ProfileID,
|
ProfileID: res.ProfileID,
|
||||||
ProfileVersion: res.ProfileVersion,
|
ProfileVersion: res.ProfileVersion,
|
||||||
@@ -101,14 +101,29 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
Endpoint: res.Endpoint,
|
Endpoint: res.Endpoint,
|
||||||
InputHashes: res.InputHashes,
|
InputHashes: res.InputHashes,
|
||||||
PromptHash: res.PromptHash,
|
PromptHash: res.PromptHash,
|
||||||
Usage: res.Usage,
|
Usage: tokenUsageDTO{
|
||||||
StartTime: res.StartTime,
|
PromptTokens: res.Usage.PromptTokens,
|
||||||
EndTime: res.EndTime,
|
CompletionTokens: res.Usage.CompletionTokens,
|
||||||
|
TotalTokens: res.Usage.TotalTokens,
|
||||||
|
},
|
||||||
|
StartTime: res.StartTime,
|
||||||
|
EndTime: res.EndTime,
|
||||||
},
|
},
|
||||||
RawModelOutput: res.RawOutput,
|
RawModelOutput: res.RawOutput,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mapValidation(v domain.ValidationResult) validationDTO {
|
||||||
|
return validationDTO{
|
||||||
|
Status: string(v.Status),
|
||||||
|
Mode: string(v.Mode),
|
||||||
|
Errors: v.Errors,
|
||||||
|
SchemaPath: v.SchemaPath,
|
||||||
|
RepairAttempts: v.RepairAttempts,
|
||||||
|
IsValid: v.IsValid,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func mapRunError(err error) (int, string) {
|
func mapRunError(err error) (int, string) {
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, profile.ErrProfileNotFound):
|
case errors.Is(err, profile.ErrProfileNotFound):
|
||||||
|
|||||||
@@ -81,6 +81,15 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
|
|||||||
if artifact["body"] != "hello" {
|
if artifact["body"] != "hello" {
|
||||||
t.Fatalf("expected artifact body hello, got %#v", artifact["body"])
|
t.Fatalf("expected artifact body hello, got %#v", artifact["body"])
|
||||||
}
|
}
|
||||||
|
validation := resp["validation"].(map[string]any)
|
||||||
|
if validation["status"] != "passed" {
|
||||||
|
t.Fatalf("expected validation.status passed, got %#v", validation["status"])
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
usage := metadata["usage"].(map[string]any)
|
||||||
|
if usage["total_tokens"] != float64(3) {
|
||||||
|
t.Fatalf("expected usage.total_tokens=3, got %#v", usage["total_tokens"])
|
||||||
|
}
|
||||||
if resp["raw_model_output"] != "hello" {
|
if resp["raw_model_output"] != "hello" {
|
||||||
t.Fatalf("expected raw model output hello, got %#v", resp["raw_model_output"])
|
t.Fatalf("expected raw model output hello, got %#v", resp["raw_model_output"])
|
||||||
}
|
}
|
||||||
@@ -172,8 +181,8 @@ func TestHandlerValidationFailureStillSuccess(t *testing.T) {
|
|||||||
t.Fatalf("invalid JSON response: %v", err)
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
}
|
}
|
||||||
validation := resp["validation"].(map[string]any)
|
validation := resp["validation"].(map[string]any)
|
||||||
if status, ok := validation["Status"].(string); !ok || status != "failed" {
|
if status, ok := validation["status"].(string); !ok || status != "failed" {
|
||||||
t.Fatalf("expected validation Status=failed, got %#v", validation["Status"])
|
t.Fatalf("expected validation status=failed, got %#v", validation["status"])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ var (
|
|||||||
ErrValidation = errors.New("failed to validate output")
|
ErrValidation = errors.New("failed to validate output")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Runner executes the Analyzer core use case.
|
// Runner executes the Scriptorium core use case.
|
||||||
type Runner struct {
|
type Runner struct {
|
||||||
profiles profile.Repository
|
profiles profile.Repository
|
||||||
artifacts artifact.Reader
|
artifacts artifact.Reader
|
||||||
|
|||||||
Reference in New Issue
Block a user