eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finish implementation of the scriptorium migration

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-05 18:26:31 -05:00
parent 31d70a2dd7
commit 7d4c027d09
10 changed files with 159 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/cli/run.go
View File

@@ -489,16 +489,10 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
seen[id] = struct{}{}
}
}
add(resolved.Input)
add(resolved.Chunk)
add(resolved.Output)
for _, lane := range resolved.ArtifactLanes {
add(lane.Extract)
add(lane.Merge)
add(lane.Normalize)
for _, validator := range lane.Validators {
add(validator)
}
}
ids := make([]string, 0, len(seen))
for id := range seen {

57
internal/cli/run_test.go
View File

@@ -707,6 +707,63 @@ pipelines:
}
}
func TestRunConfigValidateIgnoresNonLLMStageScriptoriumProfiles(t *testing.T) {
profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model")
configPath := writeTestConfig(t, `version: 2
scriptorium:
profile_file: `+profilePath+`
pipelines:
example:
input:
module: fake/input
llm_profile: missing-input
output:
module: json
llm_profile: missing-output
artifacts:
events:
extract: fake/extract
merge:
module: appendorder
llm_profile: missing-merge
`)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example"}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q, want success", code, stderr.String())
}
if strings.Contains(stderr.String(), "missing-") {
t.Fatalf("stderr = %q, want non-LLM stage profiles ignored", stderr.String())
}
}
func TestEffectiveLLMProfileIDsUsesLLMCapableStagesOnly(t *testing.T) {
resolved := pipeline.ResolvedPipeline{
Input: pipeline.ModuleBinding{LLMProfile: "input-profile"},
Chunk: pipeline.ModuleBinding{LLMProfile: "chunk-profile"},
Output: pipeline.ModuleBinding{LLMProfile: "output-profile"},
ArtifactLanes: []pipeline.ResolvedArtifactLane{
{
Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"},
Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"},
Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"},
Validators: []pipeline.ModuleBinding{{LLMProfile: "validator-profile"}},
},
},
}
got := effectiveLLMProfileIDs(resolved)
want := []string{"chunk-profile", "extract-profile", "normalize-profile"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want)
}
}
func TestRunPipelineSessionIDFlagRecordsExplicitTrimmedValue(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := writeSeriatimInput(t)

6
internal/core/config/effective_config.go
View File

@@ -65,16 +65,10 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
}
func applyLLMProfileOverride(profile *pipeline.PipelineProfile, profileID string) {
profile.Input.LLMProfile = profileID
profile.Chunk.LLMProfile = profileID
profile.Output.LLMProfile = profileID
for laneID, lane := range profile.Artifacts {
lane.Extract.LLMProfile = profileID
lane.Merge.LLMProfile = profileID
lane.Normalize.LLMProfile = profileID
for i := range lane.Validators {
lane.Validators[i].LLMProfile = profileID
}
profile.Artifacts[laneID] = lane
}
}

32
internal/core/config/effective_config_test.go
View File

@@ -156,6 +156,14 @@ func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
profile.Input.LLMProfile = "input-profile"
profile.Output.LLMProfile = "output-profile"
lane := profile.Artifacts["events"]
lane.Merge.LLMProfile = "merge-profile"
lane.Validators[0].LLMProfile = "validator-profile"
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
base, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
if err != nil {
@@ -173,18 +181,30 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
if base.ResolvedPipeline.Digest == effective.ResolvedPipeline.Digest {
t.Fatalf("expected digest to change after LLM profile override")
}
for _, binding := range resolvedBindings(effective.ResolvedPipeline) {
for _, binding := range llmCapableBindings(effective.ResolvedPipeline) {
if binding.LLMProfile != "runtime" {
t.Fatalf("binding profile = %q, want runtime", binding.LLMProfile)
t.Fatalf("LLM-capable binding profile = %q, want runtime", binding.LLMProfile)
}
}
if effective.ResolvedPipeline.Input.LLMProfile != "input-profile" {
t.Fatalf("input profile = %q, want original input-profile", effective.ResolvedPipeline.Input.LLMProfile)
}
if effective.ResolvedPipeline.Output.LLMProfile != "output-profile" {
t.Fatalf("output profile = %q, want original output-profile", effective.ResolvedPipeline.Output.LLMProfile)
}
eventLane := effective.ResolvedPipeline.ArtifactLanes[0]
if eventLane.Merge.LLMProfile != "merge-profile" {
t.Fatalf("merge profile = %q, want original merge-profile", eventLane.Merge.LLMProfile)
}
if len(eventLane.Validators) != 1 || eventLane.Validators[0].LLMProfile != "validator-profile" {
t.Fatalf("validator profiles = %#v, want original validator-profile", eventLane.Validators)
}
}
func resolvedBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
bindings := []pipeline.ModuleBinding{resolved.Input, resolved.Chunk, resolved.Output}
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
bindings := []pipeline.ModuleBinding{resolved.Chunk}
for _, lane := range resolved.ArtifactLanes {
bindings = append(bindings, lane.Extract, lane.Merge, lane.Normalize)
bindings = append(bindings, lane.Validators...)
bindings = append(bindings, lane.Extract, lane.Normalize)
}
return bindings
}

23
internal/framework/contracts/contracts.go
View File

@@ -8,23 +8,14 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
type LLMMessage struct {
Role string `json:"role"`
Content string `json:"content"`
}
type StructuredCompletionRequest struct {
StageName string `json:"stage_name"`
PromptID string `json:"prompt_id,omitempty"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
Inputs LLMInputSet `json:"inputs,omitempty"`
Vars map[string]any `json:"vars,omitempty"`
Messages []LLMMessage `json:"messages"`
Model string `json:"model,omitempty"`
ResponseSchemaName string `json:"response_schema_name,omitempty"`
ResponseSchema json.RawMessage `json:"response_schema,omitempty"`
StageName string `json:"stage_name"`
PromptID string `json:"prompt_id,omitempty"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
Inputs LLMInputSet `json:"inputs,omitempty"`
Vars map[string]any `json:"vars,omitempty"`
}
type StructuredCompletionResponse struct {

10
internal/framework/pipeline/walking_skeleton_test.go
View File

@@ -269,8 +269,9 @@ func (extractor walkingSkeletonExtractor) Extract(ctx context.Context, req contr
Call int `json:"call"`
}
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: "fake/extract",
ResponseSchemaName: "fake_event",
StageName: "fake/extract",
PromptID: "fake.event",
PromptVersion: "v1",
}, &response); err != nil {
return contracts.ExtractionResult{}, err
}
@@ -349,8 +350,9 @@ func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req c
Call int `json:"call"`
}
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: "fake/normalize",
ResponseSchemaName: "fake_normalize",
StageName: "fake/normalize",
PromptID: "fake.normalize",
PromptVersion: "v1",
}, &response); err != nil {
return contracts.NormalizeResult{}, err
}

17
internal/modules/chunk/dnd/scenes/chunker_test.go
View File

@@ -118,9 +118,6 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
if req.SessionID != "session-123" || req.ProfileID != "profile-scenes" {
t.Fatalf("session/profile = %q/%q, want session-123/profile-scenes", req.SessionID, req.ProfileID)
}
if len(req.Messages) != 0 || req.ResponseSchemaName != "" || len(req.ResponseSchema) != 0 {
t.Fatalf("legacy prompt fields set: messages=%#v schema=%q/%s", req.Messages, req.ResponseSchemaName, req.ResponseSchema)
}
transcript, ok := req.Inputs["transcript"]
if !ok {
t.Fatalf("transcript input missing from %#v", req.Inputs)
@@ -493,8 +490,18 @@ func (client *fakeScenesLLMClient) CompleteStructured(ctx context.Context, req c
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Messages = append([]contracts.LLMMessage(nil), req.Messages...)
req.ResponseSchema = append(json.RawMessage(nil), req.ResponseSchema...)
req.Inputs = req.Inputs.Clone()
req.Vars = cloneVars(req.Vars)
return req
}
func cloneVars(in map[string]any) map[string]any {
if len(in) == 0 {
return nil
}
out := make(map[string]any, len(in))
for key, value := range in {
out[key] = value
}
return out
}

21
internal/modules/extract/dnd/spells/extractor_test.go
View File

@@ -46,9 +46,6 @@ func TestExtractReturnsSpellCandidateFromStructuredOutput(t *testing.T) {
if req.SessionID != "session-123" || req.ProfileID != "profile-spells" {
t.Fatalf("session/profile = %q/%q, want session-123/profile-spells", req.SessionID, req.ProfileID)
}
if len(req.Messages) != 0 || req.ResponseSchemaName != "" || len(req.ResponseSchema) != 0 {
t.Fatalf("legacy prompt fields set: messages=%#v schema=%q/%s", req.Messages, req.ResponseSchemaName, req.ResponseSchema)
}
transcript := req.Inputs["transcript"]
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:transcript" || transcript.OriginURI != "file:///session-alpha.json" {
t.Fatalf("transcript metadata = %#v", transcript)
@@ -135,8 +132,8 @@ func TestExtractPassesReferencesAsPromptInputs(t *testing.T) {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
request := client.requests[0]
if len(request.Messages) != 0 {
t.Fatalf("Messages = %#v, want no locally rendered prompt", request.Messages)
if request.PromptID != PromptID || request.PromptVersion != SchemaVersion {
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, PromptID, SchemaVersion)
}
if got := string(request.Inputs["roster"].Content); got != "Aria Brightmantle: party cleric" {
t.Fatalf("roster input = %q, want reference content", got)
@@ -337,8 +334,18 @@ func (client *fakeSpellsLLMClient) CompleteStructured(ctx context.Context, req c
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Messages = append([]contracts.LLMMessage(nil), req.Messages...)
req.ResponseSchema = append(json.RawMessage(nil), req.ResponseSchema...)
req.Inputs = req.Inputs.Clone()
req.Vars = cloneVars(req.Vars)
return req
}
func cloneVars(in map[string]any) map[string]any {
if len(in) == 0 {
return nil
}
out := make(map[string]any, len(in))
for key, value := range in {
out[key] = value
}
return out
}

8
internal/modules/extract/dnd/spells/runner_test.go
View File

@@ -153,8 +153,8 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
request := llmClient.requests[0]
if len(request.Messages) != 0 {
t.Fatalf("Messages = %#v, want no locally rendered prompt", request.Messages)
if request.PromptID != PromptID || request.PromptVersion != SchemaVersion {
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, PromptID, SchemaVersion)
}
if got := string(request.Inputs["roster"].Content); got != "Aria: party cleric\nBorin: fighter" {
t.Fatalf("roster input = %q, want reference text", got)
@@ -191,8 +191,8 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
request := llmClient.requests[0]
if len(request.Messages) != 0 {
t.Fatalf("Messages = %#v, want no locally rendered prompt", request.Messages)
if request.PromptID != PromptID || request.PromptVersion != SchemaVersion {
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, PromptID, SchemaVersion)
}
if got := string(request.Inputs["roster"].Content); !strings.Contains(got, "Lightning Bolt") {
t.Fatalf("roster input = %q, want roster-only spell in reference input", got)