notarius/internal/framework/llm/scriptorium_api_test.go

package llm

import (
	"context"
	"errors"
	"testing"
	"testing/fstest"
	"time"

	"gitea.maximumdirect.net/eric/scriptorium"
)

func TestScriptoriumPublicAPIGrounding(t *testing.T) {
	// Keep this compile-time grounding close to the future Notarius adapter so
	// dependency upgrades reveal API drift before the runtime cutover.
	engine, err := scriptorium.NewEngine(
		scriptorium.Config{
			PromptDir:  "unused-when-prompt-option-is-set",
			ProfileDir: "",
			SchemaDir:  "",
			Timeout:    time.Second,
		},
		scriptorium.WithPromptFS(fstest.MapFS{}, "."),
		scriptorium.WithProfileFS(fstest.MapFS{}, "."),
		scriptorium.WithSchemaFS(fstest.MapFS{}, "."),
		scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
			ID:             "test-profile",
			Endpoint:       "http://127.0.0.1:1/v1",
			Model:          "test-model",
			APIKeyRequired: true,
			ExtraParams:    map[string]any{"mode": "test"},
		})),
		scriptorium.WithLLMClient(scriptoriumGroundingLLMClient{}),
	)
	if err != nil {
		t.Fatalf("NewEngine() error = %v, want nil", err)
	}
	if engine == nil {
		t.Fatalf("NewEngine() = nil, want engine")
	}

	var (
		_ func(string) scriptorium.Option = scriptorium.WithPromptFile
		_ func(string) scriptorium.Option = scriptorium.WithProfileFile
		_ func(string) scriptorium.Option = scriptorium.WithSchemaFile
	)

	req := scriptorium.RunRequest{
		PromptID:      "dnd.spells",
		PromptVersion: "v1",
		ProfileID:     "test-profile",
		APIKey:        "request-scoped-secret",
		Inputs: map[string]scriptorium.ArtifactRef{
			"transcript": scriptorium.InlineWithURI("file:///tmp/transcript.json", `{"segments":[]}`),
			"glossary":   scriptorium.Inline(""),
			"roster":     scriptorium.File("/tmp/roster.txt"),
		},
		Vars: map[string]string{
			"session_id": "session-1",
		},
		Execution: &scriptorium.ExecutionTargetOverride{
			Model:           "override-model",
			Temperature:     ptr(0.2),
			MaxTokens:       ptr(100),
			TopP:            ptr(0.9),
			TimeoutSeconds:  ptr(30),
			ServiceTier:     "standard",
			ReasoningEffort: "low",
			APIKeyEnv:       "SCRIPTORIUM_API_KEY",
			ExtraParams:     map[string]any{"provider_option": "value"},
		},
		Validation: &scriptorium.OutputContract{
			Format:         scriptorium.FormatJSON,
			ValidationMode: scriptorium.ValidationJSONSchema,
			SchemaPath:     "schemas/dnd_spells.v1.json",
			RepairAttempts: 1,
		},
		Metadata: map[string]string{
			"artifact_kind": "dnd_spell",
		},
	}
	if req.Inputs["transcript"].Type != scriptorium.ArtifactRefInline {
		t.Fatalf("inline input type = %q, want %q", req.Inputs["transcript"].Type, scriptorium.ArtifactRefInline)
	}
	if req.Inputs["roster"].Type != scriptorium.ArtifactRefFile {
		t.Fatalf("file input type = %q, want %q", req.Inputs["roster"].Type, scriptorium.ArtifactRefFile)
	}

	result := scriptorium.RunResult{
		RunID: "run-1",
		Artifact: scriptorium.Artifact{
			Name:        "output",
			ContentType: "application/json",
			Body:        []byte(`{"ok":true}`),
			URI:         "inline://output",
			Size:        int64(len(`{"ok":true}`)),
			Hash:        "sha256:abc",
		},
		RawOutput:          `{"ok":true}`,
		PromptID:           req.PromptID,
		PromptVersion:      req.PromptVersion,
		PromptHash:         "prompt-hash",
		RenderedPromptHash: "rendered-prompt-hash",
		SelectedProfileID:  req.ProfileID,
		ModelName:          "test-model",
		Endpoint:           "http://127.0.0.1:1/v1",
		EffectiveModelParams: scriptorium.ExecutionTarget{
			Model:           "test-model",
			APIKeyEnv:       "SCRIPTORIUM_API_KEY",
			ExtraParams:     map[string]any{"provider_option": "value"},
			ReasoningEffort: "low",
		},
		InputHashes: map[string]string{
			"transcript": "sha256:def",
		},
		Validation: scriptorium.ValidationResult{
			Status:         scriptorium.ValidationPassed,
			Mode:           scriptorium.ValidationJSONSchema,
			SchemaPath:     req.Validation.SchemaPath,
			RepairAttempts: 1,
			IsValid:        true,
		},
		Usage: scriptorium.TokenUsage{
			PromptTokens:     10,
			CompletionTokens: 5,
			TotalTokens:      15,
			CachedTokens:     3,
			CacheWriteTokens: 2,
		},
		StartTime: time.Unix(1, 0),
		EndTime:   time.Unix(2, 0),
		Duration:  time.Second,
	}
	if result.Validation.Status != scriptorium.ValidationPassed {
		t.Fatalf("validation status = %q, want %q", result.Validation.Status, scriptorium.ValidationPassed)
	}
	if result.Usage.TotalTokens != 15 {
		t.Fatalf("total tokens = %d, want 15", result.Usage.TotalTokens)
	}

	publicErrors := []error{
		scriptorium.ErrInvalidConfig,
		scriptorium.ErrInvalidRequest,
		scriptorium.ErrPromptNotFound,
		scriptorium.ErrProfileNotFound,
		scriptorium.ErrPromptLoad,
		scriptorium.ErrProfileLoad,
		scriptorium.ErrArtifactLoad,
		scriptorium.ErrPromptRender,
		scriptorium.ErrLLMGenerate,
		scriptorium.ErrValidation,
	}
	for _, publicErr := range publicErrors {
		if !errors.Is(publicErr, publicErr) {
			t.Fatalf("sentinel error does not match itself: %v", publicErr)
		}
	}
}

type scriptoriumGroundingLLMClient struct{}

func (scriptoriumGroundingLLMClient) Generate(context.Context, scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
	return &scriptorium.GenerateResponse{
		Content: `{"ok":true}`,
		Usage: scriptorium.TokenUsage{
			PromptTokens:     1,
			CompletionTokens: 1,
			TotalTokens:      2,
		},
	}, nil
}

func ptr[T any](v T) *T {
	return &v
}