Add generic semantic reconciliation prompt assets

internal/framework/semanticreconcile/assets.go

@@ -0,0 +1,111 @@
+package semanticreconcile
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
+)
+
+const (
+	PromptID      = "generic.semantic_reconciliation"
+	PromptVersion = "v1"
+	promptRoot    = "assets/prompts"
+)
+
+var promptFiles = []promptfs.ModulePromptFile{
+	{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
+	{Name: "system.md", Path: "prompts/system.md"},
+	{Name: "protocol.md", Path: "prompts/protocol.md"},
+	{Name: "instructions.md", Path: "prompts/instructions.md"},
+	{Name: "candidates.md", Path: "prompts/candidates.md"},
+	{Name: "transcript-windows.md", Path: "prompts/transcript-windows.md"},
+}
+
+// RegisterAssets registers the generic reconciliation prompt and response
+// schema as one production-owned asset set.
+func RegisterAssets(registry *llm.AssetRegistry) error {
+	if registry == nil {
+		return fmt.Errorf("semantic reconciliation asset registry must not be nil")
+	}
+	if err := ensureAssetsAbsent(registry); err != nil {
+		return err
+	}
+
+	assets, err := assetFS()
+	if err != nil {
+		return err
+	}
+	prompts, err := promptfs.ModulePromptFS(PromptID, assets, append([]promptfs.ModulePromptFile(nil), promptFiles...))
+	if err != nil {
+		return fmt.Errorf("prepare semantic reconciliation prompt assets: %w", err)
+	}
+	if err := registry.RegisterPromptFS(prompts, promptRoot); err != nil {
+		return fmt.Errorf("register semantic reconciliation prompt assets: %w", err)
+	}
+	if err := registry.RegisterSchemaFS(assets, "schemas"); err != nil {
+		return fmt.Errorf("register semantic reconciliation schema assets: %w", err)
+	}
+	return nil
+}
+
+// PromptHash returns the deterministic identity of the complete generic prompt.
+func PromptHash() (string, error) {
+	assets, err := assetFS()
+	if err != nil {
+		return "", err
+	}
+	parts := make([]llm.AssetHashPart, 0, len(promptFiles))
+	for _, file := range promptFiles {
+		parts = append(parts, llm.AssetHashPart{FS: assets, Path: file.Path})
+	}
+	return llm.HashAssets(parts)
+}
+
+// SchemaHash returns the deterministic identity of the response schema.
+func SchemaHash() (string, error) {
+	schema, err := LoadResponseSchema()
+	if err != nil {
+		return "", err
+	}
+	return schema.SHA256, nil
+}
+
+// SharedPromptFiles returns fresh descriptors for the mandatory protocol and
+// variable-input presentation assets that domain prompts may reuse.
+func SharedPromptFiles() ([]promptfs.SharedPromptFile, error) {
+	assets, err := assetFS()
+	if err != nil {
+		return nil, err
+	}
+	return []promptfs.SharedPromptFile{
+		{Name: "protocol.md", FS: assets, Path: "prompts/protocol.md"},
+		{Name: "candidates.md", FS: assets, Path: "prompts/candidates.md"},
+		{Name: "transcript-windows.md", FS: assets, Path: "prompts/transcript-windows.md"},
+	}, nil
+}
+
+func ensureAssetsAbsent(registry *llm.AssetRegistry) error {
+	prompts, err := registry.PromptFS()
+	if err != nil {
+		return fmt.Errorf("inspect registered prompt assets: %w", err)
+	}
+	if _, err := fs.Stat(prompts, PromptID+"/prompt.yaml"); err == nil {
+		return fmt.Errorf("semantic reconciliation prompt assets already registered")
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("inspect semantic reconciliation prompt assets: %w", err)
+	}
+
+	schemas, err := registry.SchemaFS()
+	if err != nil {
+		return fmt.Errorf("inspect registered schema assets: %w", err)
+	}
+	if _, err := fs.Stat(schemas, "semantic_reconciliation_llm.v1.json"); err == nil {
+		return fmt.Errorf("semantic reconciliation schema assets already registered")
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("inspect semantic reconciliation schema assets: %w", err)
+	}
+	return nil
+}

internal/framework/semanticreconcile/assets_test.go

@@ -0,0 +1,118 @@
+package semanticreconcile
+
+import (
+	"context"
+	"io/fs"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
+	"gitea.maximumdirect.net/eric/promptkit"
+)
+
+func TestRegisterAssetsPreparesGenericPromptOffline(t *testing.T) {
+	registry := llm.NewAssetRegistry()
+	if err := RegisterAssets(registry); err != nil {
+		t.Fatalf("RegisterAssets() error = %v, want nil", err)
+	}
+	options, err := registry.PromptKitOptions()
+	if err != nil {
+		t.Fatalf("PromptKitOptions() error = %v, want nil", err)
+	}
+	options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
+		ID: "semantic-reconciliation-test", Endpoint: "http://127.0.0.1:1/v1", Model: "offline-test-model",
+	})))
+	engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
+	if err != nil {
+		t.Fatalf("NewEngine() error = %v, want nil", err)
+	}
+	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
+		PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "semantic-reconciliation-test",
+		Inputs: map[string]promptkit.ArtifactRef{
+			"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Mira"},{"candidate_id":2,"label":"Captain Mira"}]}`),
+			"transcript": promptkit.Inline(`{"windows":[{"units":[{"unit_id":7,"text":"Mira arrived."}]}]}`),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Prepare() error = %v, want nil", err)
+	}
+
+	if prepared.PromptID != PromptID || prepared.PromptVersion != PromptVersion {
+		t.Fatalf("prepared prompt identity = %q %q, want %q %q", prepared.PromptID, prepared.PromptVersion, PromptID, PromptVersion)
+	}
+	if prepared.SelectedProfileID != "semantic-reconciliation-test" {
+		t.Fatalf("selected profile = %q, want explicit test profile", prepared.SelectedProfileID)
+	}
+	if contract := prepared.OutputContract; contract.SchemaPath != "semantic_reconciliation_llm.v1.json" || contract.RepairAttempts != 0 {
+		t.Fatalf("output contract = %#v, want generic schema without repair", contract)
+	}
+	if len(prepared.Messages) != 5 || prepared.Messages[0].Role != "system" {
+		t.Fatalf("prepared messages = %#v, want five ordered messages beginning with system", prepared.Messages)
+	}
+	if cache := prepared.Messages[2].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
+		t.Fatalf("semantic policy cache control = %#v, want ephemeral", cache)
+	}
+	for _, index := range []int{0, 1, 3, 4} {
+		if prepared.Messages[index].CacheControl != nil {
+			t.Fatalf("message %d cache control = %#v, want nil", index, prepared.Messages[index].CacheControl)
+		}
+	}
+	protocol := prepared.Messages[1].Content
+	for _, requirement := range []string{"positive integer", "Return IDs only", "do not copy candidate names", "source ranges"} {
+		if !strings.Contains(protocol, requirement) {
+			t.Fatalf("protocol message = %q, want requirement %q", protocol, requirement)
+		}
+	}
+	if !strings.Contains(prepared.Messages[3].Content, `"candidate_id":1`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) {
+		t.Fatalf("candidate message = %q, want only integer candidate material", prepared.Messages[3].Content)
+	}
+	if !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"candidate_id"`) {
+		t.Fatalf("transcript message = %q, want only transcript windows", prepared.Messages[4].Content)
+	}
+}
+
+func TestAssetHashesAreDeterministicAndComplete(t *testing.T) {
+	firstPrompt, err := PromptHash()
+	if err != nil {
+		t.Fatal(err)
+	}
+	secondPrompt, err := PromptHash()
+	if err != nil {
+		t.Fatal(err)
+	}
+	schemaHash, err := SchemaHash()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if firstPrompt == "" || firstPrompt != secondPrompt || schemaHash == "" || firstPrompt == schemaHash {
+		t.Fatalf("asset hashes = prompt %q/%q schema %q, want stable distinct hashes", firstPrompt, secondPrompt, schemaHash)
+	}
+}
+
+func TestSharedPromptFilesExposeOnlyReusableCoreAssets(t *testing.T) {
+	first, err := SharedPromptFiles()
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := SharedPromptFiles()
+	if err != nil {
+		t.Fatal(err)
+	}
+	wantNames := []string{"protocol.md", "candidates.md", "transcript-windows.md"}
+	gotNames := make([]string, len(first))
+	for index, file := range first {
+		gotNames[index] = file.Name
+		if content, err := fs.ReadFile(file.FS, file.Path); err != nil || len(content) == 0 {
+			t.Fatalf("shared file %q = %q, %v; want readable content", file.Name, content, err)
+		}
+	}
+	if !reflect.DeepEqual(gotNames, wantNames) {
+		t.Fatalf("shared files = %#v, want narrow allowlist %#v", gotNames, wantNames)
+	}
+	first[0].Name = "changed.md"
+	if second[0].Name != "protocol.md" {
+		t.Fatalf("SharedPromptFiles() reused mutable descriptors: %#v", second)
+	}
+}

internal/framework/semanticreconcile/schema.go

@@ -16,7 +16,7 @@ const (
 	SchemaAssetPath    = "schemas/semantic_reconciliation_llm.v1.json"
 )
 
-func schemaAssetFS() (fs.FS, error) {
+func assetFS() (fs.FS, error) {
 	assets, err := fs.Sub(rootassets.FS(), "generic/normalize/deduplication")
 	if err != nil {
 		return nil, fmt.Errorf("scope semantic reconciliation assets: %w", err)
@@ -27,7 +27,7 @@ func schemaAssetFS() (fs.FS, error) {
 // LoadResponseSchema returns the private request-local integer proposal
 // contract. It is separate from every durable artifact schema.
 func LoadResponseSchema() (llm.ResponseSchema, error) {
-	assets, err := schemaAssetFS()
+	assets, err := assetFS()
 	if err != nil {
 		return llm.ResponseSchema{}, err
 	}

internal/modules/dnd/shared/assets.go

@@ -11,12 +11,14 @@ import (
 )
 
 // PromptAssetManifest is the ordered set of assets that make up one prompt.
-// Module files are addressed in the owning module filesystem; shared files use
-// the names in sharedPromptPaths and are mounted beneath sharedassets.
+// Module files are addressed in the owning module filesystem. SharedFiles use
+// names in sharedPromptPaths, while ExternalSharedFiles are explicit
+// caller-owned descriptors. Both kinds are mounted beneath sharedassets.
 type PromptAssetManifest struct {
-	ModuleDir   string
-	ModuleFiles []promptfs.ModulePromptFile
-	SharedFiles []string
+	ModuleDir           string
+	ModuleFiles         []promptfs.ModulePromptFile
+	SharedFiles         []string
+	ExternalSharedFiles []promptfs.SharedPromptFile
 }
 
 var sharedPromptPaths = map[string]string{
@@ -40,7 +42,7 @@ func sharedAssetFS() (fs.FS, error) {
 }
 
 func (manifest PromptAssetManifest) PromptFS(moduleFS fs.FS) (fs.FS, error) {
-	sharedFiles, err := resolveSharedPromptFiles(manifest.SharedFiles)
+	sharedFiles, err := manifest.sharedPromptFiles()
 	if err != nil {
 		return nil, err
 	}
@@ -49,10 +51,14 @@ func (manifest PromptAssetManifest) PromptFS(moduleFS fs.FS) (fs.FS, error) {
 }
 
 func (manifest PromptAssetManifest) Hash(moduleFS fs.FS) (string, error) {
-	sharedFiles, err := resolveSharedPromptFiles(manifest.SharedFiles)
+	sharedFiles, err := manifest.sharedPromptFiles()
 	if err != nil {
 		return "", err
 	}
+	moduleFiles := append([]promptfs.ModulePromptFile(nil), manifest.ModuleFiles...)
+	if _, err := promptfs.ModulePromptFS(manifest.ModuleDir, moduleFS, moduleFiles, sharedFiles...); err != nil {
+		return "", err
+	}
 	parts := make([]llm.AssetHashPart, 0, len(manifest.ModuleFiles)+len(sharedFiles))
 	for _, file := range manifest.ModuleFiles {
 		parts = append(parts, llm.AssetHashPart{FS: moduleFS, Path: file.Path})
@@ -63,6 +69,14 @@ func (manifest PromptAssetManifest) Hash(moduleFS fs.FS) (string, error) {
 	return llm.HashAssets(parts)
 }
 
+func (manifest PromptAssetManifest) sharedPromptFiles() ([]promptfs.SharedPromptFile, error) {
+	files, err := resolveSharedPromptFiles(manifest.SharedFiles)
+	if err != nil {
+		return nil, err
+	}
+	return append(files, manifest.ExternalSharedFiles...), nil
+}
+
 func resolveSharedPromptFiles(names []string) ([]promptfs.SharedPromptFile, error) {
 	assets, err := sharedAssetFS()
 	if err != nil {

internal/modules/dnd/shared/assets_test.go

@@ -207,3 +207,106 @@ func TestSharedPromptDescriptorsReturnFreshCopies(t *testing.T) {
 		t.Fatalf("resolveSharedPromptFiles() reused descriptor state: first=%#v second=%#v", first, second)
 	}
 }
+
+func TestPromptAssetManifestMountsExternalSharedFiles(t *testing.T) {
+	external := fstest.MapFS{"core/protocol.md": {Data: []byte("integer protocol")}}
+	manifest := PromptAssetManifest{
+		ModuleDir: "dnd.test",
+		ModuleFiles: []promptfs.ModulePromptFile{
+			{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
+		},
+		ExternalSharedFiles: []promptfs.SharedPromptFile{
+			{Name: "protocol.md", FS: external, Path: "core/protocol.md"},
+		},
+	}
+	fys, err := manifest.PromptFS(fstest.MapFS{
+		"prompts/prompt.yaml": {Data: []byte("id: dnd.test")},
+	})
+	if err != nil {
+		t.Fatalf("PromptFS() error = %v, want nil", err)
+	}
+	content, err := fs.ReadFile(fys, "assets/prompts/dnd.test/sharedassets/protocol.md")
+	if err != nil || string(content) != "integer protocol" {
+		t.Fatalf("mounted external protocol = %q, %v", content, err)
+	}
+}
+
+func TestPromptAssetManifestHashIncludesExternalSharedFiles(t *testing.T) {
+	moduleFS := fstest.MapFS{"prompts/prompt.yaml": {Data: []byte("id: dnd.test")}}
+	external := fstest.MapFS{"core/protocol.md": {Data: []byte("integer protocol")}}
+	manifest := PromptAssetManifest{
+		ModuleDir: "dnd.test",
+		ModuleFiles: []promptfs.ModulePromptFile{
+			{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
+		},
+		ExternalSharedFiles: []promptfs.SharedPromptFile{
+			{Name: "protocol.md", FS: external, Path: "core/protocol.md"},
+		},
+	}
+	got, err := manifest.Hash(moduleFS)
+	if err != nil {
+		t.Fatalf("Hash() error = %v, want nil", err)
+	}
+	want, err := llm.HashAssets([]llm.AssetHashPart{
+		{FS: moduleFS, Path: "prompts/prompt.yaml"},
+		{FS: external, Path: "core/protocol.md"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Fatalf("Hash() = %q, want external-aware hash %q", got, want)
+	}
+}
+
+func TestPromptAssetManifestRejectsInvalidExternalSharedFiles(t *testing.T) {
+	moduleFS := fstest.MapFS{"prompts/prompt.yaml": {Data: []byte("id: dnd.test")}}
+	tests := []struct {
+		name     string
+		external []promptfs.SharedPromptFile
+		shared   []string
+		want     string
+	}{
+		{
+			name: "missing",
+			external: []promptfs.SharedPromptFile{
+				{Name: "protocol.md", FS: fstest.MapFS{}, Path: "core/protocol.md"},
+			},
+			want: "read shared prompt asset core/protocol.md",
+		},
+		{
+			name: "duplicate external destinations",
+			external: []promptfs.SharedPromptFile{
+				{Name: "protocol.md", FS: fstest.MapFS{"a.md": {Data: []byte("a")}}, Path: "a.md"},
+				{Name: " protocol.md ", FS: fstest.MapFS{"b.md": {Data: []byte("b")}}, Path: "b.md"},
+			},
+			want: `duplicate sharedassets prompt destination "assets/prompts/dnd.test/sharedassets/protocol.md"`,
+		},
+		{
+			name:   "duplicate named and external destinations",
+			shared: []string{"common-dnd-system.md"},
+			external: []promptfs.SharedPromptFile{
+				{Name: "common-dnd-system.md", FS: fstest.MapFS{"system.md": {Data: []byte("external")}}, Path: "system.md"},
+			},
+			want: `duplicate sharedassets prompt destination "assets/prompts/dnd.test/sharedassets/common-dnd-system.md"`,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			manifest := PromptAssetManifest{
+				ModuleDir: "dnd.test",
+				ModuleFiles: []promptfs.ModulePromptFile{
+					{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
+				},
+				SharedFiles:         test.shared,
+				ExternalSharedFiles: test.external,
+			}
+			if _, err := manifest.PromptFS(moduleFS); err == nil || !strings.Contains(err.Error(), test.want) {
+				t.Fatalf("PromptFS() error = %v, want %q", err, test.want)
+			}
+			if _, err := manifest.Hash(moduleFS); err == nil || !strings.Contains(err.Error(), test.want) {
+				t.Fatalf("Hash() error = %v, want %q", err, test.want)
+			}
+		})
+	}
+}

internal/modules/generic/register/register.go

@@ -6,6 +6,7 @@ import (
 
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
 	"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
 	jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
 	alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_accept"
@@ -16,14 +17,17 @@ import (
 
 // Register adds all production domain-neutral modules and validators.
 func Register(registries pipeline.Registries, assets *llm.AssetRegistry) error {
-	_ = assets
 	if err := validateRegistries(registries); err != nil {
 		return err
 	}
+	if assets == nil {
+		return fmt.Errorf("generic registrar: asset registry must not be nil")
+	}
 	registrations := []struct {
 		name     string
 		register func() error
 	}{
+		{name: "semantic reconciliation assets", register: func() error { return semanticreconcile.RegisterAssets(assets) }},
 		{name: "generic chunker", register: func() error { return units.Register(registries.Chunkers) }},
 		{name: "always accept validator", register: func() error { return alwaysaccept.Register(registries.Validators) }},
 		{name: "always reject validator", register: func() error { return alwaysreject.Register(registries.Validators) }},

internal/modules/generic/register/register_test.go

@@ -1,15 +1,18 @@
 package register
 
 import (
+	"io/fs"
 	"strings"
 	"testing"
 
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
 )
 
 func TestRegisterAddsGenericFamily(t *testing.T) {
 	registries := completeRegistries()
-	if err := Register(registries, nil); err != nil {
+	assets := llm.NewAssetRegistry()
+	if err := Register(registries, assets); err != nil {
 		t.Fatalf("Register() error = %v, want nil", err)
 	}
 	assertContainsKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"generic"})
@@ -30,6 +33,31 @@ func TestRegisterAddsGenericFamily(t *testing.T) {
 	if output, err := registries.Outputs.Build("json"); err != nil || output.Key() != "json" {
 		t.Fatalf("build json output = %v, %v; want json implementation", output, err)
 	}
+	promptAssets, err := assets.PromptFS()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := fs.ReadFile(promptAssets, "generic.semantic_reconciliation/prompt.yaml"); err != nil {
+		t.Fatalf("registered semantic reconciliation prompt: %v", err)
+	}
+	schemaAssets, err := assets.SchemaFS()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := fs.ReadFile(schemaAssets, "semantic_reconciliation_llm.v1.json"); err != nil {
+		t.Fatalf("registered semantic reconciliation schema: %v", err)
+	}
+}
+
+func TestRegisterRejectsNilAssetRegistryBeforeMutation(t *testing.T) {
+	registries := completeRegistries()
+	err := Register(registries, nil)
+	if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") {
+		t.Fatalf("Register() error = %v, want nil asset registry error", err)
+	}
+	if len(registries.Chunkers.RegisteredKeys()) != 0 {
+		t.Fatalf("chunker keys = %#v, want validation before mutation", registries.Chunkers.RegisteredKeys())
+	}
 }
 
 func TestRegisterRejectsMissingGenericRegistriesBeforeMutation(t *testing.T) {
@@ -48,7 +76,7 @@ func TestRegisterRejectsMissingGenericRegistriesBeforeMutation(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			registries := completeRegistries()
 			test.remove(&registries)
-			err := Register(registries, nil)
+			err := Register(registries, llm.NewAssetRegistry())
 			if err == nil || !strings.Contains(err.Error(), test.wantErr) {
 				t.Fatalf("Register() error = %v, want %q", err, test.wantErr)
 			}
@@ -61,11 +89,12 @@ func TestRegisterRejectsMissingGenericRegistriesBeforeMutation(t *testing.T) {
 
 func TestRegisterReportsDuplicateGenericRegistration(t *testing.T) {
 	registries := completeRegistries()
-	if err := Register(registries, nil); err != nil {
+	assets := llm.NewAssetRegistry()
+	if err := Register(registries, assets); err != nil {
 		t.Fatalf("first Register() error = %v, want nil", err)
 	}
-	err := Register(registries, nil)
-	if err == nil || !strings.Contains(err.Error(), "register generic chunker") || !strings.Contains(err.Error(), "already registered") {
+	err := Register(registries, assets)
+	if err == nil || !strings.Contains(err.Error(), "register semantic reconciliation assets") || !strings.Contains(err.Error(), "already registered") {
 		t.Fatalf("second Register() error = %v, want contextual duplicate error", err)
 	}
 }