From e053f7e124f65c4612af554a428d39b347571952 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 23 May 2026 17:48:37 +0000 Subject: [PATCH] Add shared metadata maps and stage-name helpers --- .../framework/proposal_generation/generate.go | 30 +++++++---------- internal/framework/responseschema/registry.go | 24 ++++++++++++++ .../framework/responseschema/registry_test.go | 17 ++++++++++ internal/framework/stagename/stagename.go | 24 ++++++++++++++ .../framework/stagename/stagename_test.go | 33 +++++++++++++++++++ .../framework/validators/llm_validators.go | 26 +++++---------- internal/modules/glossary/module.go | 29 +++++++--------- internal/modules/grammar/module.go | 29 +++++++--------- internal/modules/homophones/module.go | 29 +++++++--------- internal/modules/spoken_word/module.go | 29 +++++++--------- internal/prompts/registry.go | 10 ++++++ internal/prompts/registry_test.go | 13 ++++++++ 12 files changed, 188 insertions(+), 105 deletions(-) create mode 100644 internal/framework/stagename/stagename.go create mode 100644 internal/framework/stagename/stagename_test.go diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index d00554e..bcba659 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -15,6 +15,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" + "gitea.maximumdirect.net/eric/audita/internal/framework/stagename" stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" ) @@ -94,7 +95,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { stage := strings.TrimSpace(req.StageName) if stage == "" { - stage = buildStageName(req.ModuleInstance, req.Section) + stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section)) } model := resolveModel(req.Config, req.Model) messages := append([]contracts.LLMMessage(nil), req.Messages...) @@ -143,7 +144,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { if len(req.PromptMetadata) > 0 { requestMetadata["prompt_metadata"] = req.PromptMetadata } - requestMetadata["response_schema"] = schemaMetadata(responseSchema) + requestMetadata["response_schema"] = responseSchema.DiagnosticsMap() if writer != nil { artifacts, _ = writer.WriteInteraction( @@ -201,23 +202,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { }, nil } -func schemaMetadata(schema responseschema.Schema) map[string]any { - return map[string]any{ - "id": schema.ID, - "version": schema.Version, - "name": schema.Name, - "sha256": schema.SHA256, - } -} - -func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string { - base := fmt.Sprintf("%s:proposal-generation", moduleInstance) - if section == nil { - return base - } - return fmt.Sprintf("%s:section-%04d", base, section.Index) -} - func resolveModel(cfg *config.Config, override string) string { if strings.TrimSpace(override) != "" { return strings.TrimSpace(override) @@ -288,6 +272,14 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string { return artifacts.ResponsePayloadPath } +func sectionIndexPtr(section *contracts.SectionMetadata) *int { + if section == nil { + return nil + } + index := section.Index + return &index +} + type diagnosticsWriterAdapter struct { writer *llm.DiagnosticsWriter } diff --git a/internal/framework/responseschema/registry.go b/internal/framework/responseschema/registry.go index 02f7513..f970570 100644 --- a/internal/framework/responseschema/registry.go +++ b/internal/framework/responseschema/registry.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "sort" "strings" ) @@ -28,6 +29,15 @@ type Schema struct { SHA256 string `json:"sha256"` } +func (s Schema) DiagnosticsMap() map[string]any { + return map[string]any{ + "id": s.ID, + "version": s.Version, + "name": s.Name, + "sha256": s.SHA256, + } +} + var registry = map[Key]Schema{ CorrectionSetKey: mustBuildSchema( correctionSetSchemaID, @@ -43,6 +53,20 @@ var registry = map[Key]Schema{ ), } +func Registered() []Schema { + keys := make([]string, 0, len(registry)) + for key := range registry { + keys = append(keys, string(key)) + } + sort.Strings(keys) + + out := make([]Schema, 0, len(keys)) + for _, key := range keys { + out = append(out, cloneSchema(registry[Key(key)])) + } + return out +} + // Lookup returns a copy of the registered schema for the provided key. func Lookup(key Key) (Schema, bool) { schema, ok := registry[key] diff --git a/internal/framework/responseschema/registry_test.go b/internal/framework/responseschema/registry_test.go index 1e5bc62..9e77dbc 100644 --- a/internal/framework/responseschema/registry_test.go +++ b/internal/framework/responseschema/registry_test.go @@ -89,3 +89,20 @@ func TestLookupReturnsSchemaCopy(t *testing.T) { t.Fatalf("expected lookup to return independent schema copy") } } + +func TestDiagnosticsMapIncludesStableSchemaMetadataShapeForAllSchemas(t *testing.T) { + registered := Registered() + if len(registered) == 0 { + t.Fatalf("expected registered response schemas") + } + + for _, schema := range registered { + metadataMap := schema.DiagnosticsMap() + if metadataMap["id"] != schema.ID || + metadataMap["version"] != schema.Version || + metadataMap["name"] != schema.Name || + metadataMap["sha256"] != schema.SHA256 { + t.Fatalf("unexpected diagnostics metadata map for %q: %+v", schema.ID, metadataMap) + } + } +} diff --git a/internal/framework/stagename/stagename.go b/internal/framework/stagename/stagename.go new file mode 100644 index 0000000..67041f6 --- /dev/null +++ b/internal/framework/stagename/stagename.go @@ -0,0 +1,24 @@ +package stagename + +import ( + "fmt" +) + +func ModuleProposal(moduleInstance string, sectionIndex *int) string { + if sectionIndex == nil || *sectionIndex == 0 { + return fmt.Sprintf("%s:proposal", moduleInstance) + } + return fmt.Sprintf("%s:proposal:section-%04d", moduleInstance, *sectionIndex) +} + +func ProposalGeneration(moduleInstance string, sectionIndex *int) string { + base := fmt.Sprintf("%s:proposal-generation", moduleInstance) + if sectionIndex == nil { + return base + } + return fmt.Sprintf("%s:section-%04d", base, *sectionIndex) +} + +func ValidatorBatch(moduleInstance string, validatorName string, batchIndex int) string { + return fmt.Sprintf("%s:%s:batch-%04d", moduleInstance, validatorName, batchIndex) +} diff --git a/internal/framework/stagename/stagename_test.go b/internal/framework/stagename/stagename_test.go new file mode 100644 index 0000000..bf98fcc --- /dev/null +++ b/internal/framework/stagename/stagename_test.go @@ -0,0 +1,33 @@ +package stagename + +import ( + "testing" +) + +func TestModuleProposalStageName(t *testing.T) { + if got := ModuleProposal("grammar", nil); got != "grammar:proposal" { + t.Fatalf("unexpected stage name without section: %q", got) + } + + sectionIndex := 7 + if got := ModuleProposal("grammar", §ionIndex); got != "grammar:proposal:section-0007" { + t.Fatalf("unexpected stage name with section: %q", got) + } +} + +func TestProposalGenerationStageName(t *testing.T) { + if got := ProposalGeneration("grammar", nil); got != "grammar:proposal-generation" { + t.Fatalf("unexpected proposal generation stage name without section: %q", got) + } + + sectionIndex := 3 + if got := ProposalGeneration("grammar", §ionIndex); got != "grammar:proposal-generation:section-0003" { + t.Fatalf("unexpected proposal generation stage name with section: %q", got) + } +} + +func TestValidatorBatchStageName(t *testing.T) { + if got := ValidatorBatch("homophones_1", "spoken_form_plausibility_review", 12); got != "homophones_1:spoken_form_plausibility_review:batch-0012" { + t.Fatalf("unexpected validator batch stage name: %q", got) + } +} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index 27cd475..eca9343 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -11,6 +11,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" + "gitea.maximumdirect.net/eric/audita/internal/framework/stagename" stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" "gitea.maximumdirect.net/eric/audita/internal/prompts" ) @@ -110,9 +111,10 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, var response LLMValidationResponse responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) + stage := stagename.ValidatorBatch(req.ModuleInstance, v.name, batch.BatchIndex) call := func(callCtx context.Context) error { _, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{ - StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex), + StageName: stage, Messages: messages, Model: resolvedValidationModel(req.Config, v.model), ResponseSchema: &responseSchema, @@ -126,27 +128,15 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, } artifacts := InteractionArtifacts{} if req.DiagnosticsWriter != nil { - stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex) promptMetadata := validatorPromptMetadata(v.validatorType) artifacts, _ = req.DiagnosticsWriter.WriteInteraction( stage, map[string]any{ - "validator_name": v.name, - "validator_type": v.validatorType, - "batch_index": batch.BatchIndex, - "prompt_metadata": map[string]any{ - "prompt_id": promptMetadata.PromptID, - "prompt_version": promptMetadata.PromptVersion, - "prompt_source": promptMetadata.PromptSource, - "embedded_path": promptMetadata.EmbeddedPath, - "sha256": promptMetadata.SHA256, - }, - "response_schema": map[string]any{ - "id": responseSchema.ID, - "version": responseSchema.Version, - "name": responseSchema.Name, - "sha256": responseSchema.SHA256, - }, + "validator_name": v.name, + "validator_type": v.validatorType, + "batch_index": batch.BatchIndex, + "prompt_metadata": promptMetadata.DiagnosticsMap(), + "response_schema": responseSchema.DiagnosticsMap(), }, map[string]any{"messages": messages, "items": batch.Items}, response, diff --git a/internal/modules/glossary/module.go b/internal/modules/glossary/module.go index 4da5c1b..d497c4f 100644 --- a/internal/modules/glossary/module.go +++ b/internal/modules/glossary/module.go @@ -2,12 +2,12 @@ package glossary import ( "context" - "fmt" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/internal/framework/stagename" builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" ) @@ -60,18 +60,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co Glossary: req.Glossary, Config: req.Config, Messages: messages, - PromptMetadata: map[string]any{ - "prompt_id": proposalPromptMetadata().PromptID, - "prompt_version": proposalPromptMetadata().PromptVersion, - "prompt_source": proposalPromptMetadata().PromptSource, - "embedded_path": proposalPromptMetadata().EmbeddedPath, - "sha256": proposalPromptMetadata().SHA256, - }, - StageName: proposalStageName(req), - StartIndex: 0, - LLMClient: req.LLMClient, - Scheduler: req.LLMScheduler, - DiagnosticsDir: req.DiagnosticsDir, + PromptMetadata: proposalPromptMetadata().DiagnosticsMap(), + StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)), + StartIndex: 0, + LLMClient: req.LLMClient, + Scheduler: req.LLMScheduler, + DiagnosticsDir: req.DiagnosticsDir, }) if err != nil { return contracts.ProposalResult{}, err @@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect return &schema.Transcript{Segments: segments} } -func proposalStageName(req contracts.ProposalRequest) string { - if req.Section == nil || req.Section.Index == 0 { - return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName) +func sectionIndexPtr(section *contracts.SectionMetadata) *int { + if section == nil { + return nil } - return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index) + index := section.Index + return &index } diff --git a/internal/modules/grammar/module.go b/internal/modules/grammar/module.go index 8a5d97f..c6b030b 100644 --- a/internal/modules/grammar/module.go +++ b/internal/modules/grammar/module.go @@ -2,12 +2,12 @@ package grammar import ( "context" - "fmt" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/internal/framework/stagename" builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" ) @@ -60,18 +60,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co Glossary: req.Glossary, Config: req.Config, Messages: messages, - PromptMetadata: map[string]any{ - "prompt_id": proposalPromptMetadata().PromptID, - "prompt_version": proposalPromptMetadata().PromptVersion, - "prompt_source": proposalPromptMetadata().PromptSource, - "embedded_path": proposalPromptMetadata().EmbeddedPath, - "sha256": proposalPromptMetadata().SHA256, - }, - StageName: proposalStageName(req), - StartIndex: 0, - LLMClient: req.LLMClient, - Scheduler: req.LLMScheduler, - DiagnosticsDir: req.DiagnosticsDir, + PromptMetadata: proposalPromptMetadata().DiagnosticsMap(), + StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)), + StartIndex: 0, + LLMClient: req.LLMClient, + Scheduler: req.LLMScheduler, + DiagnosticsDir: req.DiagnosticsDir, }) if err != nil { return contracts.ProposalResult{}, err @@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect return &schema.Transcript{Segments: segments} } -func proposalStageName(req contracts.ProposalRequest) string { - if req.Section == nil || req.Section.Index == 0 { - return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName) +func sectionIndexPtr(section *contracts.SectionMetadata) *int { + if section == nil { + return nil } - return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index) + index := section.Index + return &index } diff --git a/internal/modules/homophones/module.go b/internal/modules/homophones/module.go index 4e3a068..d878af9 100644 --- a/internal/modules/homophones/module.go +++ b/internal/modules/homophones/module.go @@ -2,12 +2,12 @@ package homophones import ( "context" - "fmt" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/internal/framework/stagename" builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" ) @@ -60,18 +60,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co Glossary: req.Glossary, Config: req.Config, Messages: messages, - PromptMetadata: map[string]any{ - "prompt_id": proposalPromptMetadata().PromptID, - "prompt_version": proposalPromptMetadata().PromptVersion, - "prompt_source": proposalPromptMetadata().PromptSource, - "embedded_path": proposalPromptMetadata().EmbeddedPath, - "sha256": proposalPromptMetadata().SHA256, - }, - StageName: proposalStageName(req), - StartIndex: 0, - LLMClient: req.LLMClient, - Scheduler: req.LLMScheduler, - DiagnosticsDir: req.DiagnosticsDir, + PromptMetadata: proposalPromptMetadata().DiagnosticsMap(), + StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)), + StartIndex: 0, + LLMClient: req.LLMClient, + Scheduler: req.LLMScheduler, + DiagnosticsDir: req.DiagnosticsDir, }) if err != nil { return contracts.ProposalResult{}, err @@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect return &schema.Transcript{Segments: segments} } -func proposalStageName(req contracts.ProposalRequest) string { - if req.Section == nil || req.Section.Index == 0 { - return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName) +func sectionIndexPtr(section *contracts.SectionMetadata) *int { + if section == nil { + return nil } - return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index) + index := section.Index + return &index } diff --git a/internal/modules/spoken_word/module.go b/internal/modules/spoken_word/module.go index 60261af..e23e413 100644 --- a/internal/modules/spoken_word/module.go +++ b/internal/modules/spoken_word/module.go @@ -2,12 +2,12 @@ package spoken_word import ( "context" - "fmt" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/internal/framework/stagename" builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" ) @@ -60,18 +60,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co Glossary: req.Glossary, Config: req.Config, Messages: messages, - PromptMetadata: map[string]any{ - "prompt_id": proposalPromptMetadata().PromptID, - "prompt_version": proposalPromptMetadata().PromptVersion, - "prompt_source": proposalPromptMetadata().PromptSource, - "embedded_path": proposalPromptMetadata().EmbeddedPath, - "sha256": proposalPromptMetadata().SHA256, - }, - StageName: proposalStageName(req), - StartIndex: 0, - LLMClient: req.LLMClient, - Scheduler: req.LLMScheduler, - DiagnosticsDir: req.DiagnosticsDir, + PromptMetadata: proposalPromptMetadata().DiagnosticsMap(), + StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)), + StartIndex: 0, + LLMClient: req.LLMClient, + Scheduler: req.LLMScheduler, + DiagnosticsDir: req.DiagnosticsDir, }) if err != nil { return contracts.ProposalResult{}, err @@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect return &schema.Transcript{Segments: segments} } -func proposalStageName(req contracts.ProposalRequest) string { - if req.Section == nil || req.Section.Index == 0 { - return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName) +func sectionIndexPtr(section *contracts.SectionMetadata) *int { + if section == nil { + return nil } - return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index) + index := section.Index + return &index } diff --git a/internal/prompts/registry.go b/internal/prompts/registry.go index 668d80d..2808cdf 100644 --- a/internal/prompts/registry.go +++ b/internal/prompts/registry.go @@ -40,6 +40,16 @@ type Metadata struct { SHA256 string `json:"sha256"` } +func (m Metadata) DiagnosticsMap() map[string]any { + return map[string]any{ + "prompt_id": m.PromptID, + "prompt_version": m.PromptVersion, + "prompt_source": m.PromptSource, + "embedded_path": m.EmbeddedPath, + "sha256": m.SHA256, + } +} + type definition struct { id string version string diff --git a/internal/prompts/registry_test.go b/internal/prompts/registry_test.go index 598c0f2..16fbd3d 100644 --- a/internal/prompts/registry_test.go +++ b/internal/prompts/registry_test.go @@ -107,3 +107,16 @@ func TestRenderedPromptsContainHardening(t *testing.T) { } } } + +func TestDiagnosticsMapIncludesStablePromptMetadataShapeForAllPrompts(t *testing.T) { + for _, m := range RegisteredMetadata() { + metadataMap := m.DiagnosticsMap() + if metadataMap["prompt_id"] != m.PromptID || + metadataMap["prompt_version"] != m.PromptVersion || + metadataMap["prompt_source"] != m.PromptSource || + metadataMap["embedded_path"] != m.EmbeddedPath || + metadataMap["sha256"] != m.SHA256 { + t.Fatalf("unexpected diagnostics metadata map for %q: %+v", m.PromptID, metadataMap) + } + } +}