Add shared metadata maps and stage-name helpers

This commit is contained in:
2026-05-23 17:48:37 +00:00
parent 13029dbb33
commit e053f7e124
12 changed files with 188 additions and 105 deletions

View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
) )
@@ -94,7 +95,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
stage := strings.TrimSpace(req.StageName) stage := strings.TrimSpace(req.StageName)
if stage == "" { if stage == "" {
stage = buildStageName(req.ModuleInstance, req.Section) stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section))
} }
model := resolveModel(req.Config, req.Model) model := resolveModel(req.Config, req.Model)
messages := append([]contracts.LLMMessage(nil), req.Messages...) messages := append([]contracts.LLMMessage(nil), req.Messages...)
@@ -143,7 +144,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if len(req.PromptMetadata) > 0 { if len(req.PromptMetadata) > 0 {
requestMetadata["prompt_metadata"] = req.PromptMetadata requestMetadata["prompt_metadata"] = req.PromptMetadata
} }
requestMetadata["response_schema"] = schemaMetadata(responseSchema) requestMetadata["response_schema"] = responseSchema.DiagnosticsMap()
if writer != nil { if writer != nil {
artifacts, _ = writer.WriteInteraction( artifacts, _ = writer.WriteInteraction(
@@ -201,23 +202,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}, nil }, nil
} }
func schemaMetadata(schema responseschema.Schema) map[string]any {
return map[string]any{
"id": schema.ID,
"version": schema.Version,
"name": schema.Name,
"sha256": schema.SHA256,
}
}
func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string {
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
if section == nil {
return base
}
return fmt.Sprintf("%s:section-%04d", base, section.Index)
}
func resolveModel(cfg *config.Config, override string) string { func resolveModel(cfg *config.Config, override string) string {
if strings.TrimSpace(override) != "" { if strings.TrimSpace(override) != "" {
return strings.TrimSpace(override) return strings.TrimSpace(override)
@@ -288,6 +272,14 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
return artifacts.ResponsePayloadPath return artifacts.ResponsePayloadPath
} }
func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if section == nil {
return nil
}
index := section.Index
return &index
}
type diagnosticsWriterAdapter struct { type diagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter writer *llm.DiagnosticsWriter
} }

View File

@@ -5,6 +5,7 @@ import (
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
"fmt" "fmt"
"sort"
"strings" "strings"
) )
@@ -28,6 +29,15 @@ type Schema struct {
SHA256 string `json:"sha256"` SHA256 string `json:"sha256"`
} }
func (s Schema) DiagnosticsMap() map[string]any {
return map[string]any{
"id": s.ID,
"version": s.Version,
"name": s.Name,
"sha256": s.SHA256,
}
}
var registry = map[Key]Schema{ var registry = map[Key]Schema{
CorrectionSetKey: mustBuildSchema( CorrectionSetKey: mustBuildSchema(
correctionSetSchemaID, correctionSetSchemaID,
@@ -43,6 +53,20 @@ var registry = map[Key]Schema{
), ),
} }
func Registered() []Schema {
keys := make([]string, 0, len(registry))
for key := range registry {
keys = append(keys, string(key))
}
sort.Strings(keys)
out := make([]Schema, 0, len(keys))
for _, key := range keys {
out = append(out, cloneSchema(registry[Key(key)]))
}
return out
}
// Lookup returns a copy of the registered schema for the provided key. // Lookup returns a copy of the registered schema for the provided key.
func Lookup(key Key) (Schema, bool) { func Lookup(key Key) (Schema, bool) {
schema, ok := registry[key] schema, ok := registry[key]

View File

@@ -89,3 +89,20 @@ func TestLookupReturnsSchemaCopy(t *testing.T) {
t.Fatalf("expected lookup to return independent schema copy") t.Fatalf("expected lookup to return independent schema copy")
} }
} }
func TestDiagnosticsMapIncludesStableSchemaMetadataShapeForAllSchemas(t *testing.T) {
registered := Registered()
if len(registered) == 0 {
t.Fatalf("expected registered response schemas")
}
for _, schema := range registered {
metadataMap := schema.DiagnosticsMap()
if metadataMap["id"] != schema.ID ||
metadataMap["version"] != schema.Version ||
metadataMap["name"] != schema.Name ||
metadataMap["sha256"] != schema.SHA256 {
t.Fatalf("unexpected diagnostics metadata map for %q: %+v", schema.ID, metadataMap)
}
}
}

View File

@@ -0,0 +1,24 @@
package stagename
import (
"fmt"
)
func ModuleProposal(moduleInstance string, sectionIndex *int) string {
if sectionIndex == nil || *sectionIndex == 0 {
return fmt.Sprintf("%s:proposal", moduleInstance)
}
return fmt.Sprintf("%s:proposal:section-%04d", moduleInstance, *sectionIndex)
}
func ProposalGeneration(moduleInstance string, sectionIndex *int) string {
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
if sectionIndex == nil {
return base
}
return fmt.Sprintf("%s:section-%04d", base, *sectionIndex)
}
func ValidatorBatch(moduleInstance string, validatorName string, batchIndex int) string {
return fmt.Sprintf("%s:%s:batch-%04d", moduleInstance, validatorName, batchIndex)
}

View File

@@ -0,0 +1,33 @@
package stagename
import (
"testing"
)
func TestModuleProposalStageName(t *testing.T) {
if got := ModuleProposal("grammar", nil); got != "grammar:proposal" {
t.Fatalf("unexpected stage name without section: %q", got)
}
sectionIndex := 7
if got := ModuleProposal("grammar", &sectionIndex); got != "grammar:proposal:section-0007" {
t.Fatalf("unexpected stage name with section: %q", got)
}
}
func TestProposalGenerationStageName(t *testing.T) {
if got := ProposalGeneration("grammar", nil); got != "grammar:proposal-generation" {
t.Fatalf("unexpected proposal generation stage name without section: %q", got)
}
sectionIndex := 3
if got := ProposalGeneration("grammar", &sectionIndex); got != "grammar:proposal-generation:section-0003" {
t.Fatalf("unexpected proposal generation stage name with section: %q", got)
}
}
func TestValidatorBatchStageName(t *testing.T) {
if got := ValidatorBatch("homophones_1", "spoken_form_plausibility_review", 12); got != "homophones_1:spoken_form_plausibility_review:batch-0012" {
t.Fatalf("unexpected validator batch stage name: %q", got)
}
}

View File

@@ -11,6 +11,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
"gitea.maximumdirect.net/eric/audita/internal/prompts" "gitea.maximumdirect.net/eric/audita/internal/prompts"
) )
@@ -110,9 +111,10 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
var response LLMValidationResponse var response LLMValidationResponse
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
stage := stagename.ValidatorBatch(req.ModuleInstance, v.name, batch.BatchIndex)
call := func(callCtx context.Context) error { call := func(callCtx context.Context) error {
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{ _, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex), StageName: stage,
Messages: messages, Messages: messages,
Model: resolvedValidationModel(req.Config, v.model), Model: resolvedValidationModel(req.Config, v.model),
ResponseSchema: &responseSchema, ResponseSchema: &responseSchema,
@@ -126,7 +128,6 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
} }
artifacts := InteractionArtifacts{} artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil { if req.DiagnosticsWriter != nil {
stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex)
promptMetadata := validatorPromptMetadata(v.validatorType) promptMetadata := validatorPromptMetadata(v.validatorType)
artifacts, _ = req.DiagnosticsWriter.WriteInteraction( artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
stage, stage,
@@ -134,19 +135,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
"validator_name": v.name, "validator_name": v.name,
"validator_type": v.validatorType, "validator_type": v.validatorType,
"batch_index": batch.BatchIndex, "batch_index": batch.BatchIndex,
"prompt_metadata": map[string]any{ "prompt_metadata": promptMetadata.DiagnosticsMap(),
"prompt_id": promptMetadata.PromptID, "response_schema": responseSchema.DiagnosticsMap(),
"prompt_version": promptMetadata.PromptVersion,
"prompt_source": promptMetadata.PromptSource,
"embedded_path": promptMetadata.EmbeddedPath,
"sha256": promptMetadata.SHA256,
},
"response_schema": map[string]any{
"id": responseSchema.ID,
"version": responseSchema.Version,
"name": responseSchema.Name,
"sha256": responseSchema.SHA256,
},
}, },
map[string]any{"messages": messages, "items": batch.Items}, map[string]any{"messages": messages, "items": batch.Items},
response, response,

View File

@@ -2,12 +2,12 @@ package glossary
import ( import (
"context" "context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
) )
@@ -60,14 +60,8 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
Glossary: req.Glossary, Glossary: req.Glossary,
Config: req.Config, Config: req.Config,
Messages: messages, Messages: messages,
PromptMetadata: map[string]any{ PromptMetadata: proposalPromptMetadata().DiagnosticsMap(),
"prompt_id": proposalPromptMetadata().PromptID, StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)),
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0, StartIndex: 0,
LLMClient: req.LLMClient, LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler, Scheduler: req.LLMScheduler,
@@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect
return &schema.Transcript{Segments: segments} return &schema.Transcript{Segments: segments}
} }
func proposalStageName(req contracts.ProposalRequest) string { func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if req.Section == nil || req.Section.Index == 0 { if section == nil {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName) return nil
} }
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index) index := section.Index
return &index
} }

View File

@@ -2,12 +2,12 @@ package grammar
import ( import (
"context" "context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
) )
@@ -60,14 +60,8 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
Glossary: req.Glossary, Glossary: req.Glossary,
Config: req.Config, Config: req.Config,
Messages: messages, Messages: messages,
PromptMetadata: map[string]any{ PromptMetadata: proposalPromptMetadata().DiagnosticsMap(),
"prompt_id": proposalPromptMetadata().PromptID, StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)),
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0, StartIndex: 0,
LLMClient: req.LLMClient, LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler, Scheduler: req.LLMScheduler,
@@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect
return &schema.Transcript{Segments: segments} return &schema.Transcript{Segments: segments}
} }
func proposalStageName(req contracts.ProposalRequest) string { func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if req.Section == nil || req.Section.Index == 0 { if section == nil {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName) return nil
} }
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index) index := section.Index
return &index
} }

View File

@@ -2,12 +2,12 @@ package homophones
import ( import (
"context" "context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
) )
@@ -60,14 +60,8 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
Glossary: req.Glossary, Glossary: req.Glossary,
Config: req.Config, Config: req.Config,
Messages: messages, Messages: messages,
PromptMetadata: map[string]any{ PromptMetadata: proposalPromptMetadata().DiagnosticsMap(),
"prompt_id": proposalPromptMetadata().PromptID, StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)),
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0, StartIndex: 0,
LLMClient: req.LLMClient, LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler, Scheduler: req.LLMScheduler,
@@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect
return &schema.Transcript{Segments: segments} return &schema.Transcript{Segments: segments}
} }
func proposalStageName(req contracts.ProposalRequest) string { func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if req.Section == nil || req.Section.Index == 0 { if section == nil {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName) return nil
} }
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index) index := section.Index
return &index
} }

View File

@@ -2,12 +2,12 @@ package spoken_word
import ( import (
"context" "context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
) )
@@ -60,14 +60,8 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
Glossary: req.Glossary, Glossary: req.Glossary,
Config: req.Config, Config: req.Config,
Messages: messages, Messages: messages,
PromptMetadata: map[string]any{ PromptMetadata: proposalPromptMetadata().DiagnosticsMap(),
"prompt_id": proposalPromptMetadata().PromptID, StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)),
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0, StartIndex: 0,
LLMClient: req.LLMClient, LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler, Scheduler: req.LLMScheduler,
@@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect
return &schema.Transcript{Segments: segments} return &schema.Transcript{Segments: segments}
} }
func proposalStageName(req contracts.ProposalRequest) string { func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if req.Section == nil || req.Section.Index == 0 { if section == nil {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName) return nil
} }
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index) index := section.Index
return &index
} }

View File

@@ -40,6 +40,16 @@ type Metadata struct {
SHA256 string `json:"sha256"` SHA256 string `json:"sha256"`
} }
func (m Metadata) DiagnosticsMap() map[string]any {
return map[string]any{
"prompt_id": m.PromptID,
"prompt_version": m.PromptVersion,
"prompt_source": m.PromptSource,
"embedded_path": m.EmbeddedPath,
"sha256": m.SHA256,
}
}
type definition struct { type definition struct {
id string id string
version string version string

View File

@@ -107,3 +107,16 @@ func TestRenderedPromptsContainHardening(t *testing.T) {
} }
} }
} }
func TestDiagnosticsMapIncludesStablePromptMetadataShapeForAllPrompts(t *testing.T) {
for _, m := range RegisteredMetadata() {
metadataMap := m.DiagnosticsMap()
if metadataMap["prompt_id"] != m.PromptID ||
metadataMap["prompt_version"] != m.PromptVersion ||
metadataMap["prompt_source"] != m.PromptSource ||
metadataMap["embedded_path"] != m.EmbeddedPath ||
metadataMap["sha256"] != m.SHA256 {
t.Fatalf("unexpected diagnostics metadata map for %q: %+v", m.PromptID, metadataMap)
}
}
}