eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add shared metadata maps and stage-name helpers

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-23 17:48:37 +00:00
parent 13029dbb33
commit e053f7e124
12 changed files with 188 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
internal/framework/proposal_generation/generate.go
View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
@@ -94,7 +95,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
stage := strings.TrimSpace(req.StageName)
if stage == "" {
stage = buildStageName(req.ModuleInstance, req.Section)
stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section))
}
model := resolveModel(req.Config, req.Model)
messages := append([]contracts.LLMMessage(nil), req.Messages...)
@@ -143,7 +144,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if len(req.PromptMetadata) > 0 {
requestMetadata["prompt_metadata"] = req.PromptMetadata
}
requestMetadata["response_schema"] = schemaMetadata(responseSchema)
requestMetadata["response_schema"] = responseSchema.DiagnosticsMap()
if writer != nil {
artifacts, _ = writer.WriteInteraction(
@@ -201,23 +202,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}, nil
}
func schemaMetadata(schema responseschema.Schema) map[string]any {
return map[string]any{
"id": schema.ID,
"version": schema.Version,
"name": schema.Name,
"sha256": schema.SHA256,
}
}
func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string {
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
if section == nil {
return base
}
return fmt.Sprintf("%s:section-%04d", base, section.Index)
}
func resolveModel(cfg *config.Config, override string) string {
if strings.TrimSpace(override) != "" {
return strings.TrimSpace(override)
@@ -288,6 +272,14 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
return artifacts.ResponsePayloadPath
}
func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if section == nil {
return nil
}
index := section.Index
return &index
}
type diagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter
}

24
internal/framework/responseschema/registry.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"sort"
"strings"
)
@@ -28,6 +29,15 @@ type Schema struct {
SHA256 string `json:"sha256"`
}
func (s Schema) DiagnosticsMap() map[string]any {
return map[string]any{
"id": s.ID,
"version": s.Version,
"name": s.Name,
"sha256": s.SHA256,
}
}
var registry = map[Key]Schema{
CorrectionSetKey: mustBuildSchema(
correctionSetSchemaID,
@@ -43,6 +53,20 @@ var registry = map[Key]Schema{
),
}
func Registered() []Schema {
keys := make([]string, 0, len(registry))
for key := range registry {
keys = append(keys, string(key))
}
sort.Strings(keys)
out := make([]Schema, 0, len(keys))
for _, key := range keys {
out = append(out, cloneSchema(registry[Key(key)]))
}
return out
}
// Lookup returns a copy of the registered schema for the provided key.
func Lookup(key Key) (Schema, bool) {
schema, ok := registry[key]

17
internal/framework/responseschema/registry_test.go
View File

@@ -89,3 +89,20 @@ func TestLookupReturnsSchemaCopy(t *testing.T) {
t.Fatalf("expected lookup to return independent schema copy")
}
}
func TestDiagnosticsMapIncludesStableSchemaMetadataShapeForAllSchemas(t *testing.T) {
registered := Registered()
if len(registered) == 0 {
t.Fatalf("expected registered response schemas")
}
for _, schema := range registered {
metadataMap := schema.DiagnosticsMap()
if metadataMap["id"] != schema.ID ||
metadataMap["version"] != schema.Version ||
metadataMap["name"] != schema.Name ||
metadataMap["sha256"] != schema.SHA256 {
t.Fatalf("unexpected diagnostics metadata map for %q: %+v", schema.ID, metadataMap)
}
}
}

24
internal/framework/stagename/stagename.go Normal file
View File

@@ -0,0 +1,24 @@
package stagename
import (
"fmt"
)
func ModuleProposal(moduleInstance string, sectionIndex *int) string {
if sectionIndex == nil || *sectionIndex == 0 {
return fmt.Sprintf("%s:proposal", moduleInstance)
}
return fmt.Sprintf("%s:proposal:section-%04d", moduleInstance, *sectionIndex)
}
func ProposalGeneration(moduleInstance string, sectionIndex *int) string {
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
if sectionIndex == nil {
return base
}
return fmt.Sprintf("%s:section-%04d", base, *sectionIndex)
}
func ValidatorBatch(moduleInstance string, validatorName string, batchIndex int) string {
return fmt.Sprintf("%s:%s:batch-%04d", moduleInstance, validatorName, batchIndex)
}

33
internal/framework/stagename/stagename_test.go Normal file
View File

@@ -0,0 +1,33 @@
package stagename
import (
"testing"
)
func TestModuleProposalStageName(t *testing.T) {
if got := ModuleProposal("grammar", nil); got != "grammar:proposal" {
t.Fatalf("unexpected stage name without section: %q", got)
}
sectionIndex := 7
if got := ModuleProposal("grammar", &sectionIndex); got != "grammar:proposal:section-0007" {
t.Fatalf("unexpected stage name with section: %q", got)
}
}
func TestProposalGenerationStageName(t *testing.T) {
if got := ProposalGeneration("grammar", nil); got != "grammar:proposal-generation" {
t.Fatalf("unexpected proposal generation stage name without section: %q", got)
}
sectionIndex := 3
if got := ProposalGeneration("grammar", &sectionIndex); got != "grammar:proposal-generation:section-0003" {
t.Fatalf("unexpected proposal generation stage name with section: %q", got)
}
}
func TestValidatorBatchStageName(t *testing.T) {
if got := ValidatorBatch("homophones_1", "spoken_form_plausibility_review", 12); got != "homophones_1:spoken_form_plausibility_review:batch-0012" {
t.Fatalf("unexpected validator batch stage name: %q", got)
}
}

26
internal/framework/validators/llm_validators.go
View File

@@ -11,6 +11,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
@@ -110,9 +111,10 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
var response LLMValidationResponse
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
stage := stagename.ValidatorBatch(req.ModuleInstance, v.name, batch.BatchIndex)
call := func(callCtx context.Context) error {
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
StageName: stage,
Messages: messages,
Model: resolvedValidationModel(req.Config, v.model),
ResponseSchema: &responseSchema,
@@ -126,27 +128,15 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil {
stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex)
promptMetadata := validatorPromptMetadata(v.validatorType)
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"prompt_metadata": map[string]any{
"prompt_id": promptMetadata.PromptID,
"prompt_version": promptMetadata.PromptVersion,
"prompt_source": promptMetadata.PromptSource,
"embedded_path": promptMetadata.EmbeddedPath,
"sha256": promptMetadata.SHA256,
},
"response_schema": map[string]any{
"id": responseSchema.ID,
"version": responseSchema.Version,
"name": responseSchema.Name,
"sha256": responseSchema.SHA256,
},
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"prompt_metadata": promptMetadata.DiagnosticsMap(),
"response_schema": responseSchema.DiagnosticsMap(),
},
map[string]any{"messages": messages, "items": batch.Items},
response,

29
internal/modules/glossary/module.go
View File

@@ -2,12 +2,12 @@ package glossary
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
@@ -60,18 +60,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
PromptMetadata: map[string]any{
"prompt_id": proposalPromptMetadata().PromptID,
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
PromptMetadata: proposalPromptMetadata().DiagnosticsMap(),
StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return contracts.ProposalResult{}, err
@@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect
return &schema.Transcript{Segments: segments}
}
func proposalStageName(req contracts.ProposalRequest) string {
if req.Section == nil || req.Section.Index == 0 {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if section == nil {
return nil
}
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
index := section.Index
return &index
}

29
internal/modules/grammar/module.go
View File

@@ -2,12 +2,12 @@ package grammar
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
@@ -60,18 +60,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
PromptMetadata: map[string]any{
"prompt_id": proposalPromptMetadata().PromptID,
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
PromptMetadata: proposalPromptMetadata().DiagnosticsMap(),
StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return contracts.ProposalResult{}, err
@@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect
return &schema.Transcript{Segments: segments}
}
func proposalStageName(req contracts.ProposalRequest) string {
if req.Section == nil || req.Section.Index == 0 {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if section == nil {
return nil
}
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
index := section.Index
return &index
}

29
internal/modules/homophones/module.go
View File

@@ -2,12 +2,12 @@ package homophones
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
@@ -60,18 +60,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
PromptMetadata: map[string]any{
"prompt_id": proposalPromptMetadata().PromptID,
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
PromptMetadata: proposalPromptMetadata().DiagnosticsMap(),
StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return contracts.ProposalResult{}, err
@@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect
return &schema.Transcript{Segments: segments}
}
func proposalStageName(req contracts.ProposalRequest) string {
if req.Section == nil || req.Section.Index == 0 {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if section == nil {
return nil
}
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
index := section.Index
return &index
}

29
internal/modules/spoken_word/module.go
View File

@@ -2,12 +2,12 @@ package spoken_word
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
@@ -60,18 +60,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
PromptMetadata: map[string]any{
"prompt_id": proposalPromptMetadata().PromptID,
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
PromptMetadata: proposalPromptMetadata().DiagnosticsMap(),
StageName: stagename.ModuleProposal(req.RunSpec.InstanceName, sectionIndexPtr(req.Section)),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return contracts.ProposalResult{}, err
@@ -95,9 +89,10 @@ func transcriptForSection(transcript *schema.Transcript, section *contracts.Sect
return &schema.Transcript{Segments: segments}
}
func proposalStageName(req contracts.ProposalRequest) string {
if req.Section == nil || req.Section.Index == 0 {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if section == nil {
return nil
}
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
index := section.Index
return &index
}

10
internal/prompts/registry.go
View File

@@ -40,6 +40,16 @@ type Metadata struct {
SHA256 string `json:"sha256"`
}
func (m Metadata) DiagnosticsMap() map[string]any {
return map[string]any{
"prompt_id": m.PromptID,
"prompt_version": m.PromptVersion,
"prompt_source": m.PromptSource,
"embedded_path": m.EmbeddedPath,
"sha256": m.SHA256,
}
}
type definition struct {
id string
version string

13
internal/prompts/registry_test.go
View File

@@ -107,3 +107,16 @@ func TestRenderedPromptsContainHardening(t *testing.T) {
}
}
}
func TestDiagnosticsMapIncludesStablePromptMetadataShapeForAllPrompts(t *testing.T) {
for _, m := range RegisteredMetadata() {
metadataMap := m.DiagnosticsMap()
if metadataMap["prompt_id"] != m.PromptID ||
metadataMap["prompt_version"] != m.PromptVersion ||
metadataMap["prompt_source"] != m.PromptSource ||
metadataMap["embedded_path"] != m.EmbeddedPath ||
metadataMap["sha256"] != m.SHA256 {
t.Fatalf("unexpected diagnostics metadata map for %q: %+v", m.PromptID, metadataMap)
}
}
}