Compare commits
2 Commits
f790c1441c
...
0630d36734
| Author | SHA1 | Date | |
|---|---|---|---|
| 0630d36734 | |||
| 52c2697040 |
121
internal/cli/process_flags.go
Normal file
121
internal/cli/process_flags.go
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type processOverrideBinding func(*config.CLIOverrides, processFlags)
|
||||||
|
|
||||||
|
var processOverrideBindings = map[string]processOverrideBinding{
|
||||||
|
"modules": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ModulesCSV = flags.modules
|
||||||
|
},
|
||||||
|
"output-schema": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.OutputSchema = flags.outputSchema
|
||||||
|
},
|
||||||
|
"llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.PrimaryLLMAPIKey = flags.llmAPIKey
|
||||||
|
},
|
||||||
|
"validation-llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationLLMAPIKey = flags.validationLLMAPIKey
|
||||||
|
},
|
||||||
|
"model": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.PrimaryModel = flags.model
|
||||||
|
},
|
||||||
|
"validation-model": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationModel = flags.validationModel
|
||||||
|
},
|
||||||
|
"base-url": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.PrimaryBaseURL = flags.baseURL
|
||||||
|
},
|
||||||
|
"validation-base-url": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationBaseURL = flags.validationBaseURL
|
||||||
|
},
|
||||||
|
"llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.PrimaryLLMTimeoutSeconds = flags.llmTimeoutSeconds
|
||||||
|
},
|
||||||
|
"total-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.TotalLLMConcurrency = flags.totalLLMConcurrency
|
||||||
|
},
|
||||||
|
"proposal-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ProposalLLMConcurrency = flags.proposalLLMConcurrency
|
||||||
|
},
|
||||||
|
"llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.PrimaryLLMConcurrency = flags.llmConcurrency
|
||||||
|
},
|
||||||
|
"validation-llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationLLMTimeoutSeconds = flags.validationLLMTimeoutSeconds
|
||||||
|
},
|
||||||
|
"max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.MaxRetries = flags.maxRetries
|
||||||
|
},
|
||||||
|
"validation-max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationMaxRetries = flags.validationMaxRetries
|
||||||
|
},
|
||||||
|
"validation-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationLLMConcurrency = flags.validationLLMConcurrency
|
||||||
|
},
|
||||||
|
"validation-max-prompt-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationMaxPromptTokens = flags.validationMaxPromptTokens
|
||||||
|
},
|
||||||
|
"max-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.MaxSectionTokens = flags.maxSectionTokens
|
||||||
|
},
|
||||||
|
"min-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.MinSectionTokens = flags.minSectionTokens
|
||||||
|
},
|
||||||
|
"target-sections": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.TargetSections = flags.targetSections
|
||||||
|
},
|
||||||
|
"glossary-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.GlossaryConfidenceThreshold = flags.glossaryConfidenceThreshold
|
||||||
|
},
|
||||||
|
"grammar-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.GrammarConfidenceThreshold = flags.grammarConfidenceThreshold
|
||||||
|
},
|
||||||
|
"homophones-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.HomophonesConfidenceThreshold = flags.homophonesConfidenceThreshold
|
||||||
|
},
|
||||||
|
"spoken-word-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.SpokenWordConfidenceThreshold = flags.spokenWordConfidenceThreshold
|
||||||
|
},
|
||||||
|
"normalize-max-segment-gap": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.NormalizeMaxSegmentGap = flags.normalizeMaxSegmentGap
|
||||||
|
},
|
||||||
|
"normalize-ellipsis-gap": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.NormalizeEllipsisGap = flags.normalizeEllipsisGap
|
||||||
|
},
|
||||||
|
"normalize-max-segment-duration": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.NormalizeMaxSegmentDuration = flags.normalizeMaxSegmentDuration
|
||||||
|
},
|
||||||
|
"normalize-max-segment-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.NormalizeMaxSegmentTokens = flags.normalizeMaxSegmentTokens
|
||||||
|
},
|
||||||
|
"transcript-description": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.TranscriptDescription = flags.transcriptDescription
|
||||||
|
},
|
||||||
|
"work-dir": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.WorkDir = flags.workDir
|
||||||
|
},
|
||||||
|
"work-dir-retention": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.WorkDirRetention = flags.workDirRetention
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func processCLIOverrides(fs *flag.FlagSet, flags processFlags) (config.CLIOverrides, bool) {
|
||||||
|
overrides := config.CLIOverrides{}
|
||||||
|
explicitModules := false
|
||||||
|
fs.Visit(func(f *flag.Flag) {
|
||||||
|
if f.Name == "modules" {
|
||||||
|
explicitModules = true
|
||||||
|
}
|
||||||
|
binding, ok := processOverrideBindings[f.Name]
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
binding(&overrides, flags)
|
||||||
|
})
|
||||||
|
return overrides, explicitModules
|
||||||
|
}
|
||||||
433
internal/cli/process_flags_test.go
Normal file
433
internal/cli/process_flags_test.go
Normal file
@@ -0,0 +1,433 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProcessCLIOverridesMapsEveryConfigMutatingFlag(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
flagName string
|
||||||
|
value string
|
||||||
|
wantExplicitModules bool
|
||||||
|
assertOverrideFields func(t *testing.T, overrides config.CLIOverrides)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "modules",
|
||||||
|
flagName: "modules",
|
||||||
|
value: "grammar,glossary",
|
||||||
|
wantExplicitModules: true,
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "ModulesCSV", overrides.ModulesCSV, "grammar,glossary")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "output schema",
|
||||||
|
flagName: "output-schema",
|
||||||
|
value: "audita-v1",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "OutputSchema", overrides.OutputSchema, "audita-v1")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "primary api key",
|
||||||
|
flagName: "llm-api-key",
|
||||||
|
value: "primary-key",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "PrimaryLLMAPIKey", overrides.PrimaryLLMAPIKey, "primary-key")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation api key",
|
||||||
|
flagName: "validation-llm-api-key",
|
||||||
|
value: "validation-key",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "ValidationLLMAPIKey", overrides.ValidationLLMAPIKey, "validation-key")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "primary model",
|
||||||
|
flagName: "model",
|
||||||
|
value: "primary-model",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "PrimaryModel", overrides.PrimaryModel, "primary-model")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation model",
|
||||||
|
flagName: "validation-model",
|
||||||
|
value: "validation-model",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "ValidationModel", overrides.ValidationModel, "validation-model")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "primary base url",
|
||||||
|
flagName: "base-url",
|
||||||
|
value: "https://primary.example.test",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "PrimaryBaseURL", overrides.PrimaryBaseURL, "https://primary.example.test")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation base url",
|
||||||
|
flagName: "validation-base-url",
|
||||||
|
value: "https://validation.example.test",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "ValidationBaseURL", overrides.ValidationBaseURL, "https://validation.example.test")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "primary timeout",
|
||||||
|
flagName: "llm-timeout-seconds",
|
||||||
|
value: "101",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "PrimaryLLMTimeoutSeconds", overrides.PrimaryLLMTimeoutSeconds, 101)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "total concurrency",
|
||||||
|
flagName: "total-llm-concurrency",
|
||||||
|
value: "5",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "TotalLLMConcurrency", overrides.TotalLLMConcurrency, 5)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "proposal concurrency",
|
||||||
|
flagName: "proposal-llm-concurrency",
|
||||||
|
value: "3",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "ProposalLLMConcurrency", overrides.ProposalLLMConcurrency, 3)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "legacy concurrency alias",
|
||||||
|
flagName: "llm-concurrency",
|
||||||
|
value: "4",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "PrimaryLLMConcurrency", overrides.PrimaryLLMConcurrency, 4)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation timeout",
|
||||||
|
flagName: "validation-llm-timeout-seconds",
|
||||||
|
value: "202",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "ValidationLLMTimeoutSeconds", overrides.ValidationLLMTimeoutSeconds, 202)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "max retries",
|
||||||
|
flagName: "max-retries",
|
||||||
|
value: "6",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "MaxRetries", overrides.MaxRetries, 6)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation max retries",
|
||||||
|
flagName: "validation-max-retries",
|
||||||
|
value: "7",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "ValidationMaxRetries", overrides.ValidationMaxRetries, 7)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation concurrency",
|
||||||
|
flagName: "validation-llm-concurrency",
|
||||||
|
value: "8",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "ValidationLLMConcurrency", overrides.ValidationLLMConcurrency, 8)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation max prompt tokens",
|
||||||
|
flagName: "validation-max-prompt-tokens",
|
||||||
|
value: "4096",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "ValidationMaxPromptTokens", overrides.ValidationMaxPromptTokens, 4096)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "max section tokens",
|
||||||
|
flagName: "max-section-tokens",
|
||||||
|
value: "9000",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "MaxSectionTokens", overrides.MaxSectionTokens, 9000)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "min section tokens",
|
||||||
|
flagName: "min-section-tokens",
|
||||||
|
value: "1000",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "MinSectionTokens", overrides.MinSectionTokens, 1000)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "target sections",
|
||||||
|
flagName: "target-sections",
|
||||||
|
value: "12",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "TargetSections", overrides.TargetSections, 12)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "glossary threshold",
|
||||||
|
flagName: "glossary-confidence-threshold",
|
||||||
|
value: "0.91",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "GlossaryConfidenceThreshold", overrides.GlossaryConfidenceThreshold, 0.91)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "grammar threshold",
|
||||||
|
flagName: "grammar-confidence-threshold",
|
||||||
|
value: "0.92",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "GrammarConfidenceThreshold", overrides.GrammarConfidenceThreshold, 0.92)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "homophones threshold",
|
||||||
|
flagName: "homophones-confidence-threshold",
|
||||||
|
value: "0.93",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "HomophonesConfidenceThreshold", overrides.HomophonesConfidenceThreshold, 0.93)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "spoken word threshold",
|
||||||
|
flagName: "spoken-word-confidence-threshold",
|
||||||
|
value: "0.94",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "SpokenWordConfidenceThreshold", overrides.SpokenWordConfidenceThreshold, 0.94)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize max segment gap",
|
||||||
|
flagName: "normalize-max-segment-gap",
|
||||||
|
value: "1.2",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "NormalizeMaxSegmentGap", overrides.NormalizeMaxSegmentGap, 1.2)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize ellipsis gap",
|
||||||
|
flagName: "normalize-ellipsis-gap",
|
||||||
|
value: "2.3",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "NormalizeEllipsisGap", overrides.NormalizeEllipsisGap, 2.3)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize max segment duration",
|
||||||
|
flagName: "normalize-max-segment-duration",
|
||||||
|
value: "45.6",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "NormalizeMaxSegmentDuration", overrides.NormalizeMaxSegmentDuration, 45.6)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize max segment tokens",
|
||||||
|
flagName: "normalize-max-segment-tokens",
|
||||||
|
value: "321",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "NormalizeMaxSegmentTokens", overrides.NormalizeMaxSegmentTokens, 321)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "transcript description",
|
||||||
|
flagName: "transcript-description",
|
||||||
|
value: "podcast episode",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "TranscriptDescription", overrides.TranscriptDescription, "podcast episode")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "work dir",
|
||||||
|
flagName: "work-dir",
|
||||||
|
value: "/tmp/custom-audita",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "WorkDir", overrides.WorkDir, "/tmp/custom-audita")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "work dir retention",
|
||||||
|
flagName: "work-dir-retention",
|
||||||
|
value: "always",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "WorkDirRetention", overrides.WorkDirRetention, "always")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
|
||||||
|
if err := fs.Parse([]string{"--" + tc.flagName, tc.value}); err != nil {
|
||||||
|
t.Fatalf("parse flag: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
overrides, explicitModules := processCLIOverrides(fs, flags)
|
||||||
|
if explicitModules != tc.wantExplicitModules {
|
||||||
|
t.Fatalf("explicitModules=%v, want %v", explicitModules, tc.wantExplicitModules)
|
||||||
|
}
|
||||||
|
tc.assertOverrideFields(t, overrides)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessCLIOverridesIgnoresNonConfigFlags(t *testing.T) {
|
||||||
|
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
|
||||||
|
if err := fs.Parse([]string{
|
||||||
|
"--config", "/tmp/config.yml",
|
||||||
|
"--glossary", "/tmp/glossary.yml",
|
||||||
|
"--output", "/tmp/output.json",
|
||||||
|
"--report-json", "/tmp/report.json",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("parse flags: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
overrides, explicitModules := processCLIOverrides(fs, flags)
|
||||||
|
if explicitModules {
|
||||||
|
t.Fatal("non-config flags should not mark modules explicit")
|
||||||
|
}
|
||||||
|
assertNoCLIOverrides(t, overrides)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewProcessFlagSetDefaultsReflectEffectiveConfig(t *testing.T) {
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.Modules = []string{"grammar", "glossary"}
|
||||||
|
cfg.OutputSchema = "audita-v1"
|
||||||
|
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||||
|
cfg.ValidationLLM.APIKey = "validation-key"
|
||||||
|
cfg.PrimaryLLM.Model = "primary-model"
|
||||||
|
cfg.ValidationLLM.Model = "validation-model"
|
||||||
|
cfg.PrimaryLLM.BaseURL = "https://primary.example.test"
|
||||||
|
cfg.ValidationLLM.BaseURL = "https://validation.example.test"
|
||||||
|
cfg.PrimaryLLM.TimeoutSeconds = 101
|
||||||
|
cfg.TotalLLMConcurrency = 5
|
||||||
|
cfg.ProposalLLMConcurrency = 3
|
||||||
|
cfg.PrimaryLLM.MaxRetries = 6
|
||||||
|
cfg.ValidationMaxPromptTokens = 4096
|
||||||
|
cfg.MaxSectionTokens = 9000
|
||||||
|
cfg.MinSectionTokens = 1000
|
||||||
|
cfg.Thresholds.Glossary = 0.91
|
||||||
|
cfg.Thresholds.Grammar = 0.92
|
||||||
|
cfg.Thresholds.Homophones = 0.93
|
||||||
|
cfg.Thresholds.SpokenWord = 0.94
|
||||||
|
cfg.Normalization.MaxSegmentGap = 1.2
|
||||||
|
cfg.Normalization.EllipsisGap = 2.3
|
||||||
|
cfg.Normalization.MaxSegmentDuration = 45.6
|
||||||
|
cfg.Normalization.MaxSegmentTokens = 321
|
||||||
|
cfg.TranscriptDescription = "podcast episode"
|
||||||
|
cfg.WorkDir = "/tmp/custom-audita"
|
||||||
|
cfg.WorkDirRetention = config.WorkDirRetentionAlways
|
||||||
|
|
||||||
|
validationTimeout := 202
|
||||||
|
validationRetries := 7
|
||||||
|
validationConcurrency := 8
|
||||||
|
targetSections := 12
|
||||||
|
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
|
||||||
|
cfg.ValidationLLM.MaxRetries = &validationRetries
|
||||||
|
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||||
|
cfg.TargetSections = &targetSections
|
||||||
|
|
||||||
|
_, flags := newProcessFlagSet(cfg, io.Discard)
|
||||||
|
|
||||||
|
assertStringOverride(t, "modules default", flags.modules, "grammar,glossary")
|
||||||
|
assertStringOverride(t, "output schema default", flags.outputSchema, "audita-v1")
|
||||||
|
assertStringOverride(t, "primary api key default", flags.llmAPIKey, "primary-key")
|
||||||
|
assertStringOverride(t, "validation api key default", flags.validationLLMAPIKey, "validation-key")
|
||||||
|
assertStringOverride(t, "primary model default", flags.model, "primary-model")
|
||||||
|
assertStringOverride(t, "validation model default", flags.validationModel, "validation-model")
|
||||||
|
assertStringOverride(t, "primary base url default", flags.baseURL, "https://primary.example.test")
|
||||||
|
assertStringOverride(t, "validation base url default", flags.validationBaseURL, "https://validation.example.test")
|
||||||
|
assertIntOverride(t, "primary timeout default", flags.llmTimeoutSeconds, 101)
|
||||||
|
assertIntOverride(t, "total concurrency default", flags.totalLLMConcurrency, 5)
|
||||||
|
assertIntOverride(t, "proposal concurrency default", flags.proposalLLMConcurrency, 3)
|
||||||
|
assertIntOverride(t, "legacy concurrency alias default", flags.llmConcurrency, 5)
|
||||||
|
assertIntOverride(t, "validation timeout default", flags.validationLLMTimeoutSeconds, validationTimeout)
|
||||||
|
assertIntOverride(t, "max retries default", flags.maxRetries, 6)
|
||||||
|
assertIntOverride(t, "validation max retries default", flags.validationMaxRetries, validationRetries)
|
||||||
|
assertIntOverride(t, "validation concurrency default", flags.validationLLMConcurrency, validationConcurrency)
|
||||||
|
assertIntOverride(t, "validation max prompt tokens default", flags.validationMaxPromptTokens, 4096)
|
||||||
|
assertIntOverride(t, "max section tokens default", flags.maxSectionTokens, 9000)
|
||||||
|
assertIntOverride(t, "min section tokens default", flags.minSectionTokens, 1000)
|
||||||
|
assertIntOverride(t, "target sections default", flags.targetSections, targetSections)
|
||||||
|
assertFloatOverride(t, "glossary threshold default", flags.glossaryConfidenceThreshold, 0.91)
|
||||||
|
assertFloatOverride(t, "grammar threshold default", flags.grammarConfidenceThreshold, 0.92)
|
||||||
|
assertFloatOverride(t, "homophones threshold default", flags.homophonesConfidenceThreshold, 0.93)
|
||||||
|
assertFloatOverride(t, "spoken word threshold default", flags.spokenWordConfidenceThreshold, 0.94)
|
||||||
|
assertFloatOverride(t, "normalize max segment gap default", flags.normalizeMaxSegmentGap, 1.2)
|
||||||
|
assertFloatOverride(t, "normalize ellipsis gap default", flags.normalizeEllipsisGap, 2.3)
|
||||||
|
assertFloatOverride(t, "normalize max segment duration default", flags.normalizeMaxSegmentDuration, 45.6)
|
||||||
|
assertIntOverride(t, "normalize max segment tokens default", flags.normalizeMaxSegmentTokens, 321)
|
||||||
|
assertStringOverride(t, "transcript description default", flags.transcriptDescription, "podcast episode")
|
||||||
|
assertStringOverride(t, "work dir default", flags.workDir, "/tmp/custom-audita")
|
||||||
|
assertStringOverride(t, "work dir retention default", flags.workDirRetention, "always")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewProcessFlagSetUsesFallbackDefaultsForUnsetOptionalConfig(t *testing.T) {
|
||||||
|
cfg := config.Default()
|
||||||
|
|
||||||
|
_, flags := newProcessFlagSet(cfg, io.Discard)
|
||||||
|
|
||||||
|
assertIntOverride(t, "validation timeout fallback", flags.validationLLMTimeoutSeconds, cfg.PrimaryLLM.TimeoutSeconds)
|
||||||
|
assertIntOverride(t, "validation retries fallback", flags.validationMaxRetries, cfg.PrimaryLLM.MaxRetries)
|
||||||
|
assertIntOverride(t, "validation concurrency fallback", flags.validationLLMConcurrency, cfg.TotalLLMConcurrency)
|
||||||
|
assertIntOverride(t, "target sections fallback", flags.targetSections, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertStringOverride(t *testing.T, name string, got *string, want string) {
|
||||||
|
t.Helper()
|
||||||
|
if got == nil || *got != want {
|
||||||
|
t.Fatalf("%s=%v, want %q", name, pointerValue(got), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertIntOverride(t *testing.T, name string, got *int, want int) {
|
||||||
|
t.Helper()
|
||||||
|
if got == nil || *got != want {
|
||||||
|
t.Fatalf("%s=%v, want %d", name, pointerValue(got), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFloatOverride(t *testing.T, name string, got *float64, want float64) {
|
||||||
|
t.Helper()
|
||||||
|
if got == nil || *got != want {
|
||||||
|
t.Fatalf("%s=%v, want %v", name, pointerValue(got), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertNoCLIOverrides(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
t.Helper()
|
||||||
|
value := reflect.ValueOf(overrides)
|
||||||
|
typ := value.Type()
|
||||||
|
for i := 0; i < value.NumField(); i++ {
|
||||||
|
field := value.Field(i)
|
||||||
|
if field.Kind() != reflect.Ptr {
|
||||||
|
t.Fatalf("unexpected non-pointer CLIOverrides field %s", typ.Field(i).Name)
|
||||||
|
}
|
||||||
|
if !field.IsNil() {
|
||||||
|
t.Fatalf("expected no CLI overrides, field %s was set", typ.Field(i).Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func pointerValue[T any](ptr *T) any {
|
||||||
|
if ptr == nil {
|
||||||
|
return "<nil>"
|
||||||
|
}
|
||||||
|
if stringer, ok := any(*ptr).(interface{ String() string }); ok {
|
||||||
|
return strings.TrimSpace(stringer.String())
|
||||||
|
}
|
||||||
|
return *ptr
|
||||||
|
}
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
|
||||||
output := &runner.RunOutput{
|
|
||||||
ModuleResults: []runner.ModuleResult{
|
|
||||||
{
|
|
||||||
ModuleKey: "glossary",
|
|
||||||
ModuleInstance: "glossary",
|
|
||||||
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
|
||||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
|
||||||
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
|
||||||
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
|
||||||
},
|
|
||||||
AppliedChanges: []proposals.AppliedChange{
|
|
||||||
{
|
|
||||||
ProposalIndex: 3,
|
|
||||||
ModuleKey: "glossary",
|
|
||||||
ModuleInstance: "glossary",
|
|
||||||
TargetSegmentID: 1,
|
|
||||||
OriginalText: "gestures",
|
|
||||||
CorrectedText: "Jesters",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
ledger := buildCorrectionLedger("/tmp/audita-run-id", output)
|
|
||||||
if len(ledger) != 1 {
|
|
||||||
t.Fatalf("expected one ledger entry, got %d", len(ledger))
|
|
||||||
}
|
|
||||||
entry := ledger[0]
|
|
||||||
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
|
|
||||||
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
|
||||||
}
|
|
||||||
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
|
|
||||||
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -23,10 +23,10 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type noOpStructuredLLMClient struct{}
|
type noOpStructuredLLMClient struct{}
|
||||||
@@ -420,75 +420,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
|
||||||
overrides := config.CLIOverrides{}
|
overrides, explicitModules := processCLIOverrides(fs, pFlags)
|
||||||
explicitModules := false
|
|
||||||
fs.Visit(func(f *flag.Flag) {
|
|
||||||
switch f.Name {
|
|
||||||
case "modules":
|
|
||||||
explicitModules = true
|
|
||||||
overrides.ModulesCSV = pFlags.modules
|
|
||||||
case "output-schema":
|
|
||||||
overrides.OutputSchema = pFlags.outputSchema
|
|
||||||
case "llm-api-key":
|
|
||||||
overrides.PrimaryLLMAPIKey = pFlags.llmAPIKey
|
|
||||||
case "validation-llm-api-key":
|
|
||||||
overrides.ValidationLLMAPIKey = pFlags.validationLLMAPIKey
|
|
||||||
case "model":
|
|
||||||
overrides.PrimaryModel = pFlags.model
|
|
||||||
case "validation-model":
|
|
||||||
overrides.ValidationModel = pFlags.validationModel
|
|
||||||
case "base-url":
|
|
||||||
overrides.PrimaryBaseURL = pFlags.baseURL
|
|
||||||
case "validation-base-url":
|
|
||||||
overrides.ValidationBaseURL = pFlags.validationBaseURL
|
|
||||||
case "llm-timeout-seconds":
|
|
||||||
overrides.PrimaryLLMTimeoutSeconds = pFlags.llmTimeoutSeconds
|
|
||||||
case "total-llm-concurrency":
|
|
||||||
overrides.TotalLLMConcurrency = pFlags.totalLLMConcurrency
|
|
||||||
case "proposal-llm-concurrency":
|
|
||||||
overrides.ProposalLLMConcurrency = pFlags.proposalLLMConcurrency
|
|
||||||
case "llm-concurrency":
|
|
||||||
overrides.PrimaryLLMConcurrency = pFlags.llmConcurrency
|
|
||||||
case "validation-llm-timeout-seconds":
|
|
||||||
overrides.ValidationLLMTimeoutSeconds = pFlags.validationLLMTimeoutSeconds
|
|
||||||
case "max-retries":
|
|
||||||
overrides.MaxRetries = pFlags.maxRetries
|
|
||||||
case "validation-max-retries":
|
|
||||||
overrides.ValidationMaxRetries = pFlags.validationMaxRetries
|
|
||||||
case "validation-llm-concurrency":
|
|
||||||
overrides.ValidationLLMConcurrency = pFlags.validationLLMConcurrency
|
|
||||||
case "validation-max-prompt-tokens":
|
|
||||||
overrides.ValidationMaxPromptTokens = pFlags.validationMaxPromptTokens
|
|
||||||
case "max-section-tokens":
|
|
||||||
overrides.MaxSectionTokens = pFlags.maxSectionTokens
|
|
||||||
case "min-section-tokens":
|
|
||||||
overrides.MinSectionTokens = pFlags.minSectionTokens
|
|
||||||
case "target-sections":
|
|
||||||
overrides.TargetSections = pFlags.targetSections
|
|
||||||
case "glossary-confidence-threshold":
|
|
||||||
overrides.GlossaryConfidenceThreshold = pFlags.glossaryConfidenceThreshold
|
|
||||||
case "grammar-confidence-threshold":
|
|
||||||
overrides.GrammarConfidenceThreshold = pFlags.grammarConfidenceThreshold
|
|
||||||
case "homophones-confidence-threshold":
|
|
||||||
overrides.HomophonesConfidenceThreshold = pFlags.homophonesConfidenceThreshold
|
|
||||||
case "spoken-word-confidence-threshold":
|
|
||||||
overrides.SpokenWordConfidenceThreshold = pFlags.spokenWordConfidenceThreshold
|
|
||||||
case "normalize-max-segment-gap":
|
|
||||||
overrides.NormalizeMaxSegmentGap = pFlags.normalizeMaxSegmentGap
|
|
||||||
case "normalize-ellipsis-gap":
|
|
||||||
overrides.NormalizeEllipsisGap = pFlags.normalizeEllipsisGap
|
|
||||||
case "normalize-max-segment-duration":
|
|
||||||
overrides.NormalizeMaxSegmentDuration = pFlags.normalizeMaxSegmentDuration
|
|
||||||
case "normalize-max-segment-tokens":
|
|
||||||
overrides.NormalizeMaxSegmentTokens = pFlags.normalizeMaxSegmentTokens
|
|
||||||
case "transcript-description":
|
|
||||||
overrides.TranscriptDescription = pFlags.transcriptDescription
|
|
||||||
case "work-dir":
|
|
||||||
overrides.WorkDir = pFlags.workDir
|
|
||||||
case "work-dir-retention":
|
|
||||||
overrides.WorkDirRetention = pFlags.workDirRetention
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
||||||
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
||||||
@@ -531,10 +463,13 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
if runOutput.Utilization != nil {
|
if runOutput.Utilization != nil {
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||||
}
|
}
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||||
|
RunDirectoryPath: runDir.Path(),
|
||||||
|
RunOutput: runOutput,
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
errorPhase, errorMessage := extractErrorPhase(runErr)
|
errorPhase, errorMessage := extractErrorPhase(runErr)
|
||||||
report := buildProcessReport("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput)
|
report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
|
||||||
|
|
||||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||||
@@ -560,10 +495,13 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
if runOutput.Utilization != nil {
|
if runOutput.Utilization != nil {
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||||
}
|
}
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||||
|
RunDirectoryPath: runDir.Path(),
|
||||||
|
RunOutput: runOutput,
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
report := buildProcessReport("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput)
|
report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
|
||||||
|
|
||||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||||
@@ -578,21 +516,11 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
hasSkippedCorrections := false
|
|
||||||
if runOutput != nil {
|
|
||||||
for _, mr := range runOutput.ModuleResults {
|
|
||||||
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
|
|
||||||
hasSkippedCorrections = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if runDir != nil {
|
if runDir != nil {
|
||||||
_ = runDir.WriteReport(report)
|
_ = runDir.WriteReport(report)
|
||||||
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||||
RunSucceeded: true,
|
RunSucceeded: true,
|
||||||
HasSkippedCorrections: hasSkippedCorrections,
|
HasSkippedCorrections: processreport.HasSkippedCorrections(runOutput),
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
@@ -710,130 +638,28 @@ func extractErrorPhase(err error) (phase string, message string) {
|
|||||||
return "", msg
|
return "", msg
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport {
|
func processReportInput(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) processreport.BuildInput {
|
||||||
report := reporting.ProcessReport{
|
runDirectoryPath := ""
|
||||||
ReportMetadata: reporting.ReportMetadata{
|
|
||||||
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
|
||||||
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
|
||||||
OutputSchema: inv.Config.OutputSchema,
|
|
||||||
ConfigVersion: inv.ConfigVersion,
|
|
||||||
},
|
|
||||||
Phase: "default_pipeline",
|
|
||||||
Status: status,
|
|
||||||
Operation: "process",
|
|
||||||
TranscriptPath: inv.TranscriptPath,
|
|
||||||
GlossaryPath: inv.GlossaryPath,
|
|
||||||
OutputPath: inv.OutputPath,
|
|
||||||
Modules: append([]string(nil), inv.Config.Modules...),
|
|
||||||
StartedAt: startedAt,
|
|
||||||
CompletedAt: &completedAt,
|
|
||||||
ErrorPhase: errorPhase,
|
|
||||||
}
|
|
||||||
if runDir != nil {
|
if runDir != nil {
|
||||||
runSucceeded := status == "success"
|
runDirectoryPath = runDir.Path()
|
||||||
metadata := diagnostics.BuildDiagnosticsMetadata(runDir.Path(), runSucceeded)
|
|
||||||
report.Diagnostics = &metadata
|
|
||||||
}
|
}
|
||||||
if errorMessage != "" {
|
return processreport.BuildInput{
|
||||||
report.ErrorMessage = errorMessage
|
Status: status,
|
||||||
|
TranscriptPath: inv.TranscriptPath,
|
||||||
|
GlossaryPath: inv.GlossaryPath,
|
||||||
|
OutputPath: inv.OutputPath,
|
||||||
|
Modules: inv.Config.Modules,
|
||||||
|
OutputSchema: inv.Config.OutputSchema,
|
||||||
|
ConfigVersion: inv.ConfigVersion,
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
ErrorMessage: errorMessage,
|
||||||
|
ErrorPhase: errorPhase,
|
||||||
|
RunDirectoryPath: runDirectoryPath,
|
||||||
|
NormalizationSummary: normalizationSummary,
|
||||||
|
ChunkingSummary: chunkingSummary,
|
||||||
|
RunOutput: runOutput,
|
||||||
}
|
}
|
||||||
if normalizationSummary != nil {
|
|
||||||
report.InputSegmentCount = &normalizationSummary.InputSegmentCount
|
|
||||||
report.NormalizedSegmentCount = &normalizationSummary.OutputSegmentCount
|
|
||||||
report.NormalizationMerges = &normalizationSummary.MergesPerformed
|
|
||||||
report.NormalizationIDReassignments = &normalizationSummary.IDsReassigned
|
|
||||||
report.NormalizationSkipped.DifferentSpeakers = &normalizationSummary.SkippedMerges.DifferentSpeakers
|
|
||||||
report.NormalizationSkipped.GapTooLarge = &normalizationSummary.SkippedMerges.GapTooLarge
|
|
||||||
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
|
|
||||||
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
|
|
||||||
}
|
|
||||||
if chunkingSummary != nil {
|
|
||||||
report.Chunking = &reporting.ChunkingSummary{
|
|
||||||
ChunkCount: chunkingSummary.ChunkCount,
|
|
||||||
MinEstimatedTokens: chunkingSummary.MinEstimatedTokens,
|
|
||||||
MaxEstimatedTokens: chunkingSummary.MaxEstimatedTokens,
|
|
||||||
TotalEstimatedTokens: chunkingSummary.TotalEstimatedTokens,
|
|
||||||
TargetSections: chunkingSummary.TargetSections,
|
|
||||||
MaxSectionTokens: chunkingSummary.MaxSectionTokens,
|
|
||||||
MinSectionTokens: chunkingSummary.MinSectionTokens,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
report.ModulesSummary, report.ModuleResults = buildModuleReporting(runOutput)
|
|
||||||
return report
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
|
||||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
|
||||||
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
|
||||||
for _, r := range runOutput.ModuleResults {
|
|
||||||
startedAt := r.StartedAt
|
|
||||||
completedAt := r.CompletedAt
|
|
||||||
moduleReports = append(moduleReports, reporting.ModuleReport{
|
|
||||||
ModuleKey: r.ModuleKey,
|
|
||||||
ModuleInstance: r.ModuleInstance,
|
|
||||||
ReplacementPolicy: string(r.ReplacementPolicy),
|
|
||||||
Status: r.Status,
|
|
||||||
ProposalCount: r.ProposalCount,
|
|
||||||
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
|
||||||
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
|
||||||
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
|
||||||
AppliedChanges: r.AppliedChanges,
|
|
||||||
SkippedChanges: r.SkippedChanges,
|
|
||||||
ErrorMessage: r.ErrorMessage,
|
|
||||||
StartedAt: &startedAt,
|
|
||||||
CompletedAt: &completedAt,
|
|
||||||
})
|
|
||||||
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
|
||||||
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
|
||||||
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
|
||||||
summary.FailedModuleInstance = r.ModuleInstance
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return summary, moduleReports
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
|
||||||
if len(in) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make([]reporting.ValidatorDecisionReport, len(in))
|
|
||||||
for i, d := range in {
|
|
||||||
out[i] = reporting.ValidatorDecisionReport{
|
|
||||||
ValidatorName: d.ValidatorName,
|
|
||||||
ProposalIndex: d.ProposalIndex,
|
|
||||||
Approved: d.Approved,
|
|
||||||
ReasonCode: d.ReasonCode,
|
|
||||||
Message: d.Message,
|
|
||||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
|
||||||
if len(in) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make([]reporting.ValidatorRejectedReport, len(in))
|
|
||||||
for i, d := range in {
|
|
||||||
out[i] = reporting.ValidatorRejectedReport{
|
|
||||||
ValidatorName: d.ValidatorName,
|
|
||||||
ProposalIndex: d.ProposalIndex,
|
|
||||||
ModuleKey: d.ModuleKey,
|
|
||||||
ModuleInstance: d.ModuleInstance,
|
|
||||||
TargetSegmentID: d.TargetSegmentID,
|
|
||||||
OriginalText: d.OriginalText,
|
|
||||||
CorrectedText: d.CorrectedText,
|
|
||||||
ReasonCode: d.ReasonCode,
|
|
||||||
Message: d.Message,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type processFlags struct {
|
type processFlags struct {
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package cli
|
package processreport
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -15,7 +15,12 @@ const (
|
|||||||
correctionDispositionFailed = "failed"
|
correctionDispositionFailed = "failed"
|
||||||
)
|
)
|
||||||
|
|
||||||
type correctionLedgerEntry struct {
|
type CorrectionLedgerInput struct {
|
||||||
|
RunDirectoryPath string
|
||||||
|
RunOutput *runner.RunOutput
|
||||||
|
}
|
||||||
|
|
||||||
|
type CorrectionLedgerEntry struct {
|
||||||
RunID string `json:"run_id,omitempty"`
|
RunID string `json:"run_id,omitempty"`
|
||||||
ModuleKey string `json:"module_key"`
|
ModuleKey string `json:"module_key"`
|
||||||
ModuleInstance string `json:"module_instance"`
|
ModuleInstance string `json:"module_instance"`
|
||||||
@@ -28,27 +33,28 @@ type correctionLedgerEntry struct {
|
|||||||
Disposition string `json:"disposition"`
|
Disposition string `json:"disposition"`
|
||||||
DispositionReasonCode string `json:"disposition_reason_code,omitempty"`
|
DispositionReasonCode string `json:"disposition_reason_code,omitempty"`
|
||||||
DispositionMessage string `json:"disposition_message,omitempty"`
|
DispositionMessage string `json:"disposition_message,omitempty"`
|
||||||
DeterministicValidatorResults []ledgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
|
DeterministicValidatorResults []LedgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
|
||||||
LLMValidatorResults []ledgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ledgerValidatorDecisionRecord struct {
|
type LedgerValidatorDecisionRecord struct {
|
||||||
ValidatorKey string `json:"validator_key"`
|
ValidatorKey string `json:"validator_key"`
|
||||||
Approved bool `json:"approved"`
|
Approved bool `json:"approved"`
|
||||||
ReasonCode string `json:"reason_code"`
|
ReasonCode string `json:"reason_code"`
|
||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []correctionLedgerEntry {
|
func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry {
|
||||||
|
runOutput := input.RunOutput
|
||||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
runID := ""
|
runID := ""
|
||||||
if runDirPath != "" {
|
if input.RunDirectoryPath != "" {
|
||||||
runID = filepath.Base(runDirPath)
|
runID = filepath.Base(input.RunDirectoryPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
entries := make([]correctionLedgerEntry, 0)
|
entries := make([]CorrectionLedgerEntry, 0)
|
||||||
for _, module := range runOutput.ModuleResults {
|
for _, module := range runOutput.ModuleResults {
|
||||||
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
||||||
for _, decision := range module.ValidatorDecisions {
|
for _, decision := range module.ValidatorDecisions {
|
||||||
@@ -56,7 +62,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, change := range module.AppliedChanges {
|
for _, change := range module.AppliedChanges {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -72,7 +78,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, change := range module.SkippedChanges {
|
for _, change := range module.SkippedChanges {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -89,7 +95,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, rejection := range module.ValidatorRejected {
|
for _, rejection := range module.ValidatorRejected {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -106,7 +112,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
if module.Status == runner.ModuleStatusFailed {
|
if module.Status == runner.ModuleStatusFailed {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -130,17 +136,29 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
return entries
|
return entries
|
||||||
}
|
}
|
||||||
|
|
||||||
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord {
|
func HasSkippedCorrections(runOutput *runner.RunOutput) bool {
|
||||||
|
if runOutput == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, mr := range runOutput.ModuleResults {
|
||||||
|
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []LedgerValidatorDecisionRecord {
|
||||||
if len(in) == 0 {
|
if len(in) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
|
out := make([]LedgerValidatorDecisionRecord, 0, len(in))
|
||||||
for _, decision := range in {
|
for _, decision := range in {
|
||||||
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
|
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
|
||||||
if isLLMBacked != wantLLM {
|
if isLLMBacked != wantLLM {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, ledgerValidatorDecisionRecord{
|
out = append(out, LedgerValidatorDecisionRecord{
|
||||||
ValidatorKey: decision.ValidatorName,
|
ValidatorKey: decision.ValidatorName,
|
||||||
Approved: decision.Approved,
|
Approved: decision.Approved,
|
||||||
ReasonCode: decision.ReasonCode,
|
ReasonCode: decision.ReasonCode,
|
||||||
128
internal/framework/processreport/correction_ledger_test.go
Normal file
128
internal/framework/processreport/correction_ledger_test.go
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
package processreport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
||||||
|
output := &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "glossary",
|
||||||
|
ModuleInstance: "glossary",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
||||||
|
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||||
|
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||||
|
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||||
|
},
|
||||||
|
AppliedChanges: []proposals.AppliedChange{
|
||||||
|
{
|
||||||
|
ProposalIndex: 3,
|
||||||
|
ModuleKey: "glossary",
|
||||||
|
ModuleInstance: "glossary",
|
||||||
|
TargetSegmentID: 1,
|
||||||
|
OriginalText: "gestures",
|
||||||
|
CorrectedText: "Jesters",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ledger := BuildCorrectionLedger(CorrectionLedgerInput{
|
||||||
|
RunDirectoryPath: "/tmp/audita-run-id",
|
||||||
|
RunOutput: output,
|
||||||
|
})
|
||||||
|
if len(ledger) != 1 {
|
||||||
|
t.Fatalf("expected one ledger entry, got %d", len(ledger))
|
||||||
|
}
|
||||||
|
entry := ledger[0]
|
||||||
|
if entry.RunID != "audita-run-id" || entry.Disposition != "applied" || entry.AppliedCorrectedText != "Jesters" {
|
||||||
|
t.Fatalf("unexpected applied ledger entry: %+v", entry)
|
||||||
|
}
|
||||||
|
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
|
||||||
|
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
||||||
|
}
|
||||||
|
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
|
||||||
|
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildCorrectionLedgerPreservesDispositionPolicy(t *testing.T) {
|
||||||
|
output := &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
Status: runner.ModuleStatusSuccess,
|
||||||
|
SkippedChanges: []proposals.SkippedChange{
|
||||||
|
{
|
||||||
|
ProposalIndex: 2,
|
||||||
|
TargetSegmentID: 7,
|
||||||
|
OriginalText: "old",
|
||||||
|
CorrectedText: "new",
|
||||||
|
SkipReason: proposals.SkipReasonAmbiguousOriginal,
|
||||||
|
Message: "ambiguous",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||||
|
{
|
||||||
|
ProposalIndex: 3,
|
||||||
|
TargetSegmentID: 8,
|
||||||
|
OriginalText: "before",
|
||||||
|
CorrectedText: "after",
|
||||||
|
ReasonCode: "protected_term",
|
||||||
|
Message: "blocked",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ModuleKey: "capitalization",
|
||||||
|
ModuleInstance: "capitalization",
|
||||||
|
Status: runner.ModuleStatusFailed,
|
||||||
|
ErrorMessage: "failed",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||||
|
if len(ledger) != 3 {
|
||||||
|
t.Fatalf("expected skipped, rejected, and failed entries, got %+v", ledger)
|
||||||
|
}
|
||||||
|
|
||||||
|
byDisposition := make(map[string]CorrectionLedgerEntry)
|
||||||
|
for _, entry := range ledger {
|
||||||
|
byDisposition[entry.Disposition] = entry
|
||||||
|
}
|
||||||
|
if byDisposition["skipped"].DispositionReasonCode != string(proposals.SkipReasonAmbiguousOriginal) ||
|
||||||
|
byDisposition["skipped"].DispositionMessage != "ambiguous" {
|
||||||
|
t.Fatalf("unexpected skipped ledger entry: %+v", byDisposition["skipped"])
|
||||||
|
}
|
||||||
|
if byDisposition["rejected"].DispositionReasonCode != "protected_term" ||
|
||||||
|
byDisposition["rejected"].ProposedCorrectedText != "after" {
|
||||||
|
t.Fatalf("unexpected rejected ledger entry: %+v", byDisposition["rejected"])
|
||||||
|
}
|
||||||
|
if byDisposition["failed"].DispositionReasonCode != "module_failed" ||
|
||||||
|
byDisposition["failed"].DispositionMessage != "failed" {
|
||||||
|
t.Fatalf("unexpected failed ledger entry: %+v", byDisposition["failed"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHasSkippedCorrectionsIncludesApplicationSkipsAndValidatorRejections(t *testing.T) {
|
||||||
|
if HasSkippedCorrections(nil) {
|
||||||
|
t.Fatal("nil output should not have skipped corrections")
|
||||||
|
}
|
||||||
|
if HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 1}}}}}) {
|
||||||
|
t.Fatal("applied-only output should not have skipped corrections")
|
||||||
|
}
|
||||||
|
if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{SkippedChanges: []proposals.SkippedChange{{ProposalIndex: 1}}}}}) {
|
||||||
|
t.Fatal("application skips should count as skipped corrections")
|
||||||
|
}
|
||||||
|
if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 1}}}}}) {
|
||||||
|
t.Fatal("validator rejections should count as skipped corrections")
|
||||||
|
}
|
||||||
|
}
|
||||||
158
internal/framework/processreport/report.go
Normal file
158
internal/framework/processreport/report.go
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
package processreport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BuildInput contains already-computed process execution facts for report assembly.
|
||||||
|
type BuildInput struct {
|
||||||
|
Status string
|
||||||
|
TranscriptPath string
|
||||||
|
GlossaryPath string
|
||||||
|
OutputPath string
|
||||||
|
Modules []string
|
||||||
|
OutputSchema string
|
||||||
|
ConfigVersion *int
|
||||||
|
StartedAt time.Time
|
||||||
|
CompletedAt time.Time
|
||||||
|
ErrorMessage string
|
||||||
|
ErrorPhase string
|
||||||
|
RunDirectoryPath string
|
||||||
|
NormalizationSummary *normalization.NormalizationSummary
|
||||||
|
ChunkingSummary *chunking.Summary
|
||||||
|
RunOutput *runner.RunOutput
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build creates the public process report without owning command parsing or config loading.
|
||||||
|
func Build(input BuildInput) reporting.ProcessReport {
|
||||||
|
report := reporting.ProcessReport{
|
||||||
|
ReportMetadata: reporting.ReportMetadata{
|
||||||
|
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||||
|
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||||
|
OutputSchema: input.OutputSchema,
|
||||||
|
ConfigVersion: input.ConfigVersion,
|
||||||
|
},
|
||||||
|
Phase: "default_pipeline",
|
||||||
|
Status: input.Status,
|
||||||
|
Operation: "process",
|
||||||
|
TranscriptPath: input.TranscriptPath,
|
||||||
|
GlossaryPath: input.GlossaryPath,
|
||||||
|
OutputPath: input.OutputPath,
|
||||||
|
Modules: append([]string(nil), input.Modules...),
|
||||||
|
StartedAt: input.StartedAt,
|
||||||
|
CompletedAt: &input.CompletedAt,
|
||||||
|
ErrorPhase: input.ErrorPhase,
|
||||||
|
}
|
||||||
|
if input.RunDirectoryPath != "" {
|
||||||
|
runSucceeded := input.Status == "success"
|
||||||
|
metadata := diagnostics.BuildDiagnosticsMetadata(input.RunDirectoryPath, runSucceeded)
|
||||||
|
report.Diagnostics = &metadata
|
||||||
|
}
|
||||||
|
if input.ErrorMessage != "" {
|
||||||
|
report.ErrorMessage = input.ErrorMessage
|
||||||
|
}
|
||||||
|
if input.NormalizationSummary != nil {
|
||||||
|
report.InputSegmentCount = &input.NormalizationSummary.InputSegmentCount
|
||||||
|
report.NormalizedSegmentCount = &input.NormalizationSummary.OutputSegmentCount
|
||||||
|
report.NormalizationMerges = &input.NormalizationSummary.MergesPerformed
|
||||||
|
report.NormalizationIDReassignments = &input.NormalizationSummary.IDsReassigned
|
||||||
|
report.NormalizationSkipped.DifferentSpeakers = &input.NormalizationSummary.SkippedMerges.DifferentSpeakers
|
||||||
|
report.NormalizationSkipped.GapTooLarge = &input.NormalizationSummary.SkippedMerges.GapTooLarge
|
||||||
|
report.NormalizationSkipped.DurationExceeded = &input.NormalizationSummary.SkippedMerges.DurationExceeded
|
||||||
|
report.NormalizationSkipped.TokenLimitExceeded = &input.NormalizationSummary.SkippedMerges.TokenLimitExceeded
|
||||||
|
}
|
||||||
|
if input.ChunkingSummary != nil {
|
||||||
|
report.Chunking = &reporting.ChunkingSummary{
|
||||||
|
ChunkCount: input.ChunkingSummary.ChunkCount,
|
||||||
|
MinEstimatedTokens: input.ChunkingSummary.MinEstimatedTokens,
|
||||||
|
MaxEstimatedTokens: input.ChunkingSummary.MaxEstimatedTokens,
|
||||||
|
TotalEstimatedTokens: input.ChunkingSummary.TotalEstimatedTokens,
|
||||||
|
TargetSections: input.ChunkingSummary.TargetSections,
|
||||||
|
MaxSectionTokens: input.ChunkingSummary.MaxSectionTokens,
|
||||||
|
MinSectionTokens: input.ChunkingSummary.MinSectionTokens,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.ModulesSummary, report.ModuleResults = buildModuleReporting(input.RunOutput)
|
||||||
|
return report
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
||||||
|
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
||||||
|
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
||||||
|
for _, r := range runOutput.ModuleResults {
|
||||||
|
startedAt := r.StartedAt
|
||||||
|
completedAt := r.CompletedAt
|
||||||
|
moduleReports = append(moduleReports, reporting.ModuleReport{
|
||||||
|
ModuleKey: r.ModuleKey,
|
||||||
|
ModuleInstance: r.ModuleInstance,
|
||||||
|
ReplacementPolicy: string(r.ReplacementPolicy),
|
||||||
|
Status: r.Status,
|
||||||
|
ProposalCount: r.ProposalCount,
|
||||||
|
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
||||||
|
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
||||||
|
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
||||||
|
AppliedChanges: r.AppliedChanges,
|
||||||
|
SkippedChanges: r.SkippedChanges,
|
||||||
|
ErrorMessage: r.ErrorMessage,
|
||||||
|
StartedAt: &startedAt,
|
||||||
|
CompletedAt: &completedAt,
|
||||||
|
})
|
||||||
|
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
||||||
|
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
||||||
|
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
||||||
|
summary.FailedModuleInstance = r.ModuleInstance
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return summary, moduleReports
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
||||||
|
if len(in) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]reporting.ValidatorDecisionReport, len(in))
|
||||||
|
for i, d := range in {
|
||||||
|
out[i] = reporting.ValidatorDecisionReport{
|
||||||
|
ValidatorName: d.ValidatorName,
|
||||||
|
ProposalIndex: d.ProposalIndex,
|
||||||
|
Approved: d.Approved,
|
||||||
|
ReasonCode: d.ReasonCode,
|
||||||
|
Message: d.Message,
|
||||||
|
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
||||||
|
if len(in) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]reporting.ValidatorRejectedReport, len(in))
|
||||||
|
for i, d := range in {
|
||||||
|
out[i] = reporting.ValidatorRejectedReport{
|
||||||
|
ValidatorName: d.ValidatorName,
|
||||||
|
ProposalIndex: d.ProposalIndex,
|
||||||
|
ModuleKey: d.ModuleKey,
|
||||||
|
ModuleInstance: d.ModuleInstance,
|
||||||
|
TargetSegmentID: d.TargetSegmentID,
|
||||||
|
OriginalText: d.OriginalText,
|
||||||
|
CorrectedText: d.CorrectedText,
|
||||||
|
ReasonCode: d.ReasonCode,
|
||||||
|
Message: d.Message,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
180
internal/framework/processreport/report_test.go
Normal file
180
internal/framework/processreport/report_test.go
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
package processreport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildSuccessReportMapsExecutionFacts(t *testing.T) {
|
||||||
|
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||||
|
completedAt := startedAt.Add(time.Second)
|
||||||
|
configVersion := 4
|
||||||
|
targetSections := 2
|
||||||
|
inputSegments := 5
|
||||||
|
outputSegments := 4
|
||||||
|
merges := 1
|
||||||
|
reassigned := 2
|
||||||
|
differentSpeakers := 3
|
||||||
|
|
||||||
|
report := Build(BuildInput{
|
||||||
|
Status: "success",
|
||||||
|
TranscriptPath: "transcript.json",
|
||||||
|
GlossaryPath: "glossary.yaml",
|
||||||
|
OutputPath: "out.json",
|
||||||
|
Modules: []string{"grammar"},
|
||||||
|
OutputSchema: "default",
|
||||||
|
ConfigVersion: &configVersion,
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
RunDirectoryPath: filepath.Join(
|
||||||
|
"tmp",
|
||||||
|
"audita-run",
|
||||||
|
),
|
||||||
|
NormalizationSummary: &normalization.NormalizationSummary{
|
||||||
|
InputSegmentCount: inputSegments,
|
||||||
|
OutputSegmentCount: outputSegments,
|
||||||
|
MergesPerformed: merges,
|
||||||
|
IDsReassigned: reassigned,
|
||||||
|
SkippedMerges: struct {
|
||||||
|
DifferentSpeakers int `json:"different_speakers"`
|
||||||
|
GapTooLarge int `json:"gap_too_large"`
|
||||||
|
DurationExceeded int `json:"duration_exceeded"`
|
||||||
|
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
||||||
|
}{
|
||||||
|
DifferentSpeakers: differentSpeakers,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ChunkingSummary: &chunking.Summary{
|
||||||
|
ChunkCount: 3,
|
||||||
|
MinEstimatedTokens: 10,
|
||||||
|
MaxEstimatedTokens: 20,
|
||||||
|
TotalEstimatedTokens: 45,
|
||||||
|
TargetSections: &targetSections,
|
||||||
|
MaxSectionTokens: 200,
|
||||||
|
MinSectionTokens: 50,
|
||||||
|
},
|
||||||
|
RunOutput: &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
Status: runner.ModuleStatusSuccess,
|
||||||
|
ProposalCount: 2,
|
||||||
|
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||||
|
{
|
||||||
|
ValidatorName: "proposal_shape",
|
||||||
|
ProposalIndex: 1,
|
||||||
|
Approved: true,
|
||||||
|
ReasonCode: "approved",
|
||||||
|
Message: "ok",
|
||||||
|
DiagnosticArtifactPath: "diagnostics/validator.json",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||||
|
{
|
||||||
|
ValidatorName: "protected_term",
|
||||||
|
ProposalIndex: 2,
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
TargetSegmentID: 7,
|
||||||
|
OriginalText: "old",
|
||||||
|
CorrectedText: "new",
|
||||||
|
ReasonCode: "protected_term",
|
||||||
|
Message: "blocked",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
AppliedChanges: []proposals.AppliedChange{
|
||||||
|
{ProposalIndex: 1, TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"},
|
||||||
|
},
|
||||||
|
SkippedChanges: []proposals.SkippedChange{
|
||||||
|
{ProposalIndex: 3, TargetSegmentID: 8, SkipReason: proposals.SkipReasonMissingSegment},
|
||||||
|
},
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if report.ReportMetadata.ReportSchemaName != reporting.DefaultProcessReportSchemaName ||
|
||||||
|
report.ReportMetadata.ReportSchemaVersion != reporting.DefaultProcessReportSchemaVersion ||
|
||||||
|
report.ReportMetadata.OutputSchema != "default" ||
|
||||||
|
report.ReportMetadata.ConfigVersion == nil ||
|
||||||
|
*report.ReportMetadata.ConfigVersion != configVersion {
|
||||||
|
t.Fatalf("unexpected report metadata: %+v", report.ReportMetadata)
|
||||||
|
}
|
||||||
|
if report.Phase != "default_pipeline" || report.Operation != "process" || report.Status != "success" {
|
||||||
|
t.Fatalf("unexpected process identity fields: phase=%q operation=%q status=%q", report.Phase, report.Operation, report.Status)
|
||||||
|
}
|
||||||
|
if report.Diagnostics == nil || report.Diagnostics.CorrectionLedgerPath != filepath.Join("tmp", "audita-run", diagnostics.ArtifactCorrectionLedger) {
|
||||||
|
t.Fatalf("unexpected diagnostics metadata: %+v", report.Diagnostics)
|
||||||
|
}
|
||||||
|
if report.InputSegmentCount == nil || *report.InputSegmentCount != inputSegments ||
|
||||||
|
report.NormalizationSkipped.DifferentSpeakers == nil ||
|
||||||
|
*report.NormalizationSkipped.DifferentSpeakers != differentSpeakers {
|
||||||
|
t.Fatalf("unexpected normalization summary: %+v", report)
|
||||||
|
}
|
||||||
|
if report.Chunking == nil || report.Chunking.ChunkCount != 3 || report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != targetSections {
|
||||||
|
t.Fatalf("unexpected chunking summary: %+v", report.Chunking)
|
||||||
|
}
|
||||||
|
if report.ModulesSummary == nil ||
|
||||||
|
report.ModulesSummary.ModuleCount != 1 ||
|
||||||
|
report.ModulesSummary.TotalAppliedChanges != 1 ||
|
||||||
|
report.ModulesSummary.TotalSkippedChanges != 2 {
|
||||||
|
t.Fatalf("unexpected modules summary: %+v", report.ModulesSummary)
|
||||||
|
}
|
||||||
|
if len(report.ModuleResults) != 1 ||
|
||||||
|
len(report.ModuleResults[0].ValidatorDecisions) != 1 ||
|
||||||
|
len(report.ModuleResults[0].ValidatorRejected) != 1 {
|
||||||
|
t.Fatalf("unexpected module reports: %+v", report.ModuleResults)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildFailedReportPreservesErrorAndFailureSummary(t *testing.T) {
|
||||||
|
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||||
|
completedAt := startedAt.Add(time.Second)
|
||||||
|
|
||||||
|
report := Build(BuildInput{
|
||||||
|
Status: "failed",
|
||||||
|
TranscriptPath: "transcript.json",
|
||||||
|
GlossaryPath: "glossary.yaml",
|
||||||
|
Modules: []string{"grammar"},
|
||||||
|
OutputSchema: "default",
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
ErrorPhase: "module",
|
||||||
|
ErrorMessage: "module failed",
|
||||||
|
RunDirectoryPath: "run-dir",
|
||||||
|
RunOutput: &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
Status: runner.ModuleStatusFailed,
|
||||||
|
ErrorMessage: "module failed",
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if report.Status != "failed" || report.ErrorPhase != "module" || report.ErrorMessage != "module failed" {
|
||||||
|
t.Fatalf("unexpected failure fields: %+v", report)
|
||||||
|
}
|
||||||
|
if report.Diagnostics == nil || report.Diagnostics.ErrorLogPath == "" {
|
||||||
|
t.Fatalf("expected failure diagnostics metadata, got %+v", report.Diagnostics)
|
||||||
|
}
|
||||||
|
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "grammar" {
|
||||||
|
t.Fatalf("unexpected failed module summary: %+v", report.ModulesSummary)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user