Compare commits

...

2 Commits

8 changed files with 1086 additions and 268 deletions

View File

@@ -0,0 +1,121 @@
package cli
import (
"flag"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
)
type processOverrideBinding func(*config.CLIOverrides, processFlags)
var processOverrideBindings = map[string]processOverrideBinding{
"modules": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ModulesCSV = flags.modules
},
"output-schema": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.OutputSchema = flags.outputSchema
},
"llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryLLMAPIKey = flags.llmAPIKey
},
"validation-llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationLLMAPIKey = flags.validationLLMAPIKey
},
"model": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryModel = flags.model
},
"validation-model": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationModel = flags.validationModel
},
"base-url": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryBaseURL = flags.baseURL
},
"validation-base-url": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationBaseURL = flags.validationBaseURL
},
"llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryLLMTimeoutSeconds = flags.llmTimeoutSeconds
},
"total-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.TotalLLMConcurrency = flags.totalLLMConcurrency
},
"proposal-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ProposalLLMConcurrency = flags.proposalLLMConcurrency
},
"llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryLLMConcurrency = flags.llmConcurrency
},
"validation-llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationLLMTimeoutSeconds = flags.validationLLMTimeoutSeconds
},
"max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.MaxRetries = flags.maxRetries
},
"validation-max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationMaxRetries = flags.validationMaxRetries
},
"validation-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationLLMConcurrency = flags.validationLLMConcurrency
},
"validation-max-prompt-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationMaxPromptTokens = flags.validationMaxPromptTokens
},
"max-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.MaxSectionTokens = flags.maxSectionTokens
},
"min-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.MinSectionTokens = flags.minSectionTokens
},
"target-sections": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.TargetSections = flags.targetSections
},
"glossary-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.GlossaryConfidenceThreshold = flags.glossaryConfidenceThreshold
},
"grammar-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.GrammarConfidenceThreshold = flags.grammarConfidenceThreshold
},
"homophones-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.HomophonesConfidenceThreshold = flags.homophonesConfidenceThreshold
},
"spoken-word-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.SpokenWordConfidenceThreshold = flags.spokenWordConfidenceThreshold
},
"normalize-max-segment-gap": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeMaxSegmentGap = flags.normalizeMaxSegmentGap
},
"normalize-ellipsis-gap": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeEllipsisGap = flags.normalizeEllipsisGap
},
"normalize-max-segment-duration": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeMaxSegmentDuration = flags.normalizeMaxSegmentDuration
},
"normalize-max-segment-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeMaxSegmentTokens = flags.normalizeMaxSegmentTokens
},
"transcript-description": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.TranscriptDescription = flags.transcriptDescription
},
"work-dir": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.WorkDir = flags.workDir
},
"work-dir-retention": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.WorkDirRetention = flags.workDirRetention
},
}
func processCLIOverrides(fs *flag.FlagSet, flags processFlags) (config.CLIOverrides, bool) {
overrides := config.CLIOverrides{}
explicitModules := false
fs.Visit(func(f *flag.Flag) {
if f.Name == "modules" {
explicitModules = true
}
binding, ok := processOverrideBindings[f.Name]
if !ok {
return
}
binding(&overrides, flags)
})
return overrides, explicitModules
}

View File

@@ -0,0 +1,433 @@
package cli
import (
"io"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
)
func TestProcessCLIOverridesMapsEveryConfigMutatingFlag(t *testing.T) {
tests := []struct {
name string
flagName string
value string
wantExplicitModules bool
assertOverrideFields func(t *testing.T, overrides config.CLIOverrides)
}{
{
name: "modules",
flagName: "modules",
value: "grammar,glossary",
wantExplicitModules: true,
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ModulesCSV", overrides.ModulesCSV, "grammar,glossary")
},
},
{
name: "output schema",
flagName: "output-schema",
value: "audita-v1",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "OutputSchema", overrides.OutputSchema, "audita-v1")
},
},
{
name: "primary api key",
flagName: "llm-api-key",
value: "primary-key",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "PrimaryLLMAPIKey", overrides.PrimaryLLMAPIKey, "primary-key")
},
},
{
name: "validation api key",
flagName: "validation-llm-api-key",
value: "validation-key",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ValidationLLMAPIKey", overrides.ValidationLLMAPIKey, "validation-key")
},
},
{
name: "primary model",
flagName: "model",
value: "primary-model",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "PrimaryModel", overrides.PrimaryModel, "primary-model")
},
},
{
name: "validation model",
flagName: "validation-model",
value: "validation-model",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ValidationModel", overrides.ValidationModel, "validation-model")
},
},
{
name: "primary base url",
flagName: "base-url",
value: "https://primary.example.test",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "PrimaryBaseURL", overrides.PrimaryBaseURL, "https://primary.example.test")
},
},
{
name: "validation base url",
flagName: "validation-base-url",
value: "https://validation.example.test",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ValidationBaseURL", overrides.ValidationBaseURL, "https://validation.example.test")
},
},
{
name: "primary timeout",
flagName: "llm-timeout-seconds",
value: "101",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "PrimaryLLMTimeoutSeconds", overrides.PrimaryLLMTimeoutSeconds, 101)
},
},
{
name: "total concurrency",
flagName: "total-llm-concurrency",
value: "5",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "TotalLLMConcurrency", overrides.TotalLLMConcurrency, 5)
},
},
{
name: "proposal concurrency",
flagName: "proposal-llm-concurrency",
value: "3",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ProposalLLMConcurrency", overrides.ProposalLLMConcurrency, 3)
},
},
{
name: "legacy concurrency alias",
flagName: "llm-concurrency",
value: "4",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "PrimaryLLMConcurrency", overrides.PrimaryLLMConcurrency, 4)
},
},
{
name: "validation timeout",
flagName: "validation-llm-timeout-seconds",
value: "202",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationLLMTimeoutSeconds", overrides.ValidationLLMTimeoutSeconds, 202)
},
},
{
name: "max retries",
flagName: "max-retries",
value: "6",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "MaxRetries", overrides.MaxRetries, 6)
},
},
{
name: "validation max retries",
flagName: "validation-max-retries",
value: "7",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationMaxRetries", overrides.ValidationMaxRetries, 7)
},
},
{
name: "validation concurrency",
flagName: "validation-llm-concurrency",
value: "8",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationLLMConcurrency", overrides.ValidationLLMConcurrency, 8)
},
},
{
name: "validation max prompt tokens",
flagName: "validation-max-prompt-tokens",
value: "4096",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationMaxPromptTokens", overrides.ValidationMaxPromptTokens, 4096)
},
},
{
name: "max section tokens",
flagName: "max-section-tokens",
value: "9000",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "MaxSectionTokens", overrides.MaxSectionTokens, 9000)
},
},
{
name: "min section tokens",
flagName: "min-section-tokens",
value: "1000",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "MinSectionTokens", overrides.MinSectionTokens, 1000)
},
},
{
name: "target sections",
flagName: "target-sections",
value: "12",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "TargetSections", overrides.TargetSections, 12)
},
},
{
name: "glossary threshold",
flagName: "glossary-confidence-threshold",
value: "0.91",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "GlossaryConfidenceThreshold", overrides.GlossaryConfidenceThreshold, 0.91)
},
},
{
name: "grammar threshold",
flagName: "grammar-confidence-threshold",
value: "0.92",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "GrammarConfidenceThreshold", overrides.GrammarConfidenceThreshold, 0.92)
},
},
{
name: "homophones threshold",
flagName: "homophones-confidence-threshold",
value: "0.93",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "HomophonesConfidenceThreshold", overrides.HomophonesConfidenceThreshold, 0.93)
},
},
{
name: "spoken word threshold",
flagName: "spoken-word-confidence-threshold",
value: "0.94",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "SpokenWordConfidenceThreshold", overrides.SpokenWordConfidenceThreshold, 0.94)
},
},
{
name: "normalize max segment gap",
flagName: "normalize-max-segment-gap",
value: "1.2",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "NormalizeMaxSegmentGap", overrides.NormalizeMaxSegmentGap, 1.2)
},
},
{
name: "normalize ellipsis gap",
flagName: "normalize-ellipsis-gap",
value: "2.3",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "NormalizeEllipsisGap", overrides.NormalizeEllipsisGap, 2.3)
},
},
{
name: "normalize max segment duration",
flagName: "normalize-max-segment-duration",
value: "45.6",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "NormalizeMaxSegmentDuration", overrides.NormalizeMaxSegmentDuration, 45.6)
},
},
{
name: "normalize max segment tokens",
flagName: "normalize-max-segment-tokens",
value: "321",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "NormalizeMaxSegmentTokens", overrides.NormalizeMaxSegmentTokens, 321)
},
},
{
name: "transcript description",
flagName: "transcript-description",
value: "podcast episode",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "TranscriptDescription", overrides.TranscriptDescription, "podcast episode")
},
},
{
name: "work dir",
flagName: "work-dir",
value: "/tmp/custom-audita",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "WorkDir", overrides.WorkDir, "/tmp/custom-audita")
},
},
{
name: "work dir retention",
flagName: "work-dir-retention",
value: "always",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "WorkDirRetention", overrides.WorkDirRetention, "always")
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
if err := fs.Parse([]string{"--" + tc.flagName, tc.value}); err != nil {
t.Fatalf("parse flag: %v", err)
}
overrides, explicitModules := processCLIOverrides(fs, flags)
if explicitModules != tc.wantExplicitModules {
t.Fatalf("explicitModules=%v, want %v", explicitModules, tc.wantExplicitModules)
}
tc.assertOverrideFields(t, overrides)
})
}
}
func TestProcessCLIOverridesIgnoresNonConfigFlags(t *testing.T) {
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
if err := fs.Parse([]string{
"--config", "/tmp/config.yml",
"--glossary", "/tmp/glossary.yml",
"--output", "/tmp/output.json",
"--report-json", "/tmp/report.json",
}); err != nil {
t.Fatalf("parse flags: %v", err)
}
overrides, explicitModules := processCLIOverrides(fs, flags)
if explicitModules {
t.Fatal("non-config flags should not mark modules explicit")
}
assertNoCLIOverrides(t, overrides)
}
func TestNewProcessFlagSetDefaultsReflectEffectiveConfig(t *testing.T) {
cfg := config.Default()
cfg.Modules = []string{"grammar", "glossary"}
cfg.OutputSchema = "audita-v1"
cfg.PrimaryLLM.APIKey = "primary-key"
cfg.ValidationLLM.APIKey = "validation-key"
cfg.PrimaryLLM.Model = "primary-model"
cfg.ValidationLLM.Model = "validation-model"
cfg.PrimaryLLM.BaseURL = "https://primary.example.test"
cfg.ValidationLLM.BaseURL = "https://validation.example.test"
cfg.PrimaryLLM.TimeoutSeconds = 101
cfg.TotalLLMConcurrency = 5
cfg.ProposalLLMConcurrency = 3
cfg.PrimaryLLM.MaxRetries = 6
cfg.ValidationMaxPromptTokens = 4096
cfg.MaxSectionTokens = 9000
cfg.MinSectionTokens = 1000
cfg.Thresholds.Glossary = 0.91
cfg.Thresholds.Grammar = 0.92
cfg.Thresholds.Homophones = 0.93
cfg.Thresholds.SpokenWord = 0.94
cfg.Normalization.MaxSegmentGap = 1.2
cfg.Normalization.EllipsisGap = 2.3
cfg.Normalization.MaxSegmentDuration = 45.6
cfg.Normalization.MaxSegmentTokens = 321
cfg.TranscriptDescription = "podcast episode"
cfg.WorkDir = "/tmp/custom-audita"
cfg.WorkDirRetention = config.WorkDirRetentionAlways
validationTimeout := 202
validationRetries := 7
validationConcurrency := 8
targetSections := 12
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
cfg.ValidationLLM.MaxRetries = &validationRetries
cfg.ValidationLLMConcurrency = &validationConcurrency
cfg.TargetSections = &targetSections
_, flags := newProcessFlagSet(cfg, io.Discard)
assertStringOverride(t, "modules default", flags.modules, "grammar,glossary")
assertStringOverride(t, "output schema default", flags.outputSchema, "audita-v1")
assertStringOverride(t, "primary api key default", flags.llmAPIKey, "primary-key")
assertStringOverride(t, "validation api key default", flags.validationLLMAPIKey, "validation-key")
assertStringOverride(t, "primary model default", flags.model, "primary-model")
assertStringOverride(t, "validation model default", flags.validationModel, "validation-model")
assertStringOverride(t, "primary base url default", flags.baseURL, "https://primary.example.test")
assertStringOverride(t, "validation base url default", flags.validationBaseURL, "https://validation.example.test")
assertIntOverride(t, "primary timeout default", flags.llmTimeoutSeconds, 101)
assertIntOverride(t, "total concurrency default", flags.totalLLMConcurrency, 5)
assertIntOverride(t, "proposal concurrency default", flags.proposalLLMConcurrency, 3)
assertIntOverride(t, "legacy concurrency alias default", flags.llmConcurrency, 5)
assertIntOverride(t, "validation timeout default", flags.validationLLMTimeoutSeconds, validationTimeout)
assertIntOverride(t, "max retries default", flags.maxRetries, 6)
assertIntOverride(t, "validation max retries default", flags.validationMaxRetries, validationRetries)
assertIntOverride(t, "validation concurrency default", flags.validationLLMConcurrency, validationConcurrency)
assertIntOverride(t, "validation max prompt tokens default", flags.validationMaxPromptTokens, 4096)
assertIntOverride(t, "max section tokens default", flags.maxSectionTokens, 9000)
assertIntOverride(t, "min section tokens default", flags.minSectionTokens, 1000)
assertIntOverride(t, "target sections default", flags.targetSections, targetSections)
assertFloatOverride(t, "glossary threshold default", flags.glossaryConfidenceThreshold, 0.91)
assertFloatOverride(t, "grammar threshold default", flags.grammarConfidenceThreshold, 0.92)
assertFloatOverride(t, "homophones threshold default", flags.homophonesConfidenceThreshold, 0.93)
assertFloatOverride(t, "spoken word threshold default", flags.spokenWordConfidenceThreshold, 0.94)
assertFloatOverride(t, "normalize max segment gap default", flags.normalizeMaxSegmentGap, 1.2)
assertFloatOverride(t, "normalize ellipsis gap default", flags.normalizeEllipsisGap, 2.3)
assertFloatOverride(t, "normalize max segment duration default", flags.normalizeMaxSegmentDuration, 45.6)
assertIntOverride(t, "normalize max segment tokens default", flags.normalizeMaxSegmentTokens, 321)
assertStringOverride(t, "transcript description default", flags.transcriptDescription, "podcast episode")
assertStringOverride(t, "work dir default", flags.workDir, "/tmp/custom-audita")
assertStringOverride(t, "work dir retention default", flags.workDirRetention, "always")
}
func TestNewProcessFlagSetUsesFallbackDefaultsForUnsetOptionalConfig(t *testing.T) {
cfg := config.Default()
_, flags := newProcessFlagSet(cfg, io.Discard)
assertIntOverride(t, "validation timeout fallback", flags.validationLLMTimeoutSeconds, cfg.PrimaryLLM.TimeoutSeconds)
assertIntOverride(t, "validation retries fallback", flags.validationMaxRetries, cfg.PrimaryLLM.MaxRetries)
assertIntOverride(t, "validation concurrency fallback", flags.validationLLMConcurrency, cfg.TotalLLMConcurrency)
assertIntOverride(t, "target sections fallback", flags.targetSections, 0)
}
func assertStringOverride(t *testing.T, name string, got *string, want string) {
t.Helper()
if got == nil || *got != want {
t.Fatalf("%s=%v, want %q", name, pointerValue(got), want)
}
}
func assertIntOverride(t *testing.T, name string, got *int, want int) {
t.Helper()
if got == nil || *got != want {
t.Fatalf("%s=%v, want %d", name, pointerValue(got), want)
}
}
func assertFloatOverride(t *testing.T, name string, got *float64, want float64) {
t.Helper()
if got == nil || *got != want {
t.Fatalf("%s=%v, want %v", name, pointerValue(got), want)
}
}
func assertNoCLIOverrides(t *testing.T, overrides config.CLIOverrides) {
t.Helper()
value := reflect.ValueOf(overrides)
typ := value.Type()
for i := 0; i < value.NumField(); i++ {
field := value.Field(i)
if field.Kind() != reflect.Ptr {
t.Fatalf("unexpected non-pointer CLIOverrides field %s", typ.Field(i).Name)
}
if !field.IsNil() {
t.Fatalf("expected no CLI overrides, field %s was set", typ.Field(i).Name)
}
}
}
func pointerValue[T any](ptr *T) any {
if ptr == nil {
return "<nil>"
}
if stringer, ok := any(*ptr).(interface{ String() string }); ok {
return strings.TrimSpace(stringer.String())
}
return *ptr
}

View File

@@ -1,46 +0,0 @@
package cli
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
)
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
output := &runner.RunOutput{
ModuleResults: []runner.ModuleResult{
{
ModuleKey: "glossary",
ModuleInstance: "glossary",
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
ValidatorDecisions: []runner.ValidatorDecisionRecord{
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
},
AppliedChanges: []proposals.AppliedChange{
{
ProposalIndex: 3,
ModuleKey: "glossary",
ModuleInstance: "glossary",
TargetSegmentID: 1,
OriginalText: "gestures",
CorrectedText: "Jesters",
},
},
},
},
}
ledger := buildCorrectionLedger("/tmp/audita-run-id", output)
if len(ledger) != 1 {
t.Fatalf("expected one ledger entry, got %d", len(ledger))
}
entry := ledger[0]
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
}
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
}
}

View File

@@ -23,10 +23,10 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules" "gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
) )
type noOpStructuredLLMClient struct{} type noOpStructuredLLMClient struct{}
@@ -420,75 +420,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
return 2 return 2
} }
overrides := config.CLIOverrides{} overrides, explicitModules := processCLIOverrides(fs, pFlags)
explicitModules := false
fs.Visit(func(f *flag.Flag) {
switch f.Name {
case "modules":
explicitModules = true
overrides.ModulesCSV = pFlags.modules
case "output-schema":
overrides.OutputSchema = pFlags.outputSchema
case "llm-api-key":
overrides.PrimaryLLMAPIKey = pFlags.llmAPIKey
case "validation-llm-api-key":
overrides.ValidationLLMAPIKey = pFlags.validationLLMAPIKey
case "model":
overrides.PrimaryModel = pFlags.model
case "validation-model":
overrides.ValidationModel = pFlags.validationModel
case "base-url":
overrides.PrimaryBaseURL = pFlags.baseURL
case "validation-base-url":
overrides.ValidationBaseURL = pFlags.validationBaseURL
case "llm-timeout-seconds":
overrides.PrimaryLLMTimeoutSeconds = pFlags.llmTimeoutSeconds
case "total-llm-concurrency":
overrides.TotalLLMConcurrency = pFlags.totalLLMConcurrency
case "proposal-llm-concurrency":
overrides.ProposalLLMConcurrency = pFlags.proposalLLMConcurrency
case "llm-concurrency":
overrides.PrimaryLLMConcurrency = pFlags.llmConcurrency
case "validation-llm-timeout-seconds":
overrides.ValidationLLMTimeoutSeconds = pFlags.validationLLMTimeoutSeconds
case "max-retries":
overrides.MaxRetries = pFlags.maxRetries
case "validation-max-retries":
overrides.ValidationMaxRetries = pFlags.validationMaxRetries
case "validation-llm-concurrency":
overrides.ValidationLLMConcurrency = pFlags.validationLLMConcurrency
case "validation-max-prompt-tokens":
overrides.ValidationMaxPromptTokens = pFlags.validationMaxPromptTokens
case "max-section-tokens":
overrides.MaxSectionTokens = pFlags.maxSectionTokens
case "min-section-tokens":
overrides.MinSectionTokens = pFlags.minSectionTokens
case "target-sections":
overrides.TargetSections = pFlags.targetSections
case "glossary-confidence-threshold":
overrides.GlossaryConfidenceThreshold = pFlags.glossaryConfidenceThreshold
case "grammar-confidence-threshold":
overrides.GrammarConfidenceThreshold = pFlags.grammarConfidenceThreshold
case "homophones-confidence-threshold":
overrides.HomophonesConfidenceThreshold = pFlags.homophonesConfidenceThreshold
case "spoken-word-confidence-threshold":
overrides.SpokenWordConfidenceThreshold = pFlags.spokenWordConfidenceThreshold
case "normalize-max-segment-gap":
overrides.NormalizeMaxSegmentGap = pFlags.normalizeMaxSegmentGap
case "normalize-ellipsis-gap":
overrides.NormalizeEllipsisGap = pFlags.normalizeEllipsisGap
case "normalize-max-segment-duration":
overrides.NormalizeMaxSegmentDuration = pFlags.normalizeMaxSegmentDuration
case "normalize-max-segment-tokens":
overrides.NormalizeMaxSegmentTokens = pFlags.normalizeMaxSegmentTokens
case "transcript-description":
overrides.TranscriptDescription = pFlags.transcriptDescription
case "work-dir":
overrides.WorkDir = pFlags.workDir
case "work-dir-retention":
overrides.WorkDirRetention = pFlags.workDirRetention
}
})
if err := cfg.ApplyCLIOverrides(overrides); err != nil { if err := cfg.ApplyCLIOverrides(overrides); err != nil {
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err) fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
@@ -531,10 +463,13 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
if runOutput.Utilization != nil { if runOutput.Utilization != nil {
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization) _ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
} }
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput)) _ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
RunDirectoryPath: runDir.Path(),
RunOutput: runOutput,
}))
} }
errorPhase, errorMessage := extractErrorPhase(runErr) errorPhase, errorMessage := extractErrorPhase(runErr)
report := buildProcessReport("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput) report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
if strings.TrimSpace(inv.ReportJSONPath) != "" { if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil { if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
@@ -560,10 +495,13 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
if runOutput.Utilization != nil { if runOutput.Utilization != nil {
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization) _ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
} }
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput)) _ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
RunDirectoryPath: runDir.Path(),
RunOutput: runOutput,
}))
} }
report := buildProcessReport("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput) report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
if strings.TrimSpace(inv.ReportJSONPath) != "" { if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil { if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
@@ -578,21 +516,11 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
} }
} }
hasSkippedCorrections := false
if runOutput != nil {
for _, mr := range runOutput.ModuleResults {
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
hasSkippedCorrections = true
break
}
}
}
if runDir != nil { if runDir != nil {
_ = runDir.WriteReport(report) _ = runDir.WriteReport(report)
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{ if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
RunSucceeded: true, RunSucceeded: true,
HasSkippedCorrections: hasSkippedCorrections, HasSkippedCorrections: processreport.HasSkippedCorrections(runOutput),
}); err != nil { }); err != nil {
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err) fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
return 1 return 1
@@ -710,130 +638,28 @@ func extractErrorPhase(err error) (phase string, message string) {
return "", msg return "", msg
} }
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport { func processReportInput(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) processreport.BuildInput {
report := reporting.ProcessReport{ runDirectoryPath := ""
ReportMetadata: reporting.ReportMetadata{
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
OutputSchema: inv.Config.OutputSchema,
ConfigVersion: inv.ConfigVersion,
},
Phase: "default_pipeline",
Status: status,
Operation: "process",
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
Modules: append([]string(nil), inv.Config.Modules...),
StartedAt: startedAt,
CompletedAt: &completedAt,
ErrorPhase: errorPhase,
}
if runDir != nil { if runDir != nil {
runSucceeded := status == "success" runDirectoryPath = runDir.Path()
metadata := diagnostics.BuildDiagnosticsMetadata(runDir.Path(), runSucceeded)
report.Diagnostics = &metadata
} }
if errorMessage != "" { return processreport.BuildInput{
report.ErrorMessage = errorMessage Status: status,
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
Modules: inv.Config.Modules,
OutputSchema: inv.Config.OutputSchema,
ConfigVersion: inv.ConfigVersion,
StartedAt: startedAt,
CompletedAt: completedAt,
ErrorMessage: errorMessage,
ErrorPhase: errorPhase,
RunDirectoryPath: runDirectoryPath,
NormalizationSummary: normalizationSummary,
ChunkingSummary: chunkingSummary,
RunOutput: runOutput,
} }
if normalizationSummary != nil {
report.InputSegmentCount = &normalizationSummary.InputSegmentCount
report.NormalizedSegmentCount = &normalizationSummary.OutputSegmentCount
report.NormalizationMerges = &normalizationSummary.MergesPerformed
report.NormalizationIDReassignments = &normalizationSummary.IDsReassigned
report.NormalizationSkipped.DifferentSpeakers = &normalizationSummary.SkippedMerges.DifferentSpeakers
report.NormalizationSkipped.GapTooLarge = &normalizationSummary.SkippedMerges.GapTooLarge
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
}
if chunkingSummary != nil {
report.Chunking = &reporting.ChunkingSummary{
ChunkCount: chunkingSummary.ChunkCount,
MinEstimatedTokens: chunkingSummary.MinEstimatedTokens,
MaxEstimatedTokens: chunkingSummary.MaxEstimatedTokens,
TotalEstimatedTokens: chunkingSummary.TotalEstimatedTokens,
TargetSections: chunkingSummary.TargetSections,
MaxSectionTokens: chunkingSummary.MaxSectionTokens,
MinSectionTokens: chunkingSummary.MinSectionTokens,
}
}
report.ModulesSummary, report.ModuleResults = buildModuleReporting(runOutput)
return report
}
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
return nil, nil
}
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
for _, r := range runOutput.ModuleResults {
startedAt := r.StartedAt
completedAt := r.CompletedAt
moduleReports = append(moduleReports, reporting.ModuleReport{
ModuleKey: r.ModuleKey,
ModuleInstance: r.ModuleInstance,
ReplacementPolicy: string(r.ReplacementPolicy),
Status: r.Status,
ProposalCount: r.ProposalCount,
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
AppliedChanges: r.AppliedChanges,
SkippedChanges: r.SkippedChanges,
ErrorMessage: r.ErrorMessage,
StartedAt: &startedAt,
CompletedAt: &completedAt,
})
summary.TotalAppliedChanges += len(r.AppliedChanges)
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
summary.FailedModuleInstance = r.ModuleInstance
}
}
return summary, moduleReports
}
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
if len(in) == 0 {
return nil
}
out := make([]reporting.ValidatorDecisionReport, len(in))
for i, d := range in {
out[i] = reporting.ValidatorDecisionReport{
ValidatorName: d.ValidatorName,
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
}
}
return out
}
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
if len(in) == 0 {
return nil
}
out := make([]reporting.ValidatorRejectedReport, len(in))
for i, d := range in {
out[i] = reporting.ValidatorRejectedReport{
ValidatorName: d.ValidatorName,
ProposalIndex: d.ProposalIndex,
ModuleKey: d.ModuleKey,
ModuleInstance: d.ModuleInstance,
TargetSegmentID: d.TargetSegmentID,
OriginalText: d.OriginalText,
CorrectedText: d.CorrectedText,
ReasonCode: d.ReasonCode,
Message: d.Message,
}
}
return out
} }
type processFlags struct { type processFlags struct {

View File

@@ -1,4 +1,4 @@
package cli package processreport
import ( import (
"path/filepath" "path/filepath"
@@ -15,7 +15,12 @@ const (
correctionDispositionFailed = "failed" correctionDispositionFailed = "failed"
) )
type correctionLedgerEntry struct { type CorrectionLedgerInput struct {
RunDirectoryPath string
RunOutput *runner.RunOutput
}
type CorrectionLedgerEntry struct {
RunID string `json:"run_id,omitempty"` RunID string `json:"run_id,omitempty"`
ModuleKey string `json:"module_key"` ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"` ModuleInstance string `json:"module_instance"`
@@ -28,27 +33,28 @@ type correctionLedgerEntry struct {
Disposition string `json:"disposition"` Disposition string `json:"disposition"`
DispositionReasonCode string `json:"disposition_reason_code,omitempty"` DispositionReasonCode string `json:"disposition_reason_code,omitempty"`
DispositionMessage string `json:"disposition_message,omitempty"` DispositionMessage string `json:"disposition_message,omitempty"`
DeterministicValidatorResults []ledgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"` DeterministicValidatorResults []LedgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
LLMValidatorResults []ledgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"` LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
} }
type ledgerValidatorDecisionRecord struct { type LedgerValidatorDecisionRecord struct {
ValidatorKey string `json:"validator_key"` ValidatorKey string `json:"validator_key"`
Approved bool `json:"approved"` Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"` ReasonCode string `json:"reason_code"`
Message string `json:"message,omitempty"` Message string `json:"message,omitempty"`
} }
func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []correctionLedgerEntry { func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry {
runOutput := input.RunOutput
if runOutput == nil || len(runOutput.ModuleResults) == 0 { if runOutput == nil || len(runOutput.ModuleResults) == 0 {
return nil return nil
} }
runID := "" runID := ""
if runDirPath != "" { if input.RunDirectoryPath != "" {
runID = filepath.Base(runDirPath) runID = filepath.Base(input.RunDirectoryPath)
} }
entries := make([]correctionLedgerEntry, 0) entries := make([]CorrectionLedgerEntry, 0)
for _, module := range runOutput.ModuleResults { for _, module := range runOutput.ModuleResults {
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord) decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
for _, decision := range module.ValidatorDecisions { for _, decision := range module.ValidatorDecisions {
@@ -56,7 +62,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
} }
for _, change := range module.AppliedChanges { for _, change := range module.AppliedChanges {
entries = append(entries, correctionLedgerEntry{ entries = append(entries, CorrectionLedgerEntry{
RunID: runID, RunID: runID,
ModuleKey: module.ModuleKey, ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance, ModuleInstance: module.ModuleInstance,
@@ -72,7 +78,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
}) })
} }
for _, change := range module.SkippedChanges { for _, change := range module.SkippedChanges {
entries = append(entries, correctionLedgerEntry{ entries = append(entries, CorrectionLedgerEntry{
RunID: runID, RunID: runID,
ModuleKey: module.ModuleKey, ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance, ModuleInstance: module.ModuleInstance,
@@ -89,7 +95,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
}) })
} }
for _, rejection := range module.ValidatorRejected { for _, rejection := range module.ValidatorRejected {
entries = append(entries, correctionLedgerEntry{ entries = append(entries, CorrectionLedgerEntry{
RunID: runID, RunID: runID,
ModuleKey: module.ModuleKey, ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance, ModuleInstance: module.ModuleInstance,
@@ -106,7 +112,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
}) })
} }
if module.Status == runner.ModuleStatusFailed { if module.Status == runner.ModuleStatusFailed {
entries = append(entries, correctionLedgerEntry{ entries = append(entries, CorrectionLedgerEntry{
RunID: runID, RunID: runID,
ModuleKey: module.ModuleKey, ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance, ModuleInstance: module.ModuleInstance,
@@ -130,17 +136,29 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
return entries return entries
} }
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord { func HasSkippedCorrections(runOutput *runner.RunOutput) bool {
if runOutput == nil {
return false
}
for _, mr := range runOutput.ModuleResults {
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
return true
}
}
return false
}
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []LedgerValidatorDecisionRecord {
if len(in) == 0 { if len(in) == 0 {
return nil return nil
} }
out := make([]ledgerValidatorDecisionRecord, 0, len(in)) out := make([]LedgerValidatorDecisionRecord, 0, len(in))
for _, decision := range in { for _, decision := range in {
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
if isLLMBacked != wantLLM { if isLLMBacked != wantLLM {
continue continue
} }
out = append(out, ledgerValidatorDecisionRecord{ out = append(out, LedgerValidatorDecisionRecord{
ValidatorKey: decision.ValidatorName, ValidatorKey: decision.ValidatorName,
Approved: decision.Approved, Approved: decision.Approved,
ReasonCode: decision.ReasonCode, ReasonCode: decision.ReasonCode,

View File

@@ -0,0 +1,128 @@
package processreport
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
)
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
output := &runner.RunOutput{
ModuleResults: []runner.ModuleResult{
{
ModuleKey: "glossary",
ModuleInstance: "glossary",
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
ValidatorDecisions: []runner.ValidatorDecisionRecord{
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
},
AppliedChanges: []proposals.AppliedChange{
{
ProposalIndex: 3,
ModuleKey: "glossary",
ModuleInstance: "glossary",
TargetSegmentID: 1,
OriginalText: "gestures",
CorrectedText: "Jesters",
},
},
},
},
}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{
RunDirectoryPath: "/tmp/audita-run-id",
RunOutput: output,
})
if len(ledger) != 1 {
t.Fatalf("expected one ledger entry, got %d", len(ledger))
}
entry := ledger[0]
if entry.RunID != "audita-run-id" || entry.Disposition != "applied" || entry.AppliedCorrectedText != "Jesters" {
t.Fatalf("unexpected applied ledger entry: %+v", entry)
}
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
}
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
}
}
func TestBuildCorrectionLedgerPreservesDispositionPolicy(t *testing.T) {
output := &runner.RunOutput{
ModuleResults: []runner.ModuleResult{
{
ModuleKey: "grammar",
ModuleInstance: "grammar",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
Status: runner.ModuleStatusSuccess,
SkippedChanges: []proposals.SkippedChange{
{
ProposalIndex: 2,
TargetSegmentID: 7,
OriginalText: "old",
CorrectedText: "new",
SkipReason: proposals.SkipReasonAmbiguousOriginal,
Message: "ambiguous",
},
},
ValidatorRejected: []runner.ValidatorRejectedChange{
{
ProposalIndex: 3,
TargetSegmentID: 8,
OriginalText: "before",
CorrectedText: "after",
ReasonCode: "protected_term",
Message: "blocked",
},
},
},
{
ModuleKey: "capitalization",
ModuleInstance: "capitalization",
Status: runner.ModuleStatusFailed,
ErrorMessage: "failed",
},
},
}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
if len(ledger) != 3 {
t.Fatalf("expected skipped, rejected, and failed entries, got %+v", ledger)
}
byDisposition := make(map[string]CorrectionLedgerEntry)
for _, entry := range ledger {
byDisposition[entry.Disposition] = entry
}
if byDisposition["skipped"].DispositionReasonCode != string(proposals.SkipReasonAmbiguousOriginal) ||
byDisposition["skipped"].DispositionMessage != "ambiguous" {
t.Fatalf("unexpected skipped ledger entry: %+v", byDisposition["skipped"])
}
if byDisposition["rejected"].DispositionReasonCode != "protected_term" ||
byDisposition["rejected"].ProposedCorrectedText != "after" {
t.Fatalf("unexpected rejected ledger entry: %+v", byDisposition["rejected"])
}
if byDisposition["failed"].DispositionReasonCode != "module_failed" ||
byDisposition["failed"].DispositionMessage != "failed" {
t.Fatalf("unexpected failed ledger entry: %+v", byDisposition["failed"])
}
}
func TestHasSkippedCorrectionsIncludesApplicationSkipsAndValidatorRejections(t *testing.T) {
if HasSkippedCorrections(nil) {
t.Fatal("nil output should not have skipped corrections")
}
if HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 1}}}}}) {
t.Fatal("applied-only output should not have skipped corrections")
}
if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{SkippedChanges: []proposals.SkippedChange{{ProposalIndex: 1}}}}}) {
t.Fatal("application skips should count as skipped corrections")
}
if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 1}}}}}) {
t.Fatal("validator rejections should count as skipped corrections")
}
}

View File

@@ -0,0 +1,158 @@
package processreport
import (
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
// BuildInput contains already-computed process execution facts for report assembly.
type BuildInput struct {
Status string
TranscriptPath string
GlossaryPath string
OutputPath string
Modules []string
OutputSchema string
ConfigVersion *int
StartedAt time.Time
CompletedAt time.Time
ErrorMessage string
ErrorPhase string
RunDirectoryPath string
NormalizationSummary *normalization.NormalizationSummary
ChunkingSummary *chunking.Summary
RunOutput *runner.RunOutput
}
// Build creates the public process report without owning command parsing or config loading.
func Build(input BuildInput) reporting.ProcessReport {
report := reporting.ProcessReport{
ReportMetadata: reporting.ReportMetadata{
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
OutputSchema: input.OutputSchema,
ConfigVersion: input.ConfigVersion,
},
Phase: "default_pipeline",
Status: input.Status,
Operation: "process",
TranscriptPath: input.TranscriptPath,
GlossaryPath: input.GlossaryPath,
OutputPath: input.OutputPath,
Modules: append([]string(nil), input.Modules...),
StartedAt: input.StartedAt,
CompletedAt: &input.CompletedAt,
ErrorPhase: input.ErrorPhase,
}
if input.RunDirectoryPath != "" {
runSucceeded := input.Status == "success"
metadata := diagnostics.BuildDiagnosticsMetadata(input.RunDirectoryPath, runSucceeded)
report.Diagnostics = &metadata
}
if input.ErrorMessage != "" {
report.ErrorMessage = input.ErrorMessage
}
if input.NormalizationSummary != nil {
report.InputSegmentCount = &input.NormalizationSummary.InputSegmentCount
report.NormalizedSegmentCount = &input.NormalizationSummary.OutputSegmentCount
report.NormalizationMerges = &input.NormalizationSummary.MergesPerformed
report.NormalizationIDReassignments = &input.NormalizationSummary.IDsReassigned
report.NormalizationSkipped.DifferentSpeakers = &input.NormalizationSummary.SkippedMerges.DifferentSpeakers
report.NormalizationSkipped.GapTooLarge = &input.NormalizationSummary.SkippedMerges.GapTooLarge
report.NormalizationSkipped.DurationExceeded = &input.NormalizationSummary.SkippedMerges.DurationExceeded
report.NormalizationSkipped.TokenLimitExceeded = &input.NormalizationSummary.SkippedMerges.TokenLimitExceeded
}
if input.ChunkingSummary != nil {
report.Chunking = &reporting.ChunkingSummary{
ChunkCount: input.ChunkingSummary.ChunkCount,
MinEstimatedTokens: input.ChunkingSummary.MinEstimatedTokens,
MaxEstimatedTokens: input.ChunkingSummary.MaxEstimatedTokens,
TotalEstimatedTokens: input.ChunkingSummary.TotalEstimatedTokens,
TargetSections: input.ChunkingSummary.TargetSections,
MaxSectionTokens: input.ChunkingSummary.MaxSectionTokens,
MinSectionTokens: input.ChunkingSummary.MinSectionTokens,
}
}
report.ModulesSummary, report.ModuleResults = buildModuleReporting(input.RunOutput)
return report
}
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
return nil, nil
}
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
for _, r := range runOutput.ModuleResults {
startedAt := r.StartedAt
completedAt := r.CompletedAt
moduleReports = append(moduleReports, reporting.ModuleReport{
ModuleKey: r.ModuleKey,
ModuleInstance: r.ModuleInstance,
ReplacementPolicy: string(r.ReplacementPolicy),
Status: r.Status,
ProposalCount: r.ProposalCount,
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
AppliedChanges: r.AppliedChanges,
SkippedChanges: r.SkippedChanges,
ErrorMessage: r.ErrorMessage,
StartedAt: &startedAt,
CompletedAt: &completedAt,
})
summary.TotalAppliedChanges += len(r.AppliedChanges)
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
summary.FailedModuleInstance = r.ModuleInstance
}
}
return summary, moduleReports
}
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
if len(in) == 0 {
return nil
}
out := make([]reporting.ValidatorDecisionReport, len(in))
for i, d := range in {
out[i] = reporting.ValidatorDecisionReport{
ValidatorName: d.ValidatorName,
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
}
}
return out
}
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
if len(in) == 0 {
return nil
}
out := make([]reporting.ValidatorRejectedReport, len(in))
for i, d := range in {
out[i] = reporting.ValidatorRejectedReport{
ValidatorName: d.ValidatorName,
ProposalIndex: d.ProposalIndex,
ModuleKey: d.ModuleKey,
ModuleInstance: d.ModuleInstance,
TargetSegmentID: d.TargetSegmentID,
OriginalText: d.OriginalText,
CorrectedText: d.CorrectedText,
ReasonCode: d.ReasonCode,
Message: d.Message,
}
}
return out
}

View File

@@ -0,0 +1,180 @@
package processreport
import (
"path/filepath"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
)
func TestBuildSuccessReportMapsExecutionFacts(t *testing.T) {
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
completedAt := startedAt.Add(time.Second)
configVersion := 4
targetSections := 2
inputSegments := 5
outputSegments := 4
merges := 1
reassigned := 2
differentSpeakers := 3
report := Build(BuildInput{
Status: "success",
TranscriptPath: "transcript.json",
GlossaryPath: "glossary.yaml",
OutputPath: "out.json",
Modules: []string{"grammar"},
OutputSchema: "default",
ConfigVersion: &configVersion,
StartedAt: startedAt,
CompletedAt: completedAt,
RunDirectoryPath: filepath.Join(
"tmp",
"audita-run",
),
NormalizationSummary: &normalization.NormalizationSummary{
InputSegmentCount: inputSegments,
OutputSegmentCount: outputSegments,
MergesPerformed: merges,
IDsReassigned: reassigned,
SkippedMerges: struct {
DifferentSpeakers int `json:"different_speakers"`
GapTooLarge int `json:"gap_too_large"`
DurationExceeded int `json:"duration_exceeded"`
TokenLimitExceeded int `json:"token_limit_exceeded"`
}{
DifferentSpeakers: differentSpeakers,
},
},
ChunkingSummary: &chunking.Summary{
ChunkCount: 3,
MinEstimatedTokens: 10,
MaxEstimatedTokens: 20,
TotalEstimatedTokens: 45,
TargetSections: &targetSections,
MaxSectionTokens: 200,
MinSectionTokens: 50,
},
RunOutput: &runner.RunOutput{
ModuleResults: []runner.ModuleResult{
{
ModuleKey: "grammar",
ModuleInstance: "grammar",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
Status: runner.ModuleStatusSuccess,
ProposalCount: 2,
ValidatorDecisions: []runner.ValidatorDecisionRecord{
{
ValidatorName: "proposal_shape",
ProposalIndex: 1,
Approved: true,
ReasonCode: "approved",
Message: "ok",
DiagnosticArtifactPath: "diagnostics/validator.json",
},
},
ValidatorRejected: []runner.ValidatorRejectedChange{
{
ValidatorName: "protected_term",
ProposalIndex: 2,
ModuleKey: "grammar",
ModuleInstance: "grammar",
TargetSegmentID: 7,
OriginalText: "old",
CorrectedText: "new",
ReasonCode: "protected_term",
Message: "blocked",
},
},
AppliedChanges: []proposals.AppliedChange{
{ProposalIndex: 1, TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"},
},
SkippedChanges: []proposals.SkippedChange{
{ProposalIndex: 3, TargetSegmentID: 8, SkipReason: proposals.SkipReasonMissingSegment},
},
StartedAt: startedAt,
CompletedAt: completedAt,
},
},
},
})
if report.ReportMetadata.ReportSchemaName != reporting.DefaultProcessReportSchemaName ||
report.ReportMetadata.ReportSchemaVersion != reporting.DefaultProcessReportSchemaVersion ||
report.ReportMetadata.OutputSchema != "default" ||
report.ReportMetadata.ConfigVersion == nil ||
*report.ReportMetadata.ConfigVersion != configVersion {
t.Fatalf("unexpected report metadata: %+v", report.ReportMetadata)
}
if report.Phase != "default_pipeline" || report.Operation != "process" || report.Status != "success" {
t.Fatalf("unexpected process identity fields: phase=%q operation=%q status=%q", report.Phase, report.Operation, report.Status)
}
if report.Diagnostics == nil || report.Diagnostics.CorrectionLedgerPath != filepath.Join("tmp", "audita-run", diagnostics.ArtifactCorrectionLedger) {
t.Fatalf("unexpected diagnostics metadata: %+v", report.Diagnostics)
}
if report.InputSegmentCount == nil || *report.InputSegmentCount != inputSegments ||
report.NormalizationSkipped.DifferentSpeakers == nil ||
*report.NormalizationSkipped.DifferentSpeakers != differentSpeakers {
t.Fatalf("unexpected normalization summary: %+v", report)
}
if report.Chunking == nil || report.Chunking.ChunkCount != 3 || report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != targetSections {
t.Fatalf("unexpected chunking summary: %+v", report.Chunking)
}
if report.ModulesSummary == nil ||
report.ModulesSummary.ModuleCount != 1 ||
report.ModulesSummary.TotalAppliedChanges != 1 ||
report.ModulesSummary.TotalSkippedChanges != 2 {
t.Fatalf("unexpected modules summary: %+v", report.ModulesSummary)
}
if len(report.ModuleResults) != 1 ||
len(report.ModuleResults[0].ValidatorDecisions) != 1 ||
len(report.ModuleResults[0].ValidatorRejected) != 1 {
t.Fatalf("unexpected module reports: %+v", report.ModuleResults)
}
}
func TestBuildFailedReportPreservesErrorAndFailureSummary(t *testing.T) {
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
completedAt := startedAt.Add(time.Second)
report := Build(BuildInput{
Status: "failed",
TranscriptPath: "transcript.json",
GlossaryPath: "glossary.yaml",
Modules: []string{"grammar"},
OutputSchema: "default",
StartedAt: startedAt,
CompletedAt: completedAt,
ErrorPhase: "module",
ErrorMessage: "module failed",
RunDirectoryPath: "run-dir",
RunOutput: &runner.RunOutput{
ModuleResults: []runner.ModuleResult{
{
ModuleKey: "grammar",
ModuleInstance: "grammar",
Status: runner.ModuleStatusFailed,
ErrorMessage: "module failed",
StartedAt: startedAt,
CompletedAt: completedAt,
},
},
},
})
if report.Status != "failed" || report.ErrorPhase != "module" || report.ErrorMessage != "module failed" {
t.Fatalf("unexpected failure fields: %+v", report)
}
if report.Diagnostics == nil || report.Diagnostics.ErrorLogPath == "" {
t.Fatalf("expected failure diagnostics metadata, got %+v", report.Diagnostics)
}
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "grammar" {
t.Fatalf("unexpected failed module summary: %+v", report.ModulesSummary)
}
}