From 93f62c909587c1b9c2d559412eb4c5a2c5c236ef Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 25 Sep 2026 02:50:46 +0000 Subject: [PATCH] Accept span deletions in structured responses --- docs/internal/modules.md | 4 ++ docs/internal/prompts.md | 5 ++ docs/roadmap/implementation.md | 7 ++- internal/framework/llm/adapter_test.go | 53 +++++++++++++++++++ .../assets/correction_set.v1.json | 3 +- .../framework/responseschema/registry_test.go | 14 +---- .../validators/llm_validators_test.go | 38 +++++++++++++ .../prompts/modules.grammar.proposal.yaml | 2 + .../prompts/modules.spoken_word.proposal.yaml | 2 + .../prompts/validators.editorial_review.yaml | 1 + .../validators.meaning_reversal_review.yaml | 1 + .../validators.spoken_form_plausibility.yaml | 1 + 12 files changed, 115 insertions(+), 16 deletions(-) diff --git a/docs/internal/modules.md b/docs/internal/modules.md index 01fc639..33c82d8 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -43,6 +43,10 @@ renders their embedded definitions from those variables and validates the correction-set schema before Audita maps proposals. Domain validators still check confidence, applicability, protected glossary terms, and replacement policy. Shared context encoding lives in `internal/framework/promptcontext`. +An empty `corrected_text` deletes the identified span when the resulting segment +remains nonblank. Whitespace-only original spans can express spacing corrections; +matching stays literal under each module's replacement policy. No automatic +whitespace cleanup follows application. ## Validator Chain Ownership diff --git a/docs/internal/prompts.md b/docs/internal/prompts.md index 5294768..8a547f7 100644 --- a/docs/internal/prompts.md +++ b/docs/internal/prompts.md @@ -22,6 +22,11 @@ Validator prompts receive the correction payload and description block. The prompt assets retain the prior task instructions and output-schema association. The role-bound runtime will supply the complete JSON Schema output contract and repair count; the definition does not own an operator-facing repair default. +The correction-set schema requires a string replacement and accepts an empty +string. Proposal guidance uses it for eligible span deletion and permits literal +spacing edits; validator guidance evaluates the full before/after segment under +each validator's existing remit. A blank resulting segment remains a domain +rejection, not a structural schema error. `internal/framework/responseschema` remains the only embedded source for `correction_set.v1.json` and `validator_decision_set.v1.json`. It exposes that diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index db9aa09..83b5cab 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -2,7 +2,7 @@ **Last updated:** 2026-09-25 -**Status:** In progress; Stage 1 complete. +**Status:** In progress; Stages 1–2 complete. **Source baseline:** `a83a518` (feature roadmap); application baseline `0b6cd98`. **Audience:** gpt-5.6-terra coding agent. @@ -239,6 +239,11 @@ but schema-valid proposals still reach ordinary filtering without an outer retry **End state:** A literal empty replacement can cross the real library/schema boundary, and the model instructions and validator inputs describe its meaning. +**Completed 2026-09-25.** Relaxed only the replacement length constraint, +clarified the relevant proposal and active validation prompts, and verified +rendering, real-adapter acceptance/rejection, repair counts, and serialized +before/after validator payloads. The specified focused package tests passed. + ## Stage 3 — Preserve Empty Replacements in Debug Evidence **Work** diff --git a/internal/framework/llm/adapter_test.go b/internal/framework/llm/adapter_test.go index 9029171..b687aa0 100644 --- a/internal/framework/llm/adapter_test.go +++ b/internal/framework/llm/adapter_test.go @@ -185,6 +185,59 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) { } } +func TestCorrectionSchemaReplacementValuesThroughAdapter(t *testing.T) { + for _, tc := range []struct { + name, original, replacement string + valid bool + repairs int + }{ + {name: "empty replacement", original: "uh ", replacement: `"corrected_text":""`, valid: true, repairs: 2}, + {name: "spaces replacement", original: "hello", replacement: `"corrected_text":" "`, valid: true, repairs: 2}, + {name: "ordinary replacement", original: " ", replacement: `"corrected_text":" "`, valid: true, repairs: 2}, + {name: "missing replacement", original: "uh ", replacement: "", valid: false}, + {name: "null replacement", original: "uh ", replacement: `"corrected_text":null`, valid: false}, + {name: "numeric replacement", original: "uh ", replacement: `"corrected_text":1`, valid: false}, + {name: "empty original", original: "", replacement: `"corrected_text":"x"`, valid: false}, + } { + t.Run(tc.name, func(t *testing.T) { + fields := []string{`"id":1`, `"original_text":` + fmt.Sprintf("%q", tc.original), `"confidence":0.9`} + if tc.replacement != "" { + fields = append(fields, tc.replacement) + } + payload := `{"corrections":[{` + strings.Join(fields, ",") + `}]}` + runtime, count, _ := adapterFixture(t, []reply{{200, payload}}, 0, tc.repairs, nil) + client, err := runtime.NewRoleClient("proposal", CallHooks{}) + if err != nil { + t.Fatal(err) + } + var out map[string]any + response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out) + if got := count.Load(); got != 1 { + t.Fatalf("expected one provider generation, got %d", got) + } + if response.Execution.StructuralRepairs != 0 || response.Execution.ProviderAttempts != 1 { + t.Fatalf("unexpected repair/generation counts: %+v", response.Execution) + } + if tc.valid { + if err != nil { + t.Fatal(err) + } + corrections := out["corrections"].([]any) + item := corrections[0].(map[string]any) + var expected string + if err := json.Unmarshal([]byte(strings.TrimPrefix(tc.replacement, `"corrected_text":`)), &expected); err != nil { + t.Fatal(err) + } + if item["corrected_text"] != expected || item["original_text"] != tc.original { + t.Fatalf("span values changed: %+v", item) + } + } else if !structuredoutput.IsMalformedError(err) { + t.Fatalf("expected structural rejection, got %v", err) + } + }) + } +} + func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) { requestSeen := false var posted struct { diff --git a/internal/framework/responseschema/assets/correction_set.v1.json b/internal/framework/responseschema/assets/correction_set.v1.json index 5f53592..4e7c76b 100644 --- a/internal/framework/responseschema/assets/correction_set.v1.json +++ b/internal/framework/responseschema/assets/correction_set.v1.json @@ -26,8 +26,7 @@ "minLength": 1 }, "corrected_text": { - "type": "string", - "minLength": 1 + "type": "string" }, "confidence": { "type": "number", diff --git a/internal/framework/responseschema/registry_test.go b/internal/framework/responseschema/registry_test.go index 992f417..99f21f5 100644 --- a/internal/framework/responseschema/registry_test.go +++ b/internal/framework/responseschema/registry_test.go @@ -30,11 +30,6 @@ func TestPromptKitSchemaPathsUseCanonicalAssets(t *testing.T) { } } -const ( - expectedCorrectionSetSHA256 = "b86a2dde38d7f440d26470fa8830167512bb0aa1b35e5fee5547057be583c388" - expectedValidatorDecisionSetSHA256 = "2fe90d450e2595b57885aba91c8cc5cedf783dd36758ab430309e3eace401f54" -) - func TestLookupKnownSchemas(t *testing.T) { correction, ok := Lookup(CorrectionSetKey) if !ok { @@ -86,20 +81,13 @@ func TestRegisteredSchemasLoadReadableEmbeddedJSON(t *testing.T) { } func TestSchemaHashesMatchRegisteredJSON(t *testing.T) { - expectedByKey := map[Key]string{ - CorrectionSetKey: expectedCorrectionSetSHA256, - ValidatorDecisionSetKey: expectedValidatorDecisionSetSHA256, - } - for key, expectedHash := range expectedByKey { + for _, key := range []Key{CorrectionSetKey, ValidatorDecisionSetKey} { schema, ok := Lookup(key) if !ok { t.Fatalf("missing schema %q", key) } sum := sha256.Sum256(schema.JSONSchema) expected := hex.EncodeToString(sum[:]) - if schema.SHA256 != expectedHash { - t.Fatalf("unexpected stable hash for %q: got %q want %q", key, schema.SHA256, expectedHash) - } if schema.SHA256 != expected { t.Fatalf("unexpected hash for %q: got %q want %q", key, schema.SHA256, expected) } diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index a7e2a90..73d4d1f 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -2,6 +2,7 @@ package validators import ( "context" + "encoding/json" "errors" "strings" "testing" @@ -75,6 +76,43 @@ func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal { } } +func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.T) { + for _, tc := range []struct { + name, segment, original, replacement, result string + }{ + {name: "deletion", segment: "uh hello", original: "uh ", replacement: "", result: "hello"}, + {name: "spacing", segment: "hello world", original: " ", replacement: " ", result: "hello world"}, + } { + t.Run(tc.name, func(t *testing.T) { + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, tc.original, tc.replacement)}) + req.WorkingTranscript.Segments[0].Text = tc.segment + payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req) + if len(immediate) != 0 || len(payload.Items) != 1 { + t.Fatalf("expected previewable request: %+v, %+v", payload, immediate) + } + raw, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + var decoded struct { + ValidatorName string `json:"validator_name"` + Corrections []map[string]any `json:"corrections"` + } + if err := json.Unmarshal(raw, &decoded); err != nil { + t.Fatal(err) + } + if decoded.ValidatorName != "editorial_review" || len(decoded.Corrections) != 1 { + t.Fatalf("wrong validator identity or item count: %s", raw) + } + item := decoded.Corrections[0] + value, present := item["corrected_text"] + if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(7) { + t.Fatalf("literal spans or full segment text lost: %s", raw) + } + }) + } +} + func TestChunkLLMValidationItemsOneSmallBatch(t *testing.T) { items := []LLMValidationItem{{CorrectionIndex: 0, OriginalText: "a", CorrectedText: "b", OriginalSegmentText: "x", CorrectedSegmentText: "y"}} batches, err := ChunkLLMValidationItems(items, 1000, chunking.NewSimpleTokenEstimator()) diff --git a/internal/prompts/assets/prompts/modules.grammar.proposal.yaml b/internal/prompts/assets/prompts/modules.grammar.proposal.yaml index 28a13ae..d6250d5 100644 --- a/internal/prompts/assets/prompts/modules.grammar.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.grammar.proposal.yaml @@ -24,6 +24,8 @@ messages: - For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span. - Choose an original_text span that appears exactly once in the current segment text. - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. + - A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed. + - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. - Each returned correction must contain only id, original_text, corrected_text, and confidence. - Do not return corrections where original_text and corrected_text are identical. - Do not return speaker, start, or end fields. diff --git a/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml b/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml index 9aa0334..b48b045 100644 --- a/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml @@ -26,6 +26,8 @@ messages: - For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span. - Choose an original_text span that appears exactly once in the current segment text. - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. + - An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result. + - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. - Each returned correction must contain only id, original_text, corrected_text, and confidence. - Do not return corrections where original_text and corrected_text are identical. - Do not return speaker, start, or end fields. diff --git a/internal/prompts/assets/prompts/validators.editorial_review.yaml b/internal/prompts/assets/prompts/validators.editorial_review.yaml index 80a635d..c70e83c 100644 --- a/internal/prompts/assets/prompts/validators.editorial_review.yaml +++ b/internal/prompts/assets/prompts/validators.editorial_review.yaml @@ -20,6 +20,7 @@ messages: - Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart. - Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning. - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. + - An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty. - If a correction includes categories, treat them as additional segment context. - Each returned validation must contain only correction_index, approved, confidence, and reason. - confidence must be between 0.0 and 1.0. diff --git a/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml b/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml index 4c73c7f..2e7da72 100644 --- a/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml +++ b/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml @@ -13,6 +13,7 @@ messages: - Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment. - Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning. - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. + - An empty corrected_text deletes the identified span; decide from the full before/after segment whether that deletion reverses meaning, not from the empty string alone. - Do not reject a correction merely because the literal written word changes. - Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals. - If a correction includes categories, treat them as additional segment context. diff --git a/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml b/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml index 389323a..0532847 100644 --- a/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml +++ b/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml @@ -14,6 +14,7 @@ messages: - Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn". - Reject unrelated substitutions like changing "Lyra" to "Jesters". - Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical. + - Use the full original_segment_text and corrected_segment_text when a span is deleted; an empty corrected_text alone does not establish or rule out a spoken-form mistranscription. - Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions. - If a correction includes categories, treat them as additional segment context. - Each returned validation must contain only correction_index, approved, confidence, and reason.