Index secret failure locations lazily

docs/roadmap/implementation.md

@@ -1058,6 +1058,12 @@ O(V + F × D) location work, apart from sorting, rather than repeated full scans
 keep the environment cache and credential bytes out of diagnostics/history.
 Focused check: `go test ./internal/engine -run 'Secret|Location|Validation'`.
 
+Status: **complete** (F08). Prior completed commit: `f132cd6`. Validation
+passed: `env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -run 'Secret|Location|Validation' -count=1`,
+`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -count=1`,
+`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./... -count=1`,
+and `git diff --check`.
+
 ## Stage 33 — Integrate, Document, And Verify The Complete Sprint
 
 Findings: final acceptance for **all selected IDs**.

internal/engine/secret_reference.go

@@ -21,6 +21,22 @@ type secretResolutionResult struct {
 	diagnostics []Diagnostic
 }
 
+type secretReferenceLocationResolver struct {
+	input  *document.Node
+	lookup *documentLookup
+}
+
+func (resolver *secretReferenceLocationResolver) diagnostic(code Code, path Path, message string) Diagnostic {
+	if resolver.lookup == nil {
+		resolver.lookup = buildDocumentLookup(resolver.input)
+	}
+	diagnostic := Diagnostic{Code: code, Severity: SeverityError, Path: path.clone(), Message: message}
+	if location, ok := nearestDocumentLocationWithLookup(resolver.input, path, resolver.lookup); ok {
+		diagnostic.Primary = documentLocationPointer(location)
+	}
+	return diagnostic
+}
+
 func resolveSecretReferences(root reflect.Value, schema *schemaDescriptor, input *document.Node, lookup secretLookup) secretResolutionResult {
 	result := secretResolutionResult{
 		lookups: make(map[string]secretLookupResult), references: make(map[string]string),
@@ -32,17 +48,18 @@ func resolveSecretReferences(root reflect.Value, schema *schemaDescriptor, input
 	if lookup == nil {
 		lookup = func(string) (string, bool) { return "", false }
 	}
-	resolveSecretValue(root, schema.root, schema.secrets, Path{}, false, input, lookup, &result)
+	locations := &secretReferenceLocationResolver{input: input}
+	resolveSecretValue(root, schema.root, schema.secrets, Path{}, false, locations, lookup, &result)
 	result.diagnostics = canonicalDiagnostics(result.diagnostics)
 	return result
 }
 
-func resolveSecretValue(value reflect.Value, descriptor *typeDescriptor, secrets SecretBinding, path Path, optional bool, input *document.Node, lookup secretLookup, result *secretResolutionResult) {
+func resolveSecretValue(value reflect.Value, descriptor *typeDescriptor, secrets SecretBinding, path Path, optional bool, locations *secretReferenceLocationResolver, lookup secretLookup, result *secretResolutionResult) {
 	if descriptor.kind == descriptorPointer {
 		if value.IsNil() {
 			return
 		}
-		resolveSecretValue(value.Elem(), descriptor.element, secrets, path, descriptor.element.kind == descriptorSecret, input, lookup, result)
+		resolveSecretValue(value.Elem(), descriptor.element, secrets, path, descriptor.element.kind == descriptorSecret, locations, lookup, result)
 		return
 	}
 
@@ -57,11 +74,11 @@ func resolveSecretValue(value reflect.Value, descriptor *typeDescriptor, secrets
 				code = CodeSecretInvalidName
 				message = "secret environment name is empty"
 			}
-			result.diagnostics = append(result.diagnostics, secretDiagnostic(code, path, message, input))
+			result.diagnostics = append(result.diagnostics, locations.diagnostic(code, path, message))
 			return
 		}
 		if !validEnvironmentName(name) {
-			result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretInvalidName, path, "secret environment name is invalid", input))
+			result.diagnostics = append(result.diagnostics, locations.diagnostic(CodeSecretInvalidName, path, "secret environment name is invalid"))
 			return
 		}
 		found, exists := result.lookups[name]
@@ -70,11 +87,11 @@ func resolveSecretValue(value reflect.Value, descriptor *typeDescriptor, secrets
 			result.lookups[name] = found
 		}
 		if !found.present {
-			result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretUnset, path, "secret environment variable is not set", input))
+			result.diagnostics = append(result.diagnostics, locations.diagnostic(CodeSecretUnset, path, "secret environment variable is not set"))
 			return
 		}
 		if found.value == "" {
-			result.diagnostics = append(result.diagnostics, secretDiagnostic(CodeSecretEmpty, path, "secret environment variable is empty", input))
+			result.diagnostics = append(result.diagnostics, locations.diagnostic(CodeSecretEmpty, path, "secret environment variable is empty"))
 			return
 		}
 		value.Set(secrets.Make(SecretState{Name: name, Value: found.value, Specified: true, Resolved: true}))
@@ -85,7 +102,7 @@ func resolveSecretValue(value reflect.Value, descriptor *typeDescriptor, secrets
 			if !member.inline {
 				memberPath = path.Key(member.name)
 			}
-			resolveSecretValue(value.Field(member.index), member.descriptor, secrets, memberPath, false, input, lookup, result)
+			resolveSecretValue(value.Field(member.index), member.descriptor, secrets, memberPath, false, locations, lookup, result)
 		}
 	case descriptorMap:
 		if value.IsNil() {
@@ -96,7 +113,7 @@ func resolveSecretValue(value reflect.Value, descriptor *typeDescriptor, secrets
 		for _, key := range keys {
 			entry := reflect.New(value.Type().Elem()).Elem()
 			entry.Set(value.MapIndex(key))
-			resolveSecretValue(entry, descriptor.element, secrets, path.Key(key.String()), false, input, lookup, result)
+			resolveSecretValue(entry, descriptor.element, secrets, path.Key(key.String()), false, locations, lookup, result)
 			value.SetMapIndex(key, entry)
 		}
 	case descriptorSlice, descriptorArray:
@@ -104,19 +121,11 @@ func resolveSecretValue(value reflect.Value, descriptor *typeDescriptor, secrets
 			return
 		}
 		for index := 0; index < value.Len(); index++ {
-			resolveSecretValue(value.Index(index), descriptor.element, secrets, path.Index(index), false, input, lookup, result)
+			resolveSecretValue(value.Index(index), descriptor.element, secrets, path.Index(index), false, locations, lookup, result)
 		}
 	}
 }
 
-func secretDiagnostic(code Code, path Path, message string, input *document.Node) Diagnostic {
-	diagnostic := Diagnostic{Code: code, Severity: SeverityError, Path: path.clone(), Message: message}
-	if location, ok := nearestDocumentLocation(input, path); ok {
-		diagnostic.Primary = documentLocationPointer(location)
-	}
-	return diagnostic
-}
-
 func changedSecretStates(root reflect.Value, schema *schemaDescriptor, lookups map[string]secretLookupResult) []Path {
 	var changed []Path
 	if schema == nil || schema.root == nil {

internal/engine/secret_reference_test.go

@@ -131,6 +131,94 @@ func TestSecretReferenceErrorsAreSafeAndPathSpecific(t *testing.T) {
 	})
 }
 
+func TestSecretFailureLocationsIndexWideInputOnce(t *testing.T) {
+	type config struct {
+		Secrets     map[string]Secret `yaml:"secrets"`
+		DefaultOnly Secret            `yaml:"defaultOnly"`
+		Sequence    []Secret          `yaml:"sequence"`
+	}
+
+	keys := []string{"a/b~c", "12", "ordinary"}
+	for index := 0; index < 128; index++ {
+		keys = append(keys, fmt.Sprintf("wide-%03d", index))
+	}
+	var source strings.Builder
+	source.WriteString("secrets:\n")
+	lookupCounts := make(map[string]int)
+	wantCodes := make(map[string]Code, len(keys)+3)
+	wantLines := make(map[string]int, len(keys)+2)
+	for index, key := range keys {
+		var name string
+		var code Code
+		switch index % 4 {
+		case 0:
+			name, code = "MISSING_SHARED", CodeSecretUnset
+		case 1:
+			name, code = "EMPTY_SHARED", CodeSecretEmpty
+		case 2:
+			name, code = "INVALID-NAME", CodeSecretInvalidName
+		default:
+			name, code = "", CodeSecretInvalidName
+		}
+		line := index + 2
+		fmt.Fprintf(&source, "  %q: %q\n", key, name)
+		path := Path{}.Key("secrets").Key(key).String()
+		wantCodes[path] = code
+		wantLines[path] = line
+	}
+	sequenceHeaderLine := len(keys) + 2
+	source.WriteString("sequence:\n  - MISSING_SHARED\n  - EMPTY_SHARED\n")
+	sequencePaths := []string{Path{}.Key("sequence").Index(0).String(), Path{}.Key("sequence").Index(1).String()}
+	wantCodes[sequencePaths[0]] = CodeSecretUnset
+	wantCodes[sequencePaths[1]] = CodeSecretEmpty
+	wantLines[sequencePaths[0]] = sequenceHeaderLine + 1
+	wantLines[sequencePaths[1]] = sequenceHeaderLine + 2
+	defaultPath := Path{}.Key("defaultOnly").String()
+	wantCodes[defaultPath] = CodeSecretUnset
+
+	_, err := testLoadWithSecretLookup(Bytes("secret-index.yml", []byte(source.String())), func(name string) (string, bool) {
+		lookupCounts[name]++
+		if name == "EMPTY_SHARED" {
+			return "", true
+		}
+		return "", false
+	}, Options[config]{Definition: Definition[config]{Defaults: func() config {
+		return config{DefaultOnly: SecretEnv("MISSING_SHARED")}
+	}}})
+	loadError := requireSecretLoadCode(t, err, CodeSecretUnset)
+	diagnostics := loadError.Diagnostics()
+	if len(diagnostics) != len(wantCodes) {
+		t.Fatalf("diagnostic count = %d, want %d: %#v", len(diagnostics), len(wantCodes), diagnostics)
+	}
+	gotPaths := make(map[string]struct{}, len(diagnostics))
+	for _, diagnostic := range diagnostics {
+		path := diagnostic.Path.String()
+		gotPaths[path] = struct{}{}
+		if code, ok := wantCodes[path]; !ok || diagnostic.Code != code {
+			t.Errorf("diagnostic %q code = %q, want %q", path, diagnostic.Code, code)
+		}
+		if diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-1" {
+			t.Errorf("diagnostic %q location = %#v, want source source-1", path, diagnostic.Primary)
+			continue
+		}
+		if line, hasExactLine := wantLines[path]; hasExactLine && diagnostic.Primary.Line != line {
+			t.Errorf("diagnostic %q line = %d, want %d", path, diagnostic.Primary.Line, line)
+		}
+		if path == defaultPath && diagnostic.Primary.Line != 1 {
+			t.Errorf("defaults-only diagnostic line = %d, want nearest root line 1", diagnostic.Primary.Line)
+		}
+	}
+	if _, ok := gotPaths["/secrets/a~1b~0c"]; !ok {
+		t.Fatal("escaped mapping key diagnostic path was not preserved")
+	}
+	if _, ok := gotPaths["/secrets/12"]; !ok {
+		t.Fatal("numeric mapping key diagnostic path was not preserved")
+	}
+	if lookupCounts["MISSING_SHARED"] != 1 || lookupCounts["EMPTY_SHARED"] != 1 || len(lookupCounts) != 2 {
+		t.Fatalf("environment lookup counts = %#v, want one lookup per repeated name", lookupCounts)
+	}
+}
+
 func TestResolutionSelectsEffectiveSecretNameBeforeLookup(t *testing.T) {
 	type config struct {
 		Name   string `yaml:"name"`