Classify model failures and capture execution counts

internal/framework/llm/adapter.go

@@ -3,10 +3,13 @@ package llm
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
+	"sync"
 	"time"
 
+	"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
 	"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
 	"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
 	"gitea.maximumdirect.net/eric/audita/internal/prompts"
@@ -39,6 +42,26 @@ func (r *Runtime) NewRoleClient(role string, hooks CallHooks) (*RoleClient, erro
 func (c *RoleClient) CompleteStructured(parent context.Context, req structuredoutput.Request, out any) (response structuredoutput.Response, err error) {
 	ctx, cancel := context.WithTimeout(parent, c.runtime.Config.LLM.OperationTimeout)
 	defer cancel()
+	var terminalMu sync.Mutex
+	var terminal pk.ExecutionEvent
+	terminalAvailable := false
+	defer func() {
+		terminalMu.Lock()
+		if terminalAvailable {
+			response.Execution = outcomes.ExecutionCounts{
+				Available:         true,
+				ProviderAttempts:  int(terminal.Attempts),
+				TransientRetries:  int(terminal.TransientRetries),
+				StructuralRepairs: int(terminal.RepairCalls),
+				RetriesExhausted:  terminal.RetriesExhausted,
+			}
+			if terminal.StatusCode != 0 {
+				status := terminal.StatusCode
+				response.Execution.HTTPStatus = &status
+			}
+		}
+		terminalMu.Unlock()
+	}()
 	record := structuredoutput.CallRecord{CallID: c.runtime.nextCallID.Add(1), Context: copyContext(req.Context), Started: time.Now().UTC()}
 	record.Context.Role = c.role
 	profile := c.runtime.Profiles[c.role]
@@ -49,7 +72,8 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
 	defer func() {
 		record.Finished = time.Now().UTC()
 		if err != nil {
-			if structuredoutput.IsMalformedError(err) {
+			var failure *structuredoutput.Failure
+			if errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted {
 				record.Failure = structuredoutput.FailureMalformedOutput
 			} else {
 				record.Failure = structuredoutput.FailureRuntime
@@ -59,12 +83,15 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
 			c.hooks.OnCall(record)
 		}
 	}()
+	if parent.Err() != nil {
+		return response, classifiedFailure(parent, ctx, parent.Err(), nil)
+	}
 	if err = validateOutputTarget(out); err != nil {
-		return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "invalid structured output target", Cause: err}
+		return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err}
 	}
 	schemaPath, ok := responseschema.AssetPath(req.Schema)
 	if !ok {
-		return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "unknown structured response schema", Cause: fmt.Errorf("schema %q", req.Schema)}
+		return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: fmt.Errorf("schema %q", req.Schema)}
 	}
 	backend := c.runtime.Config.Backends[profile.EffectiveModelParams.BackendID]
 	vars := make(map[string]string, len(req.Variables)+1)
@@ -81,6 +108,12 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
 		ProfileID: profile.ProfileID, APIKey: apiKey, Vars: vars,
 		Validation: &pk.OutputContract{Format: pk.FormatJSON, ValidationMode: pk.ValidationJSONSchema, SchemaPath: schemaPath, RepairAttempts: c.runtime.Config.LLM.RepairAttempts},
 		Observer: func(event pk.ExecutionEvent) {
+			if event.Kind == pk.ExecutionFinished {
+				terminalMu.Lock()
+				terminal = event
+				terminalAvailable = true
+				terminalMu.Unlock()
+			}
 			if c.hooks.OnEvent != nil {
 				c.hooks.OnEvent(event)
 			}
@@ -90,7 +123,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
 		},
 	})
 	if prepErr != nil {
-		return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "could not prepare structured model call", Cause: prepErr}
+		return response, classifiedFailure(parent, ctx, prepErr, nil)
 	}
 	defer handle.Discard()
 	details := handle.Details()
@@ -102,7 +135,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
 	selected, exists := c.runtime.Config.Backends[details.SelectedBackendID]
 	endpoint := func(s string) string { return strings.TrimRight(strings.TrimSpace(s), "/") }
 	if !exists || details.SelectedProfileID != profile.ProfileID || details.SelectedBackendID != profile.EffectiveModelParams.BackendID || endpoint(details.EffectiveModelParams.Endpoint) != endpoint(selected.Endpoint) || details.EffectiveModelParams.APIKeyEnv != "" {
-		return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model target changed after configuration inspection", Cause: fmt.Errorf("prepared target does not match registered role backend")}
+		return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model target changed after configuration inspection", Cause: fmt.Errorf("prepared target does not match registered role backend")}
 	}
 	result, runErr := c.runtime.Engine.RunPrepared(ctx, handle)
 	if result != nil {
@@ -114,22 +147,94 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
 		response.TotalTokens = result.Usage.TotalTokens
 	}
 	if runErr != nil {
-		return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: runErr}
+		return response, classifiedFailure(parent, ctx, runErr, result)
 	}
 	if result == nil {
-		return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call returned no result"}
+		return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
 	}
 	if !result.Validation.IsValid {
-		return response, &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}
+		return response, classifiedFailure(parent, ctx, nil, result)
 	}
 	if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
-		return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "validated structured output could not be decoded", Cause: decodeErr}
+		var typeError *json.UnmarshalTypeError
+		if errors.As(decodeErr, &typeError) {
+			return response, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
+		}
+		return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
 	}
 	response.Content = json.RawMessage(result.RawOutput)
 	response.Provider = "openai-compatible"
 	return response, nil
 }
 
+// classifiedFailure deliberately checks setup defects before broad PromptKit
+// validation/generation wrappers. Provider text is kept only in the typed cause.
+func classifiedFailure(parent, local context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure {
+	failure := &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call failed", Cause: cause}
+	if parent.Err() != nil {
+		failure.Class = structuredoutput.FailureCanceled
+		failure.Reason = outcomes.ReasonCanceled
+		failure.Message = "processing canceled"
+		failure.Cause = parent.Err()
+		return failure
+	}
+	if isSetupFailure(cause) {
+		failure.Reason = outcomes.ReasonSetupFailure
+		failure.Message = "structured model setup failed"
+		return failure
+	}
+	if errors.Is(local.Err(), context.DeadlineExceeded) || errors.Is(cause, context.DeadlineExceeded) {
+		failure.Class = structuredoutput.FailureRecoverable
+		failure.Reason = outcomes.ReasonOperationTimeout
+		failure.Message = "model operation timed out"
+		return failure
+	}
+	if errors.Is(cause, pk.ErrCapacityExceeded) {
+		failure.Class = structuredoutput.FailureRecoverable
+		failure.Reason = outcomes.ReasonCapacityExceeded
+		failure.Message = "model capacity was unavailable"
+		return failure
+	}
+	if errors.Is(cause, pk.ErrTransport) {
+		failure.Class = structuredoutput.FailureRecoverable
+		failure.Reason = outcomes.ReasonTransportFailed
+		failure.Message = "model transport failed"
+		return failure
+	}
+	if errors.Is(cause, pk.ErrMalformedResponse) {
+		failure.Class = structuredoutput.FailureRecoverable
+		failure.Reason = outcomes.ReasonProviderResponseInvalid
+		failure.Message = "model provider response was invalid"
+		return failure
+	}
+	var provider *pk.GenerationError
+	if errors.As(cause, &provider) && provider.StatusCode() != 0 {
+		failure.Class = structuredoutput.FailureRecoverable
+		failure.Reason = outcomes.ReasonProviderFailed
+		failure.Message = "model provider failed"
+		return failure
+	}
+	if result != nil && !result.Validation.IsValid {
+		failure.Class = structuredoutput.FailureRecoverable
+		failure.Kind = structuredoutput.FailureMalformedOutput
+		failure.Reason = outcomes.ReasonInvalidOutputExhausted
+		failure.Message = "structured output remained invalid"
+		return failure
+	}
+	if errors.Is(cause, pk.ErrValidation) {
+		failure.Reason = outcomes.ReasonSetupFailure
+		failure.Message = "structured validation contract failed"
+	}
+	return failure
+}
+
+func isSetupFailure(err error) bool {
+	return errors.Is(err, pk.ErrInvalidConfig) || errors.Is(err, pk.ErrInvalidRequest) ||
+		errors.Is(err, pk.ErrPromptNotFound) || errors.Is(err, pk.ErrProfileNotFound) ||
+		errors.Is(err, pk.ErrPromptLoad) || errors.Is(err, pk.ErrProfileLoad) ||
+		errors.Is(err, pk.ErrArtifactLoad) || errors.Is(err, pk.ErrPromptRender)
+}
+
 func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
 	return structuredoutput.PromptMetadata{
 		PromptID: details.PromptID, PromptVersion: details.PromptVersion,