Finish audit remediation and prepare v0.6.0
This commit is contained in:
@@ -117,7 +117,7 @@ func (r *sourceRepository) GetPromptDefinition(ctx context.Context, id string, v
|
||||
relPath := r.source.displayPath(fullPath)
|
||||
data, err := r.source.readDefinition(fullPath)
|
||||
if err != nil {
|
||||
continue
|
||||
return nil, fmt.Errorf("failed to read prompt definition file %s: %w", relPath, err)
|
||||
}
|
||||
raw, err := decodePromptDefinition(data)
|
||||
if err != nil {
|
||||
|
||||
@@ -460,6 +460,42 @@ type recordingFS struct {
|
||||
opened []string
|
||||
}
|
||||
|
||||
func TestPromptRepositoryReturnsDefinitionReadFailures(t *testing.T) {
|
||||
readErr := errors.New("definition read failed")
|
||||
fsys := &definitionReadFailureFS{
|
||||
FS: fstest.MapFS{
|
||||
"prompts/target.yaml": &fstest.MapFile{Data: []byte("unread")},
|
||||
},
|
||||
target: "prompts/target.yaml",
|
||||
err: readErr,
|
||||
}
|
||||
repo := NewFSRepository(fsys, "prompts")
|
||||
|
||||
definition, err := repo.GetPromptDefinition(context.Background(), "target", "1")
|
||||
if definition != nil || !errors.Is(err, readErr) {
|
||||
t.Fatalf("GetPromptDefinition() = (%#v, %v), want nil and definition read error", definition, err)
|
||||
}
|
||||
if errors.Is(err, ErrPromptDefinitionNotFound) {
|
||||
t.Fatalf("definition read error was classified as absence: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "target.yaml") {
|
||||
t.Fatalf("definition read error lacks source context: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type definitionReadFailureFS struct {
|
||||
fs.FS
|
||||
target string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *definitionReadFailureFS) Open(name string) (fs.File, error) {
|
||||
if name == f.target {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.FS.Open(name)
|
||||
}
|
||||
|
||||
func (f *recordingFS) Open(name string) (fs.File, error) {
|
||||
f.mu.Lock()
|
||||
f.opened = append(f.opened, name)
|
||||
|
||||
@@ -170,34 +170,65 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareExecutionRejectsExcessivelyDeepPreparedSchema(t *testing.T) {
|
||||
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||
def.Validation.SchemaPath = "schema.json"
|
||||
llmClient := &fakeLLM{forbid: true}
|
||||
validator := &recordingValidationPreparer{
|
||||
plan: &recordingPreparedValidation{schemaDocument: excessivelyDeepPreparedJSONValue()},
|
||||
func TestRunnerPreparationRejectsExcessivelyDeepPreparedSchema(t *testing.T) {
|
||||
operations := []struct {
|
||||
name string
|
||||
run func(*Runner, domain.RunRequest) error
|
||||
}{
|
||||
{
|
||||
name: "Prepare",
|
||||
run: func(runner *Runner, request domain.RunRequest) error {
|
||||
_, err := runner.Prepare(context.Background(), request)
|
||||
return err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Run",
|
||||
run: func(runner *Runner, request domain.RunRequest) error {
|
||||
_, err := runner.Run(context.Background(), request)
|
||||
return err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "PrepareExecution",
|
||||
run: func(runner *Runner, request domain.RunRequest) error {
|
||||
_, err := runner.PrepareExecution(context.Background(), request)
|
||||
return err
|
||||
},
|
||||
},
|
||||
}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||
}
|
||||
if llmClient.calls != 0 {
|
||||
t.Fatalf("invalid prepared schema reached generation: %d calls", llmClient.calls)
|
||||
for _, operation := range operations {
|
||||
t.Run(operation.name, func(t *testing.T) {
|
||||
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||
def.Validation.SchemaPath = "schema.json"
|
||||
llmClient := &fakeLLM{forbid: true}
|
||||
validator := &recordingValidationPreparer{
|
||||
plan: &recordingPreparedValidation{schemaDocument: excessivelyDeepPreparedJSONValue()},
|
||||
}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
nil,
|
||||
)
|
||||
|
||||
err := operation.run(runner, domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if !errors.Is(err, ErrValidation) {
|
||||
t.Fatalf("expected ErrValidation, got %v", err)
|
||||
}
|
||||
if llmClient.calls != 0 {
|
||||
t.Fatalf("invalid prepared schema reached generation: %d calls", llmClient.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
@@ -398,6 +399,10 @@ func (r *Runner) structuredOutputFromValidationPlan(
|
||||
}
|
||||
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
|
||||
}
|
||||
schemaDocument, err := jsonvalue.Copy(schemaDocument)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: invalid prepared json_schema schema document: %v", ErrValidation, err)
|
||||
}
|
||||
return structuredOutputSpec(def, schemaDocument), nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user