From de046a8f13e227676d79c3aa3b685b049902038d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 28 Jul 2026 13:36:01 -0500 Subject: [PATCH] Keep checkpoints aligned with PromptKit profiles --- docs/internal/llm.md | 11 + docs/internal/state.md | 4 +- docs/operations.md | 7 +- docs/roadmap/implementation.md | 376 ------------------ internal/cli/production_contract_test.go | 9 + internal/cli/run.go | 53 ++- .../spell_catalog_identity_contract_test.go | 21 +- .../framework/llm/checkpoint_fingerprint.go | 14 + internal/framework/llm/promptkit_client.go | 23 +- .../framework/llm/promptkit_client_test.go | 59 +++ .../llm/promptkit_profile_fingerprint.go | 82 ++++ internal/framework/llm/scheduled_client.go | 11 + .../framework/llm/scheduled_client_test.go | 31 ++ internal/framework/promptfs/prompt_fs.go | 6 +- 14 files changed, 308 insertions(+), 399 deletions(-) delete mode 100644 docs/roadmap/implementation.md create mode 100644 internal/framework/llm/checkpoint_fingerprint.go create mode 100644 internal/framework/llm/promptkit_profile_fingerprint.go diff --git a/docs/internal/llm.md b/docs/internal/llm.md index 1799c35..3adb925 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -41,6 +41,17 @@ identity, provider, and model values for manifest use. Successful completion responses and recorded profile manifests identify the adapter provider as `promptkit`. +Before execution, the adapter also contributes a non-secret checkpoint +fingerprint for the effective PromptKit profile source. It combines the +identity of PromptKit's compiled-in profile catalog with a deterministic digest +of every YAML profile in the configured profile directory, or of the configured +profile file. The fingerprint contains neither profile content nor source +paths. It covers both explicit binding profiles and prompt-selected defaults, +so changing a model or other profile setting cannot reuse checkpoints created +under the prior profile source. This cache identity is independent of durable +profile provenance: run manifests continue to list only profiles actually +observed during LLM calls. + ## Shared Provider-Call Limit Production construction creates one PromptKit client and wraps it in one diff --git a/docs/internal/state.md b/docs/internal/state.md index 4e01327..4011d5c 100644 --- a/docs/internal/state.md +++ b/docs/internal/state.md @@ -39,7 +39,9 @@ codecs, loader, and recorder. The CLI constructs a recorder whenever checkpoint recording is enabled and constructs a loader only for a `--resume` invocation. Identity incorporates explicit stable semantic fingerprints collected from prepared modules and validators in addition to configuration, input, -references, runtime overrides, and LLM profiles. +references, runtime overrides, observed LLM profiles, and the LLM runtime's +non-secret effective profile-source identity. A profile source change therefore +causes a cold miss even when the configured profile ID remains unchanged. The serialized `workspace_schema_version` identifiers are frozen wire-compatibility fields; they do not describe a current public state surface. diff --git a/docs/operations.md b/docs/operations.md index fd29516..3e4f29f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -117,9 +117,10 @@ compatible recorded work. A resume request fails when checkpoint recording is disabled. Without **--resume**, a recording-enabled run executes normally and does not load checkpoint state. Compatibility includes the resolved pipeline, input, selected lanes, runtime overrides, reference provenance, LLM-profile -provenance, and prepared-component fingerprints. A changed identity produces a -cold miss; Notarius does not migrate, rewrite, or delete older checkpoint -directories. +provenance, the effective PromptKit profile-source fingerprint, and +prepared-component fingerprints. Changing profile content causes a cold miss +even when its profile ID is unchanged. A changed identity produces a cold miss; +Notarius does not migrate, rewrite, or delete older checkpoint directories. Checkpoint state is confined below an identity-specific path: diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md deleted file mode 100644 index 4fb66ae..0000000 --- a/docs/roadmap/implementation.md +++ /dev/null @@ -1,376 +0,0 @@ -# PromptKit Dependency Migration Implementation Plan - -## Status - -In progress. The dependency, framework adapter, version 4 PromptKit -configuration migration, provider-neutral module prompt-asset support, -provenance alignment, and canonical documentation are implemented; final -repository verification is still planned. - -## Objective - -Replace Notarius's dependency on -`gitea.maximumdirect.net/eric/scriptorium v0.11.1` with -`gitea.maximumdirect.net/eric/promptkit v0.1.0`, and consistently adopt the -PromptKit name at the adapter, configuration, provenance, documentation, and -module-support boundaries. - -The migration is a clean break. Notarius has no production compatibility -requirement for the existing Scriptorium-named configuration or development -artifacts, so the implementation must not add legacy configuration aliases, -deprecated Go APIs, schema migrations, or compatibility shims. - -PromptKit's public engine contract is source-compatible with the subset -Notarius currently consumes. Preserve current prompt content, prompt and schema -identities, structured-completion behavior, LLM scheduling, retry behavior, -debug capture, redaction, and durable output shapes except for the intentional -configuration-version and LLM-provider provenance changes described below. - -## Governing Decisions - -- Pin PromptKit at `v0.1.0` and remove Scriptorium from `go.mod` and `go.sum`. -- Rename the top-level configuration section from `scriptorium` to - `promptkit`. -- Bump the strict file-configuration version from 3 to 4. Version 4 accepts - only the new `promptkit` key; it does not recognize `scriptorium`. -- Rename the internal adapter and its exported-within-`internal` Go API to - PromptKit terminology without transitional aliases. -- Use provider-neutral names for module-owned prompt asset helpers. Those - helpers describe Notarius assets rather than the library that loads them. -- Record `promptkit`, not `scriptorium`, as the LLM adapter/provider value in - completion responses and run manifest profile provenance. -- Do not add the PromptKit package version to checkpoint identity. A library - implementation version is not itself a semantic pipeline input. Existing - prompt, schema, module, reference, profile, runtime-override, and prepared - component fingerprints remain responsible for semantic invalidation. -- Do not adopt PromptKit's optional `ArtifactReader` extension in this work. - Notarius retains ownership of materializing and supplying prompt inputs. -- Do not add native session propagation in this work. PromptKit `v0.1.0` does - not expose the desired request-level session identifier; retain the existing - prompt-variable behavior and update the future roadmap terminology only. -- Accept PromptKit's documented timeout layering: caller context cancellation - is the outer authority, a positive generation timeout supplies an inner - request deadline, zero disables only that generation deadline, and the HTTP - client timeout remains a transport-wide cap. -- PromptKit and Notarius are both GPL-3.0 licensed, so the dependency change - requires no Notarius licensing change. - -## Non-Goals - -- Changing prompt text, message ordering, prompt IDs, schema IDs, or embedded - asset paths. -- Changing public artifact schemas, output bundle contents, run-result - receipts, checkpoint formats, or filesystem layouts. -- Adding new LLM profiles, changing profile precedence, or changing credential - handling. -- Refactoring the transport-neutral `StructuredLLMClient` contract. -- Adding PromptKit features that Notarius does not currently need. -- Preserving version 3 configuration compatibility. - -## Stage 1: Replace the Dependency and Adapter - -Update the Go dependency and the framework adapter as one compilable change. - -### Implementation - -- Replace the Scriptorium requirement with - `gitea.maximumdirect.net/eric/promptkit v0.1.0`, then run `go mod tidy` so - `go.mod` and `go.sum` contain no obsolete Scriptorium module entries. -- In `internal/framework/llm`, replace imports of - `gitea.maximumdirect.net/eric/scriptorium` with PromptKit and rename: - - `scriptorium_client.go` to `promptkit_client.go`; - - `ScriptoriumClientConfig` to `PromptKitClientConfig`; - - `ScriptoriumClient` to `PromptKitClient`; - - `NewScriptoriumClient` to `NewPromptKitClient`; - - `AssetRegistry.ScriptoriumOptions` to - `AssetRegistry.PromptKitOptions`; and - - Scriptorium-named private input, variable, metadata, debug, response, - validation, and error-redaction helpers to PromptKit terminology. -- Do not retain aliases for the old types, constructor, method, filenames, or - private helpers. -- Preserve the existing adapter sequence: - 1. validate the transport-neutral request and output target; - 2. map inputs, variables, metadata, prompt identity, profile identity, and - session prompt variable into a PromptKit `RunRequest`; - 3. call `Prepare` to capture rendered debug material; - 4. call `Run`; - 5. translate PromptKit validation failure into - `contracts.ErrInvalidStructuredOutput`; - 6. decode the structured artifact into the caller's target; and - 7. return transport-neutral response, usage, profile, and debug data. -- Continue wrapping and redacting upstream errors at the same trust boundary. - Update safe wrapper text from Scriptorium to PromptKit without exposing - prompt content, credentials, artifact content, filesystem paths, or raw - upstream payloads. -- Rename the adapter provenance constant and change its value from - `scriptorium` to `promptkit`. -- Preserve PromptKit's cancellation and timeout semantics. Do not introduce - another timeout wrapper in Notarius. -- Update the production composition root in `internal/cli/catalog.go` to - construct `PromptKitClientConfig` and `NewPromptKitClient`. - -### Tests - -- Rename and adapt the existing adapter and asset-registry tests; do not add - tests whose only purpose is to enforce private symbol or filename choices. -- Through the adapter's stable behavior, retain coverage for: - - engine and asset construction failures; - - prompt, schema, input, variable, metadata, and profile forwarding; - - structured-output success and validation failure; - - malformed or empty generated output; - - caller cancellation and configured timeout behavior; - - credential and content redaction; - - raw response and prepared-prompt debug capture; - - token usage, including cached and cache-write token fields; and - - profile manifest recording. -- Use injected PromptKit clients or local deterministic HTTP test servers. - Default tests must remain offline and must not require credentials or paid - model calls. - -### Completion Criteria - -- The framework and production CLI compile against PromptKit only. -- No Go import of the Scriptorium module remains. -- Existing adapter behavior is preserved except that response and manifest - provenance now identify `promptkit`. - -## Stage 2: Introduce Version 4 PromptKit Configuration - -Make the user-visible configuration terminology agree with the dependency and -adapter. - -### Implementation - -- Change `SupportedFileConfigVersion` from 3 to 4. -- In `internal/core/config`, rename: - - `ScriptoriumConfig` to `PromptKitConfig`; - - `FileScriptoriumConfig` to `FilePromptKitConfig`; - - `Config.Scriptorium` to `Config.PromptKit`; - - `FileConfig.Scriptorium` to `FileConfig.PromptKit`; and - - Scriptorium-named validation and application helpers to PromptKit - terminology. -- Change the runtime JSON and file YAML section name from `scriptorium` to - `promptkit`. -- Preserve the two optional fields and their meaning: - - ```yaml - version: 4 - promptkit: - profile_dir: /path/to/profiles - # profile_file: /path/to/profiles.yml - ``` - -- Preserve trimming, clone, apply, effective-configuration, and redaction - behavior for `profile_dir` and `profile_file`. -- Preserve the rule that `profile_dir` and `profile_file` are mutually - exclusive and that an explicitly supplied value must not be empty. -- Update production client construction and explicit-profile validation to - read `cfg.PromptKit`. -- Rename `internal/cli/scriptorium_profiles.go` and its functions to PromptKit - terminology. Preserve the existing profile validation timing and - `errors.Is`-based handling of PromptKit's `ErrProfileNotFound`. -- Keep YAML decoding strict. A version 4 file containing `scriptorium` must - fail as an unknown field. -- Add a targeted version 3 migration diagnostic that instructs users to: - 1. change `version: 3` to `version: 4`; and - 2. rename `scriptorium:` to `promptkit:`. - Do not attempt to decode or automatically rewrite version 3 files. -- Update maintained configuration examples to version 4 and use `promptkit` - wherever profile sources are demonstrated. - -### Tests - -- Update configuration contract tests to establish: - - a minimal version 4 file applies over defaults; - - explicit PromptKit profile directory and profile file values decode and - survive apply, clone, and effective configuration; - - the two profile sources remain mutually exclusive; - - explicit empty values remain invalid; - - unknown fields remain rejected by strict decoding; - - version 4 rejects the removed `scriptorium` key; and - - version 3 produces the actionable migration classification. -- Update CLI contract tests to demonstrate that configured and explicitly - selected PromptKit profiles are validated before pipeline preparation and - that invalid profiles retain the existing process-failure behavior. -- Test configuration behavior at the parser/configuration and CLI composition - boundaries. Do not reproduce PromptKit's own profile parser test matrix. - -### Completion Criteria - -- All runtime and file configuration code uses PromptKit terminology. -- Maintained examples are valid version 4 files. -- The obsolete section is rejected rather than silently accepted or ignored. - -## Stage 3: Make Module Prompt-Asset Support Provider-Neutral - -Remove dependency-brand terminology from module-owned prompt asset -registration without changing the assets themselves. - -### Implementation - -- Across the D&D scene chunker, extractors, NPC normalizer, and registration - support, rename each `scriptorium_assets.go` and corresponding test file to - `prompt_assets.go` and `prompt_assets_test.go`. -- Rename private helpers and values such as `scriptoriumPromptRoot`, - `scriptoriumPromptMetadata`, and equivalent schema registration names to - provider-neutral forms such as `promptAssetRoot` and - `promptAssetMetadata`. -- Retain PromptKit terminology only where code directly calls a PromptKit API, - such as producing `promptkit.Option` values at the framework asset registry - boundary. -- Do not change: - - prompt or schema contents; - - prompt, schema, or asset IDs and versions; - - embedded filesystem paths; - - message ordering or cache-control placement; - - module keys or declared prompt inputs; - - metadata values or content fingerprint algorithms; or - - extraction, normalization, or validation behavior. - -### Tests - -- Update existing module prompt-preparation and registration tests to compile - through the renamed support code. -- Retain the centralized behavioral coverage that all registered production - prompts and schemas can be mounted and prepared with their declared inputs. -- Retain prompt ordering and cache-prefix contract coverage where ordering is - semantically significant. -- Do not add word-presence tests, rename detectors, or private-helper tests. - -### Completion Criteria - -- Module-owned code no longer describes its prompt assets as Scriptorium - assets. -- Prompt and schema fingerprints remain unchanged from the pre-migration - source content. - -## Stage 4: Align Provenance and Checkpoint Behavior - -Make the intentional public provenance change explicit while leaving cache -identity tied to semantic inputs. - -### Implementation - -- Record `Provider: "promptkit"` in every newly observed - `LLMProfileManifest` and `StructuredCompletionResponse` produced by the - production adapter. -- Update manifest, debug, artifact, and assembled-run expectations that - currently identify Scriptorium. -- Do not change checkpoint schemas, layouts, or the - `CheckpointFingerprintProvider` contract. -- Do not introduce a fingerprint containing the PromptKit package name or - version. Dependency implementation identity is not a stable semantic - identity. -- Verify that no accidental prompt, schema, component, reference, profile, or - runtime-override fingerprint changes result from the provider-neutral file - and helper renames. -- If an existing test fixture contains recorded LLM provenance, update only - the adapter/provider value and preserve the profile ID, model, usage, and - remaining run data. - -### Tests - -- At the adapter boundary, assert that a successful completion and recorded - profile manifest report `promptkit`. -- At one assembled run boundary, confirm that the finalized manifest carries - the PromptKit profile provenance observed by the client. -- Retain existing checkpoint identity tests for semantic prompt, module, - profile, reference, and runtime changes. Do not add a test coupled only to - the absence of a dependency-version fingerprint. - -### Completion Criteria - -- New durable and debug provenance consistently identifies PromptKit. -- No checkpoint wire-format or layout change has been introduced. - -## Stage 5: Update Canonical Documentation - -Update documentation in the same change that implements the behavior, following -the repository's canonical ownership rules. - -### Implementation - -- Update `docs/config.md` to own: - - configuration version 4; - - the `promptkit` section and its fields; - - mutual exclusion and validation rules; and - - the version 3-to-4 migration instruction. -- Update `docs/operations.md` to describe provider retries and timeouts as - PromptKit profile behavior and accurately summarize the timeout layers. -- Update `docs/internal/llm.md` to describe: - - the PromptKit-backed adapter; - - transport-neutral request and response mapping; - - asset mounting; - - preparation, execution, validation, redaction, and debug behavior; - - profile provenance; and - - caller, generation, and transport timeout ownership. -- Update `docs/internal/cli.md` to describe PromptKit profile validation and - production client composition. -- Update `docs/development.md` so its task-reading guide refers to the - PromptKit integration. -- Update any maintained examples and nearby navigation links. Configuration - definitions and defaults remain canonical in `docs/config.md`; other - documents should summarize and link rather than repeat them. -- Replace the copied `docs/integrations/pkg-promptkit.md` guide with a concise - Notarius-owned upstream boundary document. It must: - - identify the pinned PromptKit package and supported public boundary used - by Notarius; - - link to PromptKit's canonical upstream package and format documentation; - - describe only integration facts that Notarius relies upon; - - point to `docs/internal/llm.md` for Notarius implementation behavior; and - - contain no copied relative links that resolve only inside the PromptKit - source repository. -- Update `docs/roadmap/future.md` to replace Scriptorium terminology in the - native-session item with PromptKit. Keep the work deferred and state - accurately that PromptKit `v0.1.0` does not yet provide the desired direct - request-level session field. Do not imply that existing prompt-variable - propagation is native provider session support. -- Search all maintained Go, Markdown, YAML, JSON, and module files for - remaining `Scriptorium` or `scriptorium` occurrences. Retain the old name - only where necessary to explain the one-time version 3 migration or - historical context. -- Validate affected relative documentation links. - -### Completion Criteria - -- Current-behavior documentation outside `docs/roadmap` describes the - implemented PromptKit integration only. -- Configuration and operational facts have one canonical owner. -- The integration document is Notarius-specific and does not duplicate or - impersonate upstream package documentation. - -## Stage 6: Repository-Wide Verification - -Complete the migration with focused and repository-wide validation. - -### Required Checks - -Run: - -```sh -git diff --check -go mod tidy -go test ./... -go vet ./... -go build ./cmd/notarius -go test -race ./internal/framework/llm ./internal/cli ./internal/modules/dnd/... -``` - -Also verify: - -- `go.mod` and `go.sum` contain PromptKit and no Scriptorium dependency; -- the repository contains no obsolete Scriptorium Go identifiers or imports; -- any remaining textual use of Scriptorium is limited to intentional migration - or historical explanation; -- maintained configuration examples parse successfully as version 4; -- tests remain deterministic, offline, and independent of real credentials; -- no prompt, schema, artifact, or checkpoint format changed unintentionally; - and -- no secrets, prompt payloads, source content, or private infrastructure paths - were introduced into code, errors, tests, or documentation. - -## Open Questions - -None. The dependency version, compatibility policy, configuration migration, -internal naming, provenance value, checkpoint treatment, session scope, -documentation ownership, and verification boundaries are decided above. diff --git a/internal/cli/production_contract_test.go b/internal/cli/production_contract_test.go index d0fc422..7696479 100644 --- a/internal/cli/production_contract_test.go +++ b/internal/cli/production_contract_test.go @@ -20,6 +20,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes" @@ -390,6 +391,14 @@ func TestProductionLLMClientFactoriesBuildOfflineRuntime(t *testing.T) { if len(manifests) != 0 { t.Fatalf("eager profile manifests = %#v, want none", manifests) } + fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider) + if !ok { + t.Fatalf("production LLM client %T does not provide one profile-source checkpoint fingerprint", client) + } + fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints() + if err != nil || len(fingerprints) != 1 { + t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one profile-source identity", fingerprints, err) + } if _, ok := client.(contracts.LLMProfileManifestProvider); !ok { t.Fatalf("production LLM client %T does not provide profile manifests", client) } diff --git a/internal/cli/run.go b/internal/cli/run.go index b92163f..fee1fa3 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -23,6 +23,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" + frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) @@ -367,6 +368,13 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err)) } + var llmFingerprints []checkpoint.Fingerprint + if effective.Config.Cache.Checkpoints.Enabled { + llmFingerprints, err = llmCheckpointFingerprints(llmClient) + if err != nil { + return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare LLM checkpoint identity: %w", err)) + } + } llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder) prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient}) if err != nil { @@ -380,7 +388,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } - checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume) + checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } @@ -481,6 +489,7 @@ func checkpointHandlersForRun( opts Options, resolved pipeline.ResolvedPipeline, componentFingerprints []pipeline.CheckpointFingerprint, + llmFingerprints []checkpoint.Fingerprint, rawInput []byte, only []string, llmProfiles []artifacts.LLMProfileManifest, @@ -495,13 +504,17 @@ func checkpointHandlersForRun( return pipeline.NoopCheckpointRecorder(), pipeline.NoopCheckpointLoader(), nil } identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{ - Pipeline: resolved, - InputKey: resolved.Input.Module, - RawInputDigest: rawInputDigest(rawInput), - SelectedLanes: only, - RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID), - References: pipeline.ReferenceProvenance(resolved), - ProvenanceFingerprints: append(llmProfileFingerprints(llmProfiles), checkpointIdentityFingerprints(componentFingerprints)...), + Pipeline: resolved, + InputKey: resolved.Input.Module, + RawInputDigest: rawInputDigest(rawInput), + SelectedLanes: only, + RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID), + References: pipeline.ReferenceProvenance(resolved), + ProvenanceFingerprints: combineCheckpointFingerprints( + llmProfileFingerprints(llmProfiles), + llmFingerprints, + checkpointIdentityFingerprints(componentFingerprints), + ), }) if err != nil { return nil, nil, fmt.Errorf("create checkpoint identity: %w", err) @@ -527,6 +540,30 @@ func checkpointHandlersForRun( return recorder, loader, nil } +func llmCheckpointFingerprints(client contracts.StructuredLLMClient) ([]checkpoint.Fingerprint, error) { + provider, ok := client.(frameworkllm.CheckpointFingerprintProvider) + if !ok { + return nil, nil + } + values, err := provider.LLMCheckpointFingerprints() + if err != nil { + return nil, err + } + out := make([]checkpoint.Fingerprint, 0, len(values)) + for _, value := range values { + out = append(out, checkpoint.Fingerprint{Name: value.Name, Value: value.Value}) + } + return out, nil +} + +func combineCheckpointFingerprints(sources ...[]checkpoint.Fingerprint) []checkpoint.Fingerprint { + var out []checkpoint.Fingerprint + for _, source := range sources { + out = append(out, source...) + } + return out +} + func recomputePolicy(resolved pipeline.ResolvedPipeline, requestedStep string) (pipeline.CheckpointExecutionPolicy, error) { requestedStep = strings.TrimSpace(requestedStep) if requestedStep == "" { diff --git a/internal/cli/spell_catalog_identity_contract_test.go b/internal/cli/spell_catalog_identity_contract_test.go index 50560b7..b5f88fc 100644 --- a/internal/cli/spell_catalog_identity_contract_test.go +++ b/internal/cli/spell_catalog_identity_contract_test.go @@ -214,8 +214,9 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t t.Fatal(err) } fingerprints := prepared.CheckpointFingerprints() + llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}} settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()} - recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", false) + recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", false) if err != nil { t.Fatal(err) } @@ -241,7 +242,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t t.Fatal(err) } - _, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true) + _, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", true) if err != nil { t.Fatal(err) } @@ -253,7 +254,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t } changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog") assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName()) - _, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true) + _, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", true) if err != nil { t.Fatal(err) } @@ -265,13 +266,25 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t } changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3") assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName()) - _, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, []byte("same input"), nil, nil, "", "", true) + _, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", true) if err != nil { t.Fatal(err) } if _, decision := mappingLoader.Source(materialized.Input.Module); decision.Reused { t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision) } + + changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}} + _, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", true) + if err != nil { + t.Fatal(err) + } + if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused { + t.Fatalf("changed PromptKit profile source decision = %#v, want cold miss", decision) + } + if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused { + t.Fatalf("changed PromptKit profile normalize decision = %#v, want cold miss", decision) + } } func normalizeSpellCatalogFingerprintName() string { diff --git a/internal/framework/llm/checkpoint_fingerprint.go b/internal/framework/llm/checkpoint_fingerprint.go new file mode 100644 index 0000000..afbe984 --- /dev/null +++ b/internal/framework/llm/checkpoint_fingerprint.go @@ -0,0 +1,14 @@ +package llm + +// CheckpointFingerprint is a stable, non-secret semantic identity contributed +// by the LLM runtime before pipeline execution. +type CheckpointFingerprint struct { + Name string + Value string +} + +// CheckpointFingerprintProvider exposes LLM-runtime identities that must +// participate in checkpoint composition. +type CheckpointFingerprintProvider interface { + LLMCheckpointFingerprints() ([]CheckpointFingerprint, error) +} diff --git a/internal/framework/llm/promptkit_client.go b/internal/framework/llm/promptkit_client.go index 71ff51a..053eabe 100644 --- a/internal/framework/llm/promptkit_client.go +++ b/internal/framework/llm/promptkit_client.go @@ -29,8 +29,10 @@ type PromptKitClientConfig struct { } type PromptKitClient struct { - engine *promptkit.Engine - recorder *LLMProfileRecorder + engine *promptkit.Engine + recorder *LLMProfileRecorder + profileDir string + profileFile string } type LLMProfileRecorder struct { @@ -70,8 +72,10 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) { recorder = NewLLMProfileRecorder() } return &PromptKitClient{ - engine: engine, - recorder: recorder, + engine: engine, + recorder: recorder, + profileDir: strings.TrimSpace(cfg.ProfileDir), + profileFile: strings.TrimSpace(cfg.ProfileFile), }, nil } @@ -261,6 +265,17 @@ func (c *PromptKitClient) LLMProfileManifests() []artifacts.LLMProfileManifest { return c.recorder.Manifests() } +func (c *PromptKitClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint, error) { + if c == nil { + return nil, nil + } + fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile) + if err != nil { + return nil, err + } + return []CheckpointFingerprint{fingerprint}, nil +} + func NewLLMProfileRecorder() *LLMProfileRecorder { return &LLMProfileRecorder{profiles: map[string]artifacts.LLMProfileManifest{}} } diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index ee905da..67af1dc 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -6,6 +6,8 @@ import ( "errors" "io" "net/http" + "os" + "path/filepath" "strings" "sync" "sync/atomic" @@ -119,6 +121,63 @@ func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing. }) } +func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "profiles.yml") + writeProfile := func(model string) { + t.Helper() + content := "id: checkpoint-profile\nendpoint: http://promptkit.test/v1\nmodel: " + model + "\n" + if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + fingerprintFor := func() CheckpointFingerprint { + t.Helper() + client, err := NewPromptKitClient(PromptKitClientConfig{ + Assets: newTestPromptKitAssets(t), + ProfileFile: profilePath, + }) + if err != nil { + t.Fatal(err) + } + values, err := client.LLMCheckpointFingerprints() + if err != nil { + t.Fatal(err) + } + if len(values) != 1 || values[0].Name != promptKitProfileFingerprintName { + t.Fatalf("checkpoint fingerprints = %#v, want one profile-source identity", values) + } + return values[0] + } + + writeProfile("model-one") + first := fingerprintFor() + writeProfile("model-two") + second := fingerprintFor() + if first == second { + t.Fatalf("profile-source fingerprint = %#v for both profile models", first) + } + if strings.Contains(first.Value, profilePath) || strings.Contains(first.Value, "model-one") { + t.Fatalf("profile-source fingerprint exposes source details: %#v", first) + } + + client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t)}) + if err != nil { + t.Fatal(err) + } + copy, err := client.LLMCheckpointFingerprints() + if err != nil { + t.Fatal(err) + } + copy[0].Value = "mutated" + fresh, err := client.LLMCheckpointFingerprints() + if err != nil { + t.Fatal(err) + } + if fresh[0].Value == "mutated" { + t.Fatal("LLMCheckpointFingerprints exposed mutable backing storage") + } +} + func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) { fake := &fakePromptKitLLM{content: `{"ok":true}`} client := newTestPromptKitClient(t, fake) diff --git a/internal/framework/llm/promptkit_profile_fingerprint.go b/internal/framework/llm/promptkit_profile_fingerprint.go new file mode 100644 index 0000000..9570626 --- /dev/null +++ b/internal/framework/llm/promptkit_profile_fingerprint.go @@ -0,0 +1,82 @@ +package llm + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +const ( + promptKitProfileFingerprintName = "promptkit_profile_source" + // The built-in profile catalog is compiled into this pinned PromptKit + // release. Update this identity when the dependency is upgraded. + promptKitBuiltinProfileCatalogID = "promptkit:v0.1.0:builtin-profiles" +) + +func promptKitProfileFingerprint(profileDir, profileFile string) (CheckpointFingerprint, error) { + hasher := sha256.New() + writeFingerprintPart(hasher, []byte(promptKitBuiltinProfileCatalogID)) + + switch { + case strings.TrimSpace(profileFile) != "": + data, err := os.ReadFile(strings.TrimSpace(profileFile)) + if err != nil { + return CheckpointFingerprint{}, fmt.Errorf("read PromptKit profile file for checkpoint identity: %w", err) + } + writeFingerprintPart(hasher, data) + case strings.TrimSpace(profileDir) != "": + digests, err := promptKitProfileFileDigests(strings.TrimSpace(profileDir)) + if err != nil { + return CheckpointFingerprint{}, err + } + for _, digest := range digests { + writeFingerprintPart(hasher, digest) + } + } + + return CheckpointFingerprint{ + Name: promptKitProfileFingerprintName, + Value: "sha256:" + hex.EncodeToString(hasher.Sum(nil)), + }, nil +} + +func promptKitProfileFileDigests(root string) ([][]byte, error) { + var digests [][]byte + err := filepath.WalkDir(root, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + extension := filepath.Ext(entry.Name()) + if extension != ".yaml" && extension != ".yml" { + return nil + } + data, err := os.ReadFile(name) + if err != nil { + return err + } + sum := sha256.Sum256(data) + digests = append(digests, append([]byte(nil), sum[:]...)) + return nil + }) + if err != nil { + return nil, fmt.Errorf("read PromptKit profile directory for checkpoint identity: %w", err) + } + sort.Slice(digests, func(i, j int) bool { + return string(digests[i]) < string(digests[j]) + }) + return digests, nil +} + +func writeFingerprintPart(hasher interface{ Write([]byte) (int, error) }, value []byte) { + length := []byte(fmt.Sprintf("%d:", len(value))) + _, _ = hasher.Write(length) + _, _ = hasher.Write(value) +} diff --git a/internal/framework/llm/scheduled_client.go b/internal/framework/llm/scheduled_client.go index 379f64c..f2610ca 100644 --- a/internal/framework/llm/scheduled_client.go +++ b/internal/framework/llm/scheduled_client.go @@ -53,3 +53,14 @@ func (c *scheduledClient) LLMProfileManifests() []artifacts.LLMProfileManifest { } return provider.LLMProfileManifests() } + +func (c *scheduledClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint, error) { + if c == nil || c.client == nil { + return nil, nil + } + provider, ok := c.client.(CheckpointFingerprintProvider) + if !ok { + return nil, nil + } + return provider.LLMCheckpointFingerprints() +} diff --git a/internal/framework/llm/scheduled_client_test.go b/internal/framework/llm/scheduled_client_test.go index b4b4835..e31f97c 100644 --- a/internal/framework/llm/scheduled_client_test.go +++ b/internal/framework/llm/scheduled_client_test.go @@ -75,6 +75,28 @@ func TestScheduledClientPropagatesSchedulerError(t *testing.T) { } } +func TestScheduledClientPreservesCheckpointFingerprints(t *testing.T) { + scheduler, err := NewScheduler(1) + if err != nil { + t.Fatal(err) + } + inner := &fingerprintedStructuredClient{ + fingerprints: []CheckpointFingerprint{{Name: "profile_source", Value: "sha256:one"}}, + } + client := NewScheduledClient(inner, scheduler) + provider, ok := client.(CheckpointFingerprintProvider) + if !ok { + t.Fatalf("scheduled client %T does not preserve checkpoint fingerprints", client) + } + got, err := provider.LLMCheckpointFingerprints() + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0] != inner.fingerprints[0] { + t.Fatalf("checkpoint fingerprints = %#v, want %#v", got, inner.fingerprints) + } +} + type blockingStructuredClient struct { release chan struct{} inFlight int32 @@ -110,6 +132,15 @@ type errorStructuredClient struct { err error } +type fingerprintedStructuredClient struct { + errorStructuredClient + fingerprints []CheckpointFingerprint +} + +func (c *fingerprintedStructuredClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint, error) { + return append([]CheckpointFingerprint(nil), c.fingerprints...), nil +} + func (c *errorStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { return contracts.StructuredCompletionResponse{}, c.err } diff --git a/internal/framework/promptfs/prompt_fs.go b/internal/framework/promptfs/prompt_fs.go index feccd7b..021413d 100644 --- a/internal/framework/promptfs/prompt_fs.go +++ b/internal/framework/promptfs/prompt_fs.go @@ -11,15 +11,15 @@ import ( "time" ) -// ModulePromptFile maps a module-owned embedded prompt file into the -// PromptKit-visible module prompt directory. +// ModulePromptFile maps a module-owned embedded prompt file into the registered +// module prompt directory. type ModulePromptFile struct { Name string Path string } // SharedPromptFile maps a caller-owned shared prompt file into a module's -// PromptKit-visible sharedassets prompt subdirectory. +// registered sharedassets prompt subdirectory. type SharedPromptFile struct { Name string FS fs.FS