From 7cc819a589027a25c9259ceb694d3a199daefc88 Mon Sep 17 00:00:00 2001 From: Elizabeth W Date: Mon, 25 May 2026 01:49:37 -0600 Subject: [PATCH] refactor of ingest snapshot workflow --- src/domain/models/IngestSnapshot.ts | 110 +----------- src/domain/models/ingestSnapshot/factories.ts | 64 +++++++ src/domain/models/ingestSnapshot/ops.ts | 156 ++++++++++++++++++ src/domain/models/ingestSnapshot/shared.ts | 102 ++++++++++++ src/domain/models/ingestSnapshot/types.ts | 103 ++++++++++++ src/policies/decideSnapshotIngest.ts | 84 ++-------- src/policies/ingestSnapshot/segments.ts | 42 +++++ src/policies/ingestSnapshot/selection.ts | 48 ++++++ 8 files changed, 529 insertions(+), 180 deletions(-) create mode 100644 src/domain/models/ingestSnapshot/factories.ts create mode 100644 src/domain/models/ingestSnapshot/ops.ts create mode 100644 src/domain/models/ingestSnapshot/shared.ts create mode 100644 src/domain/models/ingestSnapshot/types.ts create mode 100644 src/policies/ingestSnapshot/segments.ts create mode 100644 src/policies/ingestSnapshot/selection.ts diff --git a/src/domain/models/IngestSnapshot.ts b/src/domain/models/IngestSnapshot.ts index 06611f1..d38ce32 100644 --- a/src/domain/models/IngestSnapshot.ts +++ b/src/domain/models/IngestSnapshot.ts @@ -1,106 +1,4 @@ -import { Schema } from "@effect/schema" -import { Either } from "effect" - -const NonEmptyString = Schema.String.pipe( - Schema.filter((value) => value.trim().length > 0), -) - -export const SnapshotIdentity = Schema.String.pipe(Schema.brand("SnapshotIdentity")) -export type SnapshotIdentity = Schema.Schema.Type -export const TaintedBundleLocation = Schema.String.pipe(Schema.brand("TaintedBundleLocation")) -export type TaintedBundleLocation = Schema.Schema.Type -export const TrustedBundleLocation = Schema.String.pipe(Schema.brand("TrustedBundleLocation")) -export type TrustedBundleLocation = Schema.Schema.Type -export const RunIdentity = Schema.String.pipe(Schema.brand("RunIdentity")) -export type RunIdentity = Schema.Schema.Type -export const AstNodeKind = Schema.String.pipe(Schema.brand("AstNodeKind")) -export type AstNodeKind = Schema.Schema.Type -export const RawHash = Schema.String.pipe(Schema.brand("RawHash")) -export type RawHash = Schema.Schema.Type -export const NormalizedHash = Schema.String.pipe(Schema.brand("NormalizedHash")) -export type NormalizedHash = Schema.Schema.Type -export const ShapeHash = Schema.String.pipe(Schema.brand("ShapeHash")) -export type ShapeHash = Schema.Schema.Type -export const TrustedManifestPath = Schema.String.pipe(Schema.brand("TrustedManifestPath")) -export type TrustedManifestPath = Schema.Schema.Type -export const TrustedSegmentsPath = Schema.String.pipe(Schema.brand("TrustedSegmentsPath")) -export type TrustedSegmentsPath = Schema.Schema.Type -export const TrustedCanonicalProjectionPath = Schema.String.pipe(Schema.brand("TrustedCanonicalProjectionPath")) -export type TrustedCanonicalProjectionPath = Schema.Schema.Type -export const TrustedSummaryPath = Schema.String.pipe(Schema.brand("TrustedSummaryPath")) -export type TrustedSummaryPath = Schema.Schema.Type - -export const SourceSpan = Schema.Struct({ StartOffset: Schema.Number, EndOffset: Schema.Number }) -export type SourceSpan = Schema.Schema.Type -export const SnapshotMetadata = Schema.Struct({ ReleaseNotesSource: Schema.NullOr(Schema.String), CollectedAt: Schema.NullOr(Schema.String) }) -export type SnapshotMetadata = Schema.Schema.Type -export const SelectedSnapshot = Schema.Struct({ SnapshotIdentity, BundleLocation: TrustedBundleLocation, SnapshotMetadata: Schema.NullOr(SnapshotMetadata) }) -export type SelectedSnapshot = Schema.Schema.Type -export const SegmentHashes = Schema.Struct({ RawHash, NormalizedHash, ShapeHash }) -export type SegmentHashes = Schema.Schema.Type -export const SegmentRecord = Schema.Struct({ SegmentId: Schema.String, SourceSpan, AstNodeKind, CanonicalSource: Schema.String, Hashes: SegmentHashes }) -export type SegmentRecord = Schema.Schema.Type -export const RunManifest = Schema.Struct({ RunIdentity, SnapshotIdentity, ManifestPath: TrustedManifestPath, SegmentsPath: TrustedSegmentsPath, CanonicalProjectionPath: TrustedCanonicalProjectionPath, SummaryPath: Schema.NullOr(TrustedSummaryPath) }) -export type RunManifest = Schema.Schema.Type - -export type VerifiedPreviousRunManifest = { readonly _tag: "VerifiedPreviousRunManifest"; readonly manifest: RunManifest } -export type TaintedBundleInput = { readonly _tag: "TaintedBundleInput"; readonly location: TaintedBundleLocation } -export type DerivedRunIdentity = { readonly _tag: "DerivedRunIdentity"; readonly value: RunIdentity } -export type RequiredArtifact = "RunManifestArtifact" | "SegmentRecordsArtifact" | "CanonicalProjectionArtifact" -export type IngestFailureReason = - | "BundleNotParseable" - | "RunIdentityCouldNotBeDerived" - | "PreviousRunManifestNotVerified" - | { readonly _tag: "BundleTooLarge"; readonly maxBundleBytes: number } - | { readonly _tag: "ParseBudgetExceeded"; readonly parseBudget: number } - | "NoDeterministicBoundaryProven" - | { readonly _tag: "RequiredArtifactMissing"; readonly artifact: RequiredArtifact } -export type IngestUpstreamSnapshot = { readonly SnapshotIdentity: SnapshotIdentity; readonly BundleInput: TaintedBundleInput; readonly SnapshotMetadata: SnapshotMetadata | null; readonly PreviousRunManifest: VerifiedPreviousRunManifest | null } -export type UpstreamSnapshotIngested = { readonly RunManifest: RunManifest; readonly SegmentRecords: ReadonlyArray; readonly CanonicalProjectionPath: TrustedCanonicalProjectionPath; readonly SummaryPath: TrustedSummaryPath | null } -export type SnapshotIngestHardStopped = { readonly SnapshotIdentity: SnapshotIdentity; readonly Reason: IngestFailureReason } -export type Event = { readonly _tag: "UpstreamSnapshotIngested"; readonly payload: UpstreamSnapshotIngested } -export type Error = { readonly _tag: "SnapshotIngestHardStopped"; readonly payload: SnapshotIngestHardStopped } -export type AwaitingSnapshotSelection = { readonly _tag: "AwaitingSnapshotSelection"; readonly RunIdentityRulesDescription: string; readonly BoundaryRulesDescription: string; readonly RequiredArtifacts: ReadonlyArray; readonly MaxBundleBytes: number; readonly ParseBudget: number } -export type SnapshotReady = { readonly _tag: "SnapshotReady"; readonly SelectedSnapshot: SelectedSnapshot; readonly PreviousRunManifest: VerifiedPreviousRunManifest | null; readonly RequiredArtifacts: ReadonlyArray; readonly MaxBundleBytes: number; readonly ParseBudget: number } -export type DeterministicSegmentsReady = { readonly _tag: "DeterministicSegmentsReady"; readonly RunIdentity: RunIdentity; readonly SelectedSnapshot: SelectedSnapshot; readonly PreviousRunManifest: VerifiedPreviousRunManifest | null; readonly SegmentRecords: ReadonlyArray; readonly BoundaryProofs: ReadonlyArray; readonly RequiredArtifacts: ReadonlyArray } -export type State = AwaitingSnapshotSelection | SnapshotReady | DeterministicSegmentsReady | ({ readonly _tag: "SnapshotIngested" } & UpstreamSnapshotIngested) - -const asBrand = (value: string): T => value as T -export const makeSnapshotIdentity = (value: string): SnapshotIdentity => asBrand(value) -export const makeTaintedBundleLocation = (value: string): TaintedBundleLocation => asBrand(value) -export const makeTrustedBundleLocation = (value: string): TrustedBundleLocation => asBrand(value) -export const makeRunIdentity = (value: string): RunIdentity => asBrand(value) -export const makeAstNodeKind = (value: string): AstNodeKind => asBrand(value) -export const makeRawHash = (value: string): RawHash => asBrand(value) -export const makeNormalizedHash = (value: string): NormalizedHash => asBrand(value) -export const makeShapeHash = (value: string): ShapeHash => asBrand(value) -export const makeTrustedManifestPath = (value: string): TrustedManifestPath => asBrand(value) -export const makeTrustedSegmentsPath = (value: string): TrustedSegmentsPath => asBrand(value) -export const makeTrustedCanonicalProjectionPath = (value: string): TrustedCanonicalProjectionPath => asBrand(value) -export const makeTrustedSummaryPath = (value: string): TrustedSummaryPath => asBrand(value) -export const makeVerifiedPreviousRunManifest = (manifest: RunManifest): VerifiedPreviousRunManifest => ({ _tag: "VerifiedPreviousRunManifest", manifest }) -export const makeTaintedBundleInput = (location: TaintedBundleLocation): TaintedBundleInput => ({ _tag: "TaintedBundleInput", location }) -export const makeDerivedRunIdentity = (value: RunIdentity): DerivedRunIdentity => ({ _tag: "DerivedRunIdentity", value }) - -export const foldFailure = (snapshotIdentity: SnapshotIdentity, reason: IngestFailureReason): Error => ({ _tag: "SnapshotIngestHardStopped", payload: { SnapshotIdentity: snapshotIdentity, Reason: reason } }) -export const parseBundleLocation = (snapshotIdentity: SnapshotIdentity, input: TaintedBundleInput): Either.Either => { - const location = input.location as string - return location.trim().length === 0 || !location.includes("/") ? Either.left(foldFailure(snapshotIdentity, "BundleNotParseable")) : Either.right(makeTrustedBundleLocation(location.trim())) -} -export const applyRunIdentityRules = (selectedSnapshot: SelectedSnapshot): Either.Either => { - const snapshotIdentity = selectedSnapshot.SnapshotIdentity as string - return Schema.is(NonEmptyString)(snapshotIdentity) ? Either.right(makeDerivedRunIdentity(makeRunIdentity(`run:${snapshotIdentity}`))) : Either.left(foldFailure(selectedSnapshot.SnapshotIdentity, "RunIdentityCouldNotBeDerived")) -} -export const validateSegmentRecords = (selectedSnapshot: SelectedSnapshot): Either.Either, Error> => { - const snapshotIdentity = selectedSnapshot.SnapshotIdentity as string - const bundleLocation = selectedSnapshot.BundleLocation as string - if (bundleLocation.includes("too-large")) return Either.left(foldFailure(selectedSnapshot.SnapshotIdentity, { _tag: "BundleTooLarge", maxBundleBytes: 1024 * 1024 })) - if (bundleLocation.includes("budget-exceeded")) return Either.left(foldFailure(selectedSnapshot.SnapshotIdentity, { _tag: "ParseBudgetExceeded", parseBudget: 50_000 })) - return Either.right([{ SegmentId: `${snapshotIdentity}:root`, SourceSpan: { StartOffset: 0, EndOffset: bundleLocation.length }, AstNodeKind: makeAstNodeKind("Program"), CanonicalSource: `// canonical projection for ${snapshotIdentity}`, Hashes: { RawHash: makeRawHash(`raw:${snapshotIdentity}`), NormalizedHash: makeNormalizedHash(`normalized:${snapshotIdentity}`), ShapeHash: makeShapeHash(`shape:${snapshotIdentity}`) } }]) -} -export const validateBoundaryProofs = (snapshotIdentity: SnapshotIdentity, segmentRecords: ReadonlyArray): Either.Either, Error> => segmentRecords[0] ? Either.right([`boundary:${segmentRecords[0].SegmentId}`]) : Either.left(foldFailure(snapshotIdentity, "NoDeterministicBoundaryProven")) -export const validateRequiredArtifacts = (snapshotIdentity: SnapshotIdentity, requiredArtifacts: ReadonlyArray, segmentRecords: ReadonlyArray): Either.Either, Error> => segmentRecords[0] ? Either.right(requiredArtifacts) : Either.left(foldFailure(snapshotIdentity, { _tag: "RequiredArtifactMissing", artifact: requiredArtifacts[0] ?? "RunManifestArtifact" })) -export const deriveRequiredArtifactPaths = (runIdentity: RunIdentity): { readonly ManifestPath: TrustedManifestPath; readonly SegmentsPath: TrustedSegmentsPath; readonly CanonicalProjectionPath: TrustedCanonicalProjectionPath; readonly SummaryPath: TrustedSummaryPath } => { - const basePath = `runs/${runIdentity as string}` - return { ManifestPath: makeTrustedManifestPath(`${basePath}/manifest.json`), SegmentsPath: makeTrustedSegmentsPath(`${basePath}/segments.json`), CanonicalProjectionPath: makeTrustedCanonicalProjectionPath(`${basePath}/canonical.ts`), SummaryPath: makeTrustedSummaryPath(`${basePath}/summary.json`) } -} +export * from "./ingestSnapshot/shared.js" +export * from "./ingestSnapshot/types.js" +export * from "./ingestSnapshot/factories.js" +export * from "./ingestSnapshot/ops.js" diff --git a/src/domain/models/ingestSnapshot/factories.ts b/src/domain/models/ingestSnapshot/factories.ts new file mode 100644 index 0000000..bfb50ba --- /dev/null +++ b/src/domain/models/ingestSnapshot/factories.ts @@ -0,0 +1,64 @@ +import type { + DerivedRunIdentity, + Error, + IngestFailureReason, + TaintedBundleInput, + VerifiedPreviousRunManifest, +} from "./types.js" +import type { + AstNodeKind, + NormalizedHash, + RawHash, + RunIdentity, + RunManifest, + ShapeHash, + SnapshotIdentity, + TaintedBundleLocation, + TrustedBundleLocation, + TrustedCanonicalProjectionPath, + TrustedManifestPath, + TrustedSegmentsPath, + TrustedSummaryPath, +} from "./shared.js" + +const asBrand = (value: string): T => value as T + +export const makeSnapshotIdentity = (value: string): SnapshotIdentity => asBrand(value) +export const makeTaintedBundleLocation = (value: string): TaintedBundleLocation => + asBrand(value) +export const makeTrustedBundleLocation = (value: string): TrustedBundleLocation => + asBrand(value) +export const makeRunIdentity = (value: string): RunIdentity => asBrand(value) +export const makeAstNodeKind = (value: string): AstNodeKind => asBrand(value) +export const makeRawHash = (value: string): RawHash => asBrand(value) +export const makeNormalizedHash = (value: string): NormalizedHash => asBrand(value) +export const makeShapeHash = (value: string): ShapeHash => asBrand(value) +export const makeTrustedManifestPath = (value: string): TrustedManifestPath => + asBrand(value) +export const makeTrustedSegmentsPath = (value: string): TrustedSegmentsPath => + asBrand(value) +export const makeTrustedCanonicalProjectionPath = ( + value: string, +): TrustedCanonicalProjectionPath => asBrand(value) +export const makeTrustedSummaryPath = (value: string): TrustedSummaryPath => + asBrand(value) + +export const makeVerifiedPreviousRunManifest = ( + manifest: RunManifest, +): VerifiedPreviousRunManifest => ({ _tag: "VerifiedPreviousRunManifest", manifest }) + +export const makeTaintedBundleInput = ( + location: TaintedBundleLocation, +): TaintedBundleInput => ({ _tag: "TaintedBundleInput", location }) + +export const makeDerivedRunIdentity = ( + value: RunIdentity, +): DerivedRunIdentity => ({ _tag: "DerivedRunIdentity", value }) + +export const foldFailure = ( + snapshotIdentity: SnapshotIdentity, + reason: IngestFailureReason, +): Error => ({ + _tag: "SnapshotIngestHardStopped", + payload: { SnapshotIdentity: snapshotIdentity, Reason: reason }, +}) diff --git a/src/domain/models/ingestSnapshot/ops.ts b/src/domain/models/ingestSnapshot/ops.ts new file mode 100644 index 0000000..10f4bbe --- /dev/null +++ b/src/domain/models/ingestSnapshot/ops.ts @@ -0,0 +1,156 @@ +import { Either } from "effect" + +import { + isNonEmptyString, + type RunIdentity, + type SegmentRecord, + type SelectedSnapshot, + type SnapshotIdentity, + type TrustedCanonicalProjectionPath, + type TrustedManifestPath, + type TrustedSegmentsPath, + type TrustedSummaryPath, +} from "./shared.js" +import { + foldFailure, + makeAstNodeKind, + makeDerivedRunIdentity, + makeNormalizedHash, + makeRawHash, + makeRunIdentity, + makeShapeHash, + makeTrustedBundleLocation, + makeTrustedCanonicalProjectionPath, + makeTrustedManifestPath, + makeTrustedSegmentsPath, + makeTrustedSummaryPath, +} from "./factories.js" +import type { + DerivedRunIdentity, + Error, + RequiredArtifact, + TaintedBundleInput, +} from "./types.js" + +const parseBundleLocationText = (location: string): string | null => { + const trimmedLocation = location.trim() + return trimmedLocation.length === 0 || !trimmedLocation.includes("/") + ? null + : trimmedLocation +} + +const decideSegmentRecordFailure = ( + selectedSnapshot: SelectedSnapshot, + bundleLocation: string, +): Error | null => { + if (bundleLocation.includes("too-large")) { + return foldFailure(selectedSnapshot.SnapshotIdentity, { + _tag: "BundleTooLarge", + maxBundleBytes: 1024 * 1024, + }) + } + + if (bundleLocation.includes("budget-exceeded")) { + return foldFailure(selectedSnapshot.SnapshotIdentity, { + _tag: "ParseBudgetExceeded", + parseBudget: 50_000, + }) + } + + return null +} + +export const parseBundleLocation = ( + snapshotIdentity: SnapshotIdentity, + input: TaintedBundleInput, +): Either.Either, Error> => { + const location = parseBundleLocationText(input.location as string) + return location === null + ? Either.left(foldFailure(snapshotIdentity, "BundleNotParseable")) + : Either.right(makeTrustedBundleLocation(location)) +} + +export const applyRunIdentityRules = ( + selectedSnapshot: SelectedSnapshot, +): Either.Either => { + const snapshotIdentity = selectedSnapshot.SnapshotIdentity as string + return isNonEmptyString(snapshotIdentity) + ? Either.right(makeDerivedRunIdentity(makeRunIdentity(`run:${snapshotIdentity}`))) + : Either.left( + foldFailure( + selectedSnapshot.SnapshotIdentity, + "RunIdentityCouldNotBeDerived", + ), + ) +} + +export const validateSegmentRecords = ( + selectedSnapshot: SelectedSnapshot, +): Either.Either, Error> => { + const snapshotIdentity = selectedSnapshot.SnapshotIdentity as string + const bundleLocation = selectedSnapshot.BundleLocation as string + const failure = decideSegmentRecordFailure(selectedSnapshot, bundleLocation) + + if (failure) { + return Either.left(failure) + } + + return Either.right([ + { + SegmentId: `${snapshotIdentity}:root`, + SourceSpan: { StartOffset: 0, EndOffset: bundleLocation.length }, + AstNodeKind: makeAstNodeKind("Program"), + CanonicalSource: `// canonical projection for ${snapshotIdentity}`, + Hashes: { + RawHash: makeRawHash(`raw:${snapshotIdentity}`), + NormalizedHash: makeNormalizedHash(`normalized:${snapshotIdentity}`), + ShapeHash: makeShapeHash(`shape:${snapshotIdentity}`), + }, + }, + ]) +} + +export const validateBoundaryProofs = ( + snapshotIdentity: SnapshotIdentity, + segmentRecords: ReadonlyArray, +): Either.Either, Error> => { + const firstSegment = segmentRecords[0] + return firstSegment + ? Either.right([`boundary:${firstSegment.SegmentId}`]) + : Either.left(foldFailure(snapshotIdentity, "NoDeterministicBoundaryProven")) +} + +export const validateRequiredArtifacts = ( + snapshotIdentity: SnapshotIdentity, + requiredArtifacts: ReadonlyArray, + segmentRecords: ReadonlyArray, +): Either.Either, Error> => { + const missingArtifact = segmentRecords[0] ? null : requiredArtifacts[0] + return missingArtifact === null + ? Either.right(requiredArtifacts) + : Either.left( + foldFailure(snapshotIdentity, { + _tag: "RequiredArtifactMissing", + artifact: missingArtifact ?? "RunManifestArtifact", + }), + ) +} + +export const deriveRequiredArtifactPaths = ( + runIdentity: RunIdentity, +): { + readonly ManifestPath: TrustedManifestPath + readonly SegmentsPath: TrustedSegmentsPath + readonly CanonicalProjectionPath: TrustedCanonicalProjectionPath + readonly SummaryPath: TrustedSummaryPath +} => { + const basePath = `runs/${runIdentity as string}` + return { + ManifestPath: makeTrustedManifestPath(`${basePath}/manifest.json`), + SegmentsPath: makeTrustedSegmentsPath(`${basePath}/segments.json`), + CanonicalProjectionPath: makeTrustedCanonicalProjectionPath( + `${basePath}/canonical.ts`, + ), + SummaryPath: makeTrustedSummaryPath(`${basePath}/summary.json`), + } +} diff --git a/src/domain/models/ingestSnapshot/shared.ts b/src/domain/models/ingestSnapshot/shared.ts new file mode 100644 index 0000000..ea18397 --- /dev/null +++ b/src/domain/models/ingestSnapshot/shared.ts @@ -0,0 +1,102 @@ +import { Schema } from "@effect/schema" + +const NonEmptyString = Schema.String.pipe( + Schema.filter((value) => value.trim().length > 0), +) + +export const SnapshotIdentity = Schema.String.pipe(Schema.brand("SnapshotIdentity")) +export type SnapshotIdentity = Schema.Schema.Type + +export const TaintedBundleLocation = Schema.String.pipe( + Schema.brand("TaintedBundleLocation"), +) +export type TaintedBundleLocation = Schema.Schema.Type + +export const TrustedBundleLocation = Schema.String.pipe( + Schema.brand("TrustedBundleLocation"), +) +export type TrustedBundleLocation = Schema.Schema.Type + +export const RunIdentity = Schema.String.pipe(Schema.brand("RunIdentity")) +export type RunIdentity = Schema.Schema.Type + +export const AstNodeKind = Schema.String.pipe(Schema.brand("AstNodeKind")) +export type AstNodeKind = Schema.Schema.Type + +export const RawHash = Schema.String.pipe(Schema.brand("RawHash")) +export type RawHash = Schema.Schema.Type + +export const NormalizedHash = Schema.String.pipe(Schema.brand("NormalizedHash")) +export type NormalizedHash = Schema.Schema.Type + +export const ShapeHash = Schema.String.pipe(Schema.brand("ShapeHash")) +export type ShapeHash = Schema.Schema.Type + +export const TrustedManifestPath = Schema.String.pipe( + Schema.brand("TrustedManifestPath"), +) +export type TrustedManifestPath = Schema.Schema.Type + +export const TrustedSegmentsPath = Schema.String.pipe( + Schema.brand("TrustedSegmentsPath"), +) +export type TrustedSegmentsPath = Schema.Schema.Type + +export const TrustedCanonicalProjectionPath = Schema.String.pipe( + Schema.brand("TrustedCanonicalProjectionPath"), +) +export type TrustedCanonicalProjectionPath = + Schema.Schema.Type + +export const TrustedSummaryPath = Schema.String.pipe( + Schema.brand("TrustedSummaryPath"), +) +export type TrustedSummaryPath = Schema.Schema.Type + +export const SourceSpan = Schema.Struct({ + StartOffset: Schema.Number, + EndOffset: Schema.Number, +}) +export type SourceSpan = Schema.Schema.Type + +export const SnapshotMetadata = Schema.Struct({ + ReleaseNotesSource: Schema.NullOr(Schema.String), + CollectedAt: Schema.NullOr(Schema.String), +}) +export type SnapshotMetadata = Schema.Schema.Type + +export const SelectedSnapshot = Schema.Struct({ + SnapshotIdentity, + BundleLocation: TrustedBundleLocation, + SnapshotMetadata: Schema.NullOr(SnapshotMetadata), +}) +export type SelectedSnapshot = Schema.Schema.Type + +export const SegmentHashes = Schema.Struct({ + RawHash, + NormalizedHash, + ShapeHash, +}) +export type SegmentHashes = Schema.Schema.Type + +export const SegmentRecord = Schema.Struct({ + SegmentId: Schema.String, + SourceSpan, + AstNodeKind, + CanonicalSource: Schema.String, + Hashes: SegmentHashes, +}) +export type SegmentRecord = Schema.Schema.Type + +export const RunManifest = Schema.Struct({ + RunIdentity, + SnapshotIdentity, + ManifestPath: TrustedManifestPath, + SegmentsPath: TrustedSegmentsPath, + CanonicalProjectionPath: TrustedCanonicalProjectionPath, + SummaryPath: Schema.NullOr(TrustedSummaryPath), +}) +export type RunManifest = Schema.Schema.Type + +export const isNonEmptyString = (value: string): boolean => + Schema.is(NonEmptyString)(value) diff --git a/src/domain/models/ingestSnapshot/types.ts b/src/domain/models/ingestSnapshot/types.ts new file mode 100644 index 0000000..86dbbc9 --- /dev/null +++ b/src/domain/models/ingestSnapshot/types.ts @@ -0,0 +1,103 @@ +import type { + RunIdentity, + RunManifest, + SegmentRecord, + SelectedSnapshot, + SnapshotIdentity, + SnapshotMetadata, + TrustedCanonicalProjectionPath, + TrustedSummaryPath, + TaintedBundleLocation, +} from "./shared.js" + +export type VerifiedPreviousRunManifest = { + readonly _tag: "VerifiedPreviousRunManifest" + readonly manifest: RunManifest +} + +export type TaintedBundleInput = { + readonly _tag: "TaintedBundleInput" + readonly location: TaintedBundleLocation +} + +export type DerivedRunIdentity = { + readonly _tag: "DerivedRunIdentity" + readonly value: RunIdentity +} + +export type RequiredArtifact = + | "RunManifestArtifact" + | "SegmentRecordsArtifact" + | "CanonicalProjectionArtifact" + +export type IngestFailureReason = + | "BundleNotParseable" + | "RunIdentityCouldNotBeDerived" + | "PreviousRunManifestNotVerified" + | { readonly _tag: "BundleTooLarge"; readonly maxBundleBytes: number } + | { readonly _tag: "ParseBudgetExceeded"; readonly parseBudget: number } + | "NoDeterministicBoundaryProven" + | { readonly _tag: "RequiredArtifactMissing"; readonly artifact: RequiredArtifact } + +export type IngestUpstreamSnapshot = { + readonly SnapshotIdentity: SnapshotIdentity + readonly BundleInput: TaintedBundleInput + readonly SnapshotMetadata: SnapshotMetadata | null + readonly PreviousRunManifest: VerifiedPreviousRunManifest | null +} + +export type UpstreamSnapshotIngested = { + readonly RunManifest: RunManifest + readonly SegmentRecords: ReadonlyArray + readonly CanonicalProjectionPath: TrustedCanonicalProjectionPath + readonly SummaryPath: TrustedSummaryPath | null +} + +export type SnapshotIngestHardStopped = { + readonly SnapshotIdentity: SnapshotIdentity + readonly Reason: IngestFailureReason +} + +export type Event = { + readonly _tag: "UpstreamSnapshotIngested" + readonly payload: UpstreamSnapshotIngested +} + +export type Error = { + readonly _tag: "SnapshotIngestHardStopped" + readonly payload: SnapshotIngestHardStopped +} + +export type AwaitingSnapshotSelection = { + readonly _tag: "AwaitingSnapshotSelection" + readonly RunIdentityRulesDescription: string + readonly BoundaryRulesDescription: string + readonly RequiredArtifacts: ReadonlyArray + readonly MaxBundleBytes: number + readonly ParseBudget: number +} + +export type SnapshotReady = { + readonly _tag: "SnapshotReady" + readonly SelectedSnapshot: SelectedSnapshot + readonly PreviousRunManifest: VerifiedPreviousRunManifest | null + readonly RequiredArtifacts: ReadonlyArray + readonly MaxBundleBytes: number + readonly ParseBudget: number +} + +export type DeterministicSegmentsReady = { + readonly _tag: "DeterministicSegmentsReady" + readonly RunIdentity: RunIdentity + readonly SelectedSnapshot: SelectedSnapshot + readonly PreviousRunManifest: VerifiedPreviousRunManifest | null + readonly SegmentRecords: ReadonlyArray + readonly BoundaryProofs: ReadonlyArray + readonly RequiredArtifacts: ReadonlyArray +} + +export type State = + | AwaitingSnapshotSelection + | SnapshotReady + | DeterministicSegmentsReady + | ({ readonly _tag: "SnapshotIngested" } & UpstreamSnapshotIngested) diff --git a/src/policies/decideSnapshotIngest.ts b/src/policies/decideSnapshotIngest.ts index 7f2fa1d..68536c5 100644 --- a/src/policies/decideSnapshotIngest.ts +++ b/src/policies/decideSnapshotIngest.ts @@ -3,86 +3,20 @@ import { Either } from "effect" import { type AwaitingSnapshotSelection, type DeterministicSegmentsReady, - type Error, type Event, type IngestUpstreamSnapshot, type RunManifest, - type SnapshotReady, type State, type UpstreamSnapshotIngested, - applyRunIdentityRules, deriveRequiredArtifactPaths, - foldFailure, - makeVerifiedPreviousRunManifest, - parseBundleLocation, - validateBoundaryProofs, - validateRequiredArtifacts, - validateSegmentRecords, } from "../domain/models/IngestSnapshot.js" - -export const validatePreviousRunManifest = ( - manifest: RunManifest, -): Either.Either, Error> => - manifest.ManifestPath && manifest.SegmentsPath && manifest.CanonicalProjectionPath - ? Either.right(makeVerifiedPreviousRunManifest(manifest)) - : Either.left( - foldFailure(manifest.SnapshotIdentity, "PreviousRunManifestNotVerified"), - ) - -export const validateSnapshotSelection = ( - state: State, - command: IngestUpstreamSnapshot, -): Either.Either => { - if (state._tag !== "AwaitingSnapshotSelection") { - return Either.left( - foldFailure(command.SnapshotIdentity, "RunIdentityCouldNotBeDerived"), - ) - } - - return Either.map(parseBundleLocation(command.SnapshotIdentity, command.BundleInput), (bundleLocation) => ({ - _tag: "SnapshotReady" as const, - SelectedSnapshot: { - SnapshotIdentity: command.SnapshotIdentity, - BundleLocation: bundleLocation, - SnapshotMetadata: command.SnapshotMetadata, - }, - PreviousRunManifest: command.PreviousRunManifest, - RequiredArtifacts: state.RequiredArtifacts, - MaxBundleBytes: state.MaxBundleBytes, - ParseBudget: state.ParseBudget, - })) -} - -export const decideSegmentRecords = ( - snapshotReady: SnapshotReady, -): Either.Either => - Either.flatMap(applyRunIdentityRules(snapshotReady.SelectedSnapshot), (derivedRunIdentity) => - Either.flatMap(validateSegmentRecords(snapshotReady.SelectedSnapshot), (segmentRecords) => - Either.flatMap( - validateBoundaryProofs( - snapshotReady.SelectedSnapshot.SnapshotIdentity, - segmentRecords, - ), - (boundaryProofs) => - Either.map( - validateRequiredArtifacts( - snapshotReady.SelectedSnapshot.SnapshotIdentity, - snapshotReady.RequiredArtifacts, - segmentRecords, - ), - () => ({ - _tag: "DeterministicSegmentsReady" as const, - RunIdentity: derivedRunIdentity.value, - SelectedSnapshot: snapshotReady.SelectedSnapshot, - PreviousRunManifest: snapshotReady.PreviousRunManifest, - SegmentRecords: segmentRecords, - BoundaryProofs: boundaryProofs, - RequiredArtifacts: snapshotReady.RequiredArtifacts, - }), - ), - ), - ), - ) +import { + decideSegmentRecords, +} from "./ingestSnapshot/segments.js" +import { + validatePreviousRunManifest, + validateSnapshotSelection, +} from "./ingestSnapshot/selection.js" const toEvent = ( deterministicSegmentsReady: DeterministicSegmentsReady, @@ -113,7 +47,7 @@ const toEvent = ( export const decide = ( state: State, command: IngestUpstreamSnapshot, -): Either.Either => +) => Either.flatMap(validateSnapshotSelection(state, command), (snapshotReady) => Either.map(decideSegmentRecords(snapshotReady), toEvent), ) @@ -140,3 +74,5 @@ export const makeAwaitingSnapshotSelection = ( ParseBudget: 50_000, ...overrides, }) + +export { decideSegmentRecords, validatePreviousRunManifest, validateSnapshotSelection } diff --git a/src/policies/ingestSnapshot/segments.ts b/src/policies/ingestSnapshot/segments.ts new file mode 100644 index 0000000..9a8eca7 --- /dev/null +++ b/src/policies/ingestSnapshot/segments.ts @@ -0,0 +1,42 @@ +import { Either } from "effect" + +import { + type DeterministicSegmentsReady, + type Error, + type SnapshotReady, + applyRunIdentityRules, + validateBoundaryProofs, + validateRequiredArtifacts, + validateSegmentRecords, +} from "../../domain/models/IngestSnapshot.js" + +export const decideSegmentRecords = ( + snapshotReady: SnapshotReady, +): Either.Either => + Either.flatMap(applyRunIdentityRules(snapshotReady.SelectedSnapshot), (derivedRunIdentity) => + Either.flatMap(validateSegmentRecords(snapshotReady.SelectedSnapshot), (segmentRecords) => + Either.flatMap( + validateBoundaryProofs( + snapshotReady.SelectedSnapshot.SnapshotIdentity, + segmentRecords, + ), + (boundaryProofs) => + Either.map( + validateRequiredArtifacts( + snapshotReady.SelectedSnapshot.SnapshotIdentity, + snapshotReady.RequiredArtifacts, + segmentRecords, + ), + () => ({ + _tag: "DeterministicSegmentsReady" as const, + RunIdentity: derivedRunIdentity.value, + SelectedSnapshot: snapshotReady.SelectedSnapshot, + PreviousRunManifest: snapshotReady.PreviousRunManifest, + SegmentRecords: segmentRecords, + BoundaryProofs: boundaryProofs, + RequiredArtifacts: snapshotReady.RequiredArtifacts, + }), + ), + ), + ), + ) diff --git a/src/policies/ingestSnapshot/selection.ts b/src/policies/ingestSnapshot/selection.ts new file mode 100644 index 0000000..9f49f2d --- /dev/null +++ b/src/policies/ingestSnapshot/selection.ts @@ -0,0 +1,48 @@ +import { Either } from "effect" + +import { + type Error, + type IngestUpstreamSnapshot, + type RunManifest, + type SnapshotReady, + type State, + foldFailure, + makeVerifiedPreviousRunManifest, + parseBundleLocation, +} from "../../domain/models/IngestSnapshot.js" + +export const validatePreviousRunManifest = ( + manifest: RunManifest, +): Either.Either, Error> => + manifest.ManifestPath && manifest.SegmentsPath && manifest.CanonicalProjectionPath + ? Either.right(makeVerifiedPreviousRunManifest(manifest)) + : Either.left( + foldFailure(manifest.SnapshotIdentity, "PreviousRunManifestNotVerified"), + ) + +export const validateSnapshotSelection = ( + state: State, + command: IngestUpstreamSnapshot, +): Either.Either => { + if (state._tag !== "AwaitingSnapshotSelection") { + return Either.left( + foldFailure(command.SnapshotIdentity, "RunIdentityCouldNotBeDerived"), + ) + } + + return Either.map( + parseBundleLocation(command.SnapshotIdentity, command.BundleInput), + (bundleLocation) => ({ + _tag: "SnapshotReady" as const, + SelectedSnapshot: { + SnapshotIdentity: command.SnapshotIdentity, + BundleLocation: bundleLocation, + SnapshotMetadata: command.SnapshotMetadata, + }, + PreviousRunManifest: command.PreviousRunManifest, + RequiredArtifacts: state.RequiredArtifacts, + MaxBundleBytes: state.MaxBundleBytes, + ParseBudget: state.ParseBudget, + }), + ) +}