From 486ec62b8e3697cfde73c9342ed42bb38293ff4e Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 14:17:40 +0200 Subject: [PATCH 01/13] feat(webapp): deployment lifecycle telemetry events per build path Replaces the deployment.outcome span with a wide deployment.lifecycle event emitted once per terminal transition (DEPLOYED/FAILED/TIMED_OUT/ CANCELED), backdated createdAt-to-terminal, carrying build path (depot/ native/local_bundle), per-phase durations derived from the persisted timestamp chain, error class, org/project/env, runtime, CLI version and trigger source as attributes. A zero-duration deployment.initialized event at creation provides the funnel denominator for stuck-deployment detection. Events are emitted on ROOT_CONTEXT with the forceRecording attribute: the previous span was started under the ambient request context, where the parent-based sampler drops ~95% of traffic before the force-record check runs. SEMINTATTRS_FORCE_RECORDING is now exported for this. The fail, timeout and finalize transitions now use guarded updateMany writes so exactly one caller commits a terminal status and emits the event; this also stops a late timeout from overwriting DEPLOYED. The cancel path now emits too (it previously recorded nothing). Also: cliVersion is stamped onto WorkerDeployment at initialization from the x-trigger-cli-version header (previously only available post-index via BackgroundWorker, i.e. null for pre-index failures); an optional second OTLP exporter (INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL) mirrors deployment.* spans into a dedicated dataset; the tracer provider is flushed on SIGTERM/SIGINT so shutdowns stop dropping the last batch. --- apps/webapp/app/env.server.ts | 6 + apps/webapp/app/routes/api.v1.deployments.ts | 4 +- .../app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md | 44 ++++ apps/webapp/app/v3/deploymentTelemetry.ts | 95 +++++++++ ...eateDeploymentBackgroundWorkerV4.server.ts | 30 +-- .../app/v3/services/deployment.server.ts | 38 ++++ .../app/v3/services/failDeployment.server.ts | 43 +++- .../v3/services/finalizeDeployment.server.ts | 50 +++-- .../services/initializeDeployment.server.ts | 16 +- .../recordDeploymentLifecycle.server.ts | 191 ++++++++++++++++++ .../recordDeploymentOutcome.server.ts | 50 ----- .../v3/services/timeoutDeployment.server.ts | 46 ++++- apps/webapp/app/v3/tracer.server.ts | 75 ++++++- apps/webapp/test/deploymentTelemetry.test.ts | 83 ++++++++ .../migration.sql | 2 + .../database/prisma/schema.prisma | 3 + 16 files changed, 680 insertions(+), 96 deletions(-) create mode 100644 apps/webapp/app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md create mode 100644 apps/webapp/app/v3/deploymentTelemetry.ts create mode 100644 apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts delete mode 100644 apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts create mode 100644 apps/webapp/test/deploymentTelemetry.test.ts create mode 100644 internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index c63c79d41d6..1e43725d573 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -935,6 +935,12 @@ const EnvironmentSchema = z DISABLE_HTTP_INSTRUMENTATION: BoolEnv.default(false), INTERNAL_OTEL_LOG_EXPORTER_URL: z.string().optional(), + + // Optional second OTLP trace exporter that receives only `deployment.*` + // spans (deployment lifecycle analytics), e.g. a dedicated long-retention + // Axiom dataset. The spans also still flow to the main trace exporter. + INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL: z.string().optional(), + INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_URL: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_ENABLED: z.string().default("0"), diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index 5be291bae27..9014343ccaf 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -42,7 +42,9 @@ export async function action({ request, params }: ActionFunctionArgs) { const service = new InitializeDeploymentService(); try { - const result = await service.call(authenticatedEnv, body.data); + const result = await service.call(authenticatedEnv, body.data, { + cliVersion: request.headers.get("x-trigger-cli-version") ?? undefined, + }); const { deployment, imageRef } = result; const responseBody: InitializeDeploymentResponseBody = { diff --git a/apps/webapp/app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md b/apps/webapp/app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md new file mode 100644 index 00000000000..4ca68437892 --- /dev/null +++ b/apps/webapp/app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md @@ -0,0 +1,44 @@ +# Deployment telemetry attributes + +`deploymentTelemetry.ts` is the single owner of these names. `deployment.lifecycle` +(one wide event per terminal transition, span backdated createdAt → terminal) and +`deployment.initialized` (zero-duration funnel event at creation) are emitted by +`services/recordDeploymentLifecycle.server.ts`. Axiom queries, dashboards, and +monitors reference these names — treat renames as breaking changes. + +| Attribute | Events | Values / notes | +| -------------------------------- | ---------------- | -------------------------------------------------------------------------- | +| `$trigger.org.id` | both | Organization id | +| `$trigger.project.id` | both | Project id | +| `$trigger.project.ref` | both | Project external ref (`proj_…`) | +| `$trigger.env.id` | both | Environment id | +| `$trigger.env.type` | both | `PRODUCTION` / `STAGING` / `PREVIEW` / `DEVELOPMENT` | +| `deployment.id` | both | Deployment friendly id — dedup key (`arg_max(_time, *) by deployment.id`) | +| `deployment.version` | both | Deployment version, e.g. `20260825.3` | +| `deployment.status` | both | lifecycle: terminal status; initialized: initial status (`PENDING`/`BUILDING`) | +| `deployment.success` | lifecycle | `status === "DEPLOYED"`. CANCELED is excluded from failure rates | +| `deployment.build_path` | both | `depot` / `native` / `local_bundle` (rare `--local-build` lands in `depot`) | +| `deployment.worker_type` | both | `V1` / `MANAGED` (run engine) | +| `deployment.runtime` | both | `node` / `node-22` / `bun` / … | +| `deployment.runtime_version` | lifecycle | Set at indexing; null for pre-index failures | +| `deployment.cli_version` | both | From `x-trigger-cli-version` at init; null for pre-column history | +| `deployment.triggered_via` | both | e.g. `cli`, GitHub/Vercel integrations | +| `deployment.commit_sha` | lifecycle | From git meta when present | +| `deployment.error.name` | lifecycle | Error class from `errorData` (`TimeoutError`, build errors, …) | +| `deployment.error.message` | lifecycle | Human-readable failure reason | +| `deployment.canceled_reason` | lifecycle | Only on CANCELED | +| `deployment.duration.total_ms` | lifecycle | createdAt → terminal (also the span's own duration) | +| `deployment.duration.queue_ms` | lifecycle | createdAt → startedAt; ≈0 when created directly in BUILDING (depot) | +| `deployment.duration.install_ms` | lifecycle | startedAt → installedAt; build-server paths only (depot never sets it) | +| `deployment.duration.building_ms`| lifecycle | (installedAt ?? startedAt) → builtAt | +| `deployment.duration.deploying_ms`| lifecycle | builtAt → terminal; for depot dominated by the server-side registry push | + +The span's `_time` is the deployment's **createdAt**, so a TIMED_OUT event lands +backdated by up to the full deploy timeout (~23 min at current defaults) — monitors +must use windows longer than the max timeout or they will systematically miss the +stuck deployments they exist to catch. + +Phase durations are omitted (not zero) when a boundary timestamp is missing — +timestamp chains are path-shaped. Compare only shared phases across build paths; +`total_ms` excludes local-bundle's pre-init client work (esbuild + upload) until +the CLI passes client timings. diff --git a/apps/webapp/app/v3/deploymentTelemetry.ts b/apps/webapp/app/v3/deploymentTelemetry.ts new file mode 100644 index 00000000000..d4a519d2b96 --- /dev/null +++ b/apps/webapp/app/v3/deploymentTelemetry.ts @@ -0,0 +1,95 @@ +import { BuildServerMetadata } from "@trigger.dev/core/v3"; + +// Attribute names for the deployment telemetry events (see +// DEPLOYMENT_TELEMETRY_ATTRIBUTES.md next to this file). This module is the single owner of these names — Axiom +// queries, dashboards, and monitors reference them, so treat renames as +// breaking changes. +export const DeploymentTelemetryAttributes = { + ORG_ID: "$trigger.org.id", + PROJECT_ID: "$trigger.project.id", + PROJECT_REF: "$trigger.project.ref", + ENV_ID: "$trigger.env.id", + ENV_TYPE: "$trigger.env.type", + DEPLOYMENT_ID: "deployment.id", + VERSION: "deployment.version", + STATUS: "deployment.status", + SUCCESS: "deployment.success", + BUILD_PATH: "deployment.build_path", + WORKER_TYPE: "deployment.worker_type", + RUNTIME: "deployment.runtime", + RUNTIME_VERSION: "deployment.runtime_version", + CLI_VERSION: "deployment.cli_version", + TRIGGERED_VIA: "deployment.triggered_via", + COMMIT_SHA: "deployment.commit_sha", + ERROR_NAME: "deployment.error.name", + ERROR_MESSAGE: "deployment.error.message", + CANCELED_REASON: "deployment.canceled_reason", + DURATION_TOTAL_MS: "deployment.duration.total_ms", + DURATION_QUEUE_MS: "deployment.duration.queue_ms", + DURATION_INSTALL_MS: "deployment.duration.install_ms", + DURATION_BUILDING_MS: "deployment.duration.building_ms", + DURATION_DEPLOYING_MS: "deployment.duration.deploying_ms", +} as const; + +export type DeploymentBuildPath = "local_bundle" | "native" | "depot"; + +/** + * Classifies which build path produced a deployment, from its persisted + * metadata. Everything that is not a native-build-server deployment falls into + * the depot bucket — including rare `--local-build` deploys, whose flag is not + * persisted. `externalBuildData` is NOT usable as a depot signal: init writes a + * placeholder (`"-"` fields) for every path. + */ +export function deriveBuildPath(buildServerMetadata: unknown): DeploymentBuildPath { + const metadata = BuildServerMetadata.safeParse(buildServerMetadata); + + if (metadata.success && metadata.data.isNativeBuild) { + return metadata.data.fromBundle ? "local_bundle" : "native"; + } + + return "depot"; +} + +export type DeploymentTimestamps = { + createdAt: Date; + startedAt?: Date | null; + installedAt?: Date | null; + builtAt?: Date | null; +}; + +export type DeploymentDurations = { + totalMs: number; + queueMs?: number; + installMs?: number; + buildingMs?: number; + deployingMs?: number; +}; + +/** + * Derives per-phase durations from the persisted timestamp chain + * (createdAt → startedAt → installedAt → builtAt → terminal). Chains are + * path-shaped: depot never sets installedAt (the /progress route is + * build-server-only) and PENDING-skipping deploys have queue ≈ 0 — each phase + * is emitted only when both of its boundary timestamps exist and are ordered. + */ +export function deriveDeploymentDurations( + timestamps: DeploymentTimestamps, + terminalAt: Date +): DeploymentDurations { + const { createdAt, startedAt, installedAt, builtAt } = timestamps; + const buildingFrom = installedAt ?? startedAt; + + return { + totalMs: Math.max(terminalAt.getTime() - createdAt.getTime(), 0), + queueMs: msBetween(createdAt, startedAt), + installMs: msBetween(startedAt, installedAt), + buildingMs: msBetween(buildingFrom, builtAt), + deployingMs: msBetween(builtAt, terminalAt), + }; +} + +function msBetween(from?: Date | null, to?: Date | null): number | undefined { + if (!from || !to) return undefined; + const ms = to.getTime() - from.getTime(); + return ms >= 0 ? ms : undefined; +} diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index d09707a0e83..60016d52f2c 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -18,7 +18,7 @@ import { } from "./createBackgroundWorker.server"; import { findOrCreateBackgroundWorker } from "./createDeploymentBackgroundWorkerV4/findOrCreateBackgroundWorker.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server"; import { env } from "~/env.server"; import { webhookPrisma } from "~/db.server"; @@ -298,6 +298,12 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { error: Error, environment: AuthenticatedEnvironment ) { + const failedAt = new Date(); + const errorData = { + name: error.name, + message: error.message, + }; + // Guarded BUILDING → FAILED transition, symmetric with the BUILDING → DEPLOYING // transition in `call()`. With idempotent retries, two attempts can run side-by-side; // without the predicate, one attempt's failure could downgrade the deployment after @@ -309,11 +315,8 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { }, data: { status: "FAILED", - failedAt: new Date(), - errorData: { - name: error.name, - message: error.message, - }, + failedAt, + errorData, buildEnvVars: Prisma.DbNull, }, }); @@ -332,13 +335,16 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { // BUILDING → DEPLOYING transition. await TimeoutDeploymentService.dequeue(deployment.id, this._prisma); - recordDeploymentOutcome({ + recordDeploymentLifecycle({ status: "FAILED", - deploymentFriendlyId: deployment.friendlyId, - organizationId: environment.organizationId, - projectId: environment.projectId, - environmentId: environment.id, - environmentType: environment.type, + deployment: { ...deployment, status: "FAILED", failedAt, errorData }, + environment: { + organizationId: environment.organizationId, + projectId: environment.projectId, + projectRef: environment.project.externalRef, + environmentId: environment.id, + environmentType: environment.type, + }, reason: error.message, }); } diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index 7a891ae4f61..1fdbb36fd25 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -9,6 +9,7 @@ import { type DeploymentEvent, } from "@trigger.dev/core/v3"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; +import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server"; import { env } from "~/env.server"; import { createRemoteImageBuild } from "../remoteImageBuilder.server"; import { FINAL_DEPLOYMENT_STATUSES } from "./failDeployment.server"; @@ -238,6 +239,8 @@ export class DeploymentService extends BaseService { if (result.count === 0) { return errAsync({ type: "deployment_cannot_be_cancelled" as const }); } + // Fire-and-forget: telemetry must never affect the cancel result. + void this.#recordCanceledLifecycle(deployment.id); return okAsync({ deployment }); }); @@ -474,6 +477,41 @@ export class DeploymentService extends BaseService { ); } + // The cancel path only carries a narrow row selection, so re-fetch the full + // row (post-update, status already CANCELED) for the lifecycle event. + async #recordCanceledLifecycle(deploymentId: string) { + try { + const canceled = await this._prisma.workerDeployment.findFirst({ + where: { id: deploymentId }, + include: { + environment: { + include: { + project: { + select: { id: true, organizationId: true, externalRef: true }, + }, + }, + }, + }, + }); + + if (!canceled || canceled.status !== "CANCELED") return; + + recordDeploymentLifecycle({ + status: "CANCELED", + deployment: canceled, + environment: { + organizationId: canceled.environment.project.organizationId, + projectId: canceled.environment.project.id, + projectRef: canceled.environment.project.externalRef, + environmentId: canceled.environmentId, + environmentType: canceled.environment.type, + }, + }); + } catch (error) { + logger.error("Failed to record canceled deployment lifecycle", { deploymentId, error }); + } + } + private getDeployment(environmentId: string, friendlyId: string) { return fromPromise( this._prisma.workerDeployment.findFirst({ diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index cb5c622b7b2..670eafb68fe 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -5,7 +5,7 @@ import { Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server"; export const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [ "CANCELED", @@ -41,25 +41,50 @@ export class FailDeploymentService extends BaseService { return; } - const failedDeployment = await this._prisma.workerDeployment.update({ + const failedAt = new Date(); + + // Guarded transition: a concurrent finalize/timeout/cancel can win between + // the check above and this write; the predicate makes exactly one caller + // commit the terminal status (and emit the lifecycle event). + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: { notIn: FINAL_DEPLOYMENT_STATUSES }, }, data: { status: "FAILED", - failedAt: new Date(), + failedAt, errorData: params.error, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Worker deployment reached a final state concurrently, skipping fail", { + id: deployment.id, + friendlyId, + }); + return; + } + + const failedDeployment = { + ...deployment, + status: "FAILED" as const, + failedAt, + errorData: params.error, + buildEnvVars: null, + }; + + recordDeploymentLifecycle({ status: "FAILED", - deploymentFriendlyId: friendlyId, - organizationId: authenticatedEnv.organizationId, - projectId: authenticatedEnv.projectId, - environmentId: authenticatedEnv.id, - environmentType: authenticatedEnv.type, + deployment: failedDeployment, + environment: { + organizationId: authenticatedEnv.organizationId, + projectId: authenticatedEnv.projectId, + projectRef: authenticatedEnv.project.externalRef, + environmentId: authenticatedEnv.id, + environmentType: authenticatedEnv.type, + }, reason: params.error.message, }); diff --git a/apps/webapp/app/v3/services/finalizeDeployment.server.ts b/apps/webapp/app/v3/services/finalizeDeployment.server.ts index 51f5b1e37c4..acc9b30eb8d 100644 --- a/apps/webapp/app/v3/services/finalizeDeployment.server.ts +++ b/apps/webapp/app/v3/services/finalizeDeployment.server.ts @@ -10,7 +10,7 @@ import { projectPubSub } from "./projectPubSub.server"; import { FailDeploymentService } from "./failDeployment.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server"; import { engine } from "../runEngine.server"; import { tryCatch } from "@trigger.dev/core"; import { externalDeploymentCacheInstance } from "~/services/externalDeploymentCacheInstance.server"; @@ -66,28 +66,54 @@ export class FinalizeDeploymentService extends BaseService { } const imageDigest = validatedImageDigest(body.imageDigest); - - // Link the deployment with the background worker - const finalizedDeployment = await this._prisma.workerDeployment.update({ + const deployedAt = new Date(); + const imageReference = imageDigest + ? `${deployment.imageReference}@${imageDigest}` + : deployment.imageReference; + + // Guarded transition: a concurrent timeout/fail/cancel can win between the + // status check above and this write; the predicate makes exactly one caller + // commit the terminal status (and emit the lifecycle event). It also stops + // a late timeout from overwriting DEPLOYED. + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: "DEPLOYING", }, data: { status: "DEPLOYED", - deployedAt: new Date(), + deployedAt, // Only add the digest, if any - imageReference: imageDigest ? `${deployment.imageReference}@${imageDigest}` : undefined, + imageReference: imageDigest ? imageReference : undefined, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Worker deployment left DEPLOYING concurrently, skipping finalize", { + id: deployment.id, + }); + throw new ServiceValidationError("Worker deployment is not in DEPLOYING status"); + } + + const finalizedDeployment = { + ...deployment, + status: "DEPLOYED" as const, + deployedAt, + imageReference, + buildEnvVars: null, + }; + + recordDeploymentLifecycle({ status: "DEPLOYED", - deploymentFriendlyId: deployment.friendlyId, - organizationId: authenticatedEnv.organizationId, - projectId: authenticatedEnv.projectId, - environmentId: authenticatedEnv.id, - environmentType: authenticatedEnv.type, + deployment: finalizedDeployment, + environment: { + organizationId: authenticatedEnv.organizationId, + projectId: authenticatedEnv.projectId, + projectRef: authenticatedEnv.project.externalRef, + environmentId: authenticatedEnv.id, + environmentType: authenticatedEnv.type, + }, }); const deploymentService = new DeploymentService(); diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index ee55d8bd8d6..7a182a8379b 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -16,6 +16,7 @@ import { getDeploymentImageRef } from "../getDeploymentImageRef.server"; import { tryCatch } from "@trigger.dev/core"; import { getRegistryConfig } from "../registryConfig.server"; import { DeploymentService } from "./deployment.server"; +import { recordDeploymentInitialized } from "./recordDeploymentLifecycle.server"; import { createDeploymentWithNextVersion } from "./initializeDeployment/createDeploymentWithNextVersion.server"; import { cancelSupersededDeployments, @@ -56,7 +57,8 @@ export type InitializeDeploymentResult = export class InitializeDeploymentService extends BaseService { public async call( environment: AuthenticatedEnvironment, - payload: InitializeDeploymentRequestBody + payload: InitializeDeploymentRequestBody, + options?: { cliVersion?: string } ): Promise { return this.traceWithEnv("call", environment, async (span) => { if (payload.externalId) { @@ -386,12 +388,24 @@ export class InitializeDeploymentService extends BaseService { commitSHA: payload.gitMeta?.commitSha ?? undefined, externalId: payload.externalId, runtime: payload.runtime ?? environment.project.defaultRuntime ?? undefined, + cliVersion: options?.cliVersion, triggeredVia: payload.triggeredVia ?? undefined, startedAt: initialStatus === "BUILDING" ? new Date() : undefined, }; } ); + recordDeploymentInitialized({ + deployment, + environment: { + organizationId: environment.organizationId, + projectId: environment.projectId, + projectRef: environment.project.externalRef, + environmentId: environment.id, + environmentType: environment.type, + }, + }); + const timeoutMs = deployment.status === "PENDING" ? env.DEPLOY_QUEUE_TIMEOUT_MS : env.DEPLOY_TIMEOUT_MS; diff --git a/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts b/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts new file mode 100644 index 00000000000..3a87267d291 --- /dev/null +++ b/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts @@ -0,0 +1,191 @@ +import { ROOT_CONTEXT, SpanStatusCode } from "@opentelemetry/api"; +import { type WorkerDeployment, type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { logger } from "~/services/logger.server"; +import { SEMINTATTRS_FORCE_RECORDING, tracer } from "~/v3/tracer.server"; +import { + DeploymentTelemetryAttributes as ATTRS, + deriveBuildPath, + deriveDeploymentDurations, +} from "~/v3/deploymentTelemetry"; + +type TerminalDeploymentStatus = Extract< + WorkerDeploymentStatus, + "DEPLOYED" | "FAILED" | "TIMED_OUT" | "CANCELED" +>; + +type LifecycleDeployment = Pick< + WorkerDeployment, + | "friendlyId" + | "version" + | "type" + | "status" + | "createdAt" + | "startedAt" + | "installedAt" + | "builtAt" + | "deployedAt" + | "failedAt" + | "canceledAt" + | "canceledReason" + | "buildServerMetadata" + | "errorData" + | "runtime" + | "runtimeVersion" + | "cliVersion" + | "triggeredVia" + | "commitSHA" +>; + +type EnvironmentInfo = { + organizationId?: string; + projectId?: string; + projectRef?: string; + environmentId?: string; + environmentType?: string; +}; + +/** + * Records a deployment's terminal transition as a single wide + * `deployment.lifecycle` span, backdated to span the deployment's real + * lifetime (createdAt → terminal) and carrying per-phase durations as + * attributes. This is THE per-deployment analytics event: build-path + * comparison dashboards and monitors are built on it (see + * ../deploymentTelemetry.ts and DEPLOYMENT_TELEMETRY_ATTRIBUTES.md for the attribute contract). + * + * Call exactly once per terminal transition, only after a guarded status + * write confirmed this caller won the transition. Emitted on ROOT_CONTEXT + * with forceRecording so the trace sampler can never drop it. Never throws. + */ +export function recordDeploymentLifecycle(params: { + status: TerminalDeploymentStatus; + deployment: LifecycleDeployment; + environment: EnvironmentInfo; + reason?: string; +}): void { + try { + const { status, deployment, environment, reason } = params; + + const isFailure = status === "FAILED" || status === "TIMED_OUT"; + const terminalAt = + deployment.deployedAt ?? deployment.failedAt ?? deployment.canceledAt ?? new Date(); + const durations = deriveDeploymentDurations(deployment, terminalAt); + const errorData = parseErrorData(deployment.errorData); + + const span = tracer.startSpan( + "deployment.lifecycle", + { + startTime: deployment.createdAt, + attributes: { + [SEMINTATTRS_FORCE_RECORDING]: true, + [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_REF]: environment.projectRef, + [ATTRS.ENV_ID]: environment.environmentId, + [ATTRS.ENV_TYPE]: environment.environmentType, + [ATTRS.DEPLOYMENT_ID]: deployment.friendlyId, + [ATTRS.VERSION]: deployment.version, + [ATTRS.STATUS]: status, + [ATTRS.SUCCESS]: status === "DEPLOYED", + [ATTRS.BUILD_PATH]: deriveBuildPath(deployment.buildServerMetadata), + [ATTRS.WORKER_TYPE]: deployment.type, + [ATTRS.RUNTIME]: deployment.runtime ?? undefined, + [ATTRS.RUNTIME_VERSION]: deployment.runtimeVersion ?? undefined, + [ATTRS.CLI_VERSION]: deployment.cliVersion ?? undefined, + [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, + [ATTRS.COMMIT_SHA]: deployment.commitSHA ?? undefined, + [ATTRS.ERROR_NAME]: isFailure ? errorData?.name : undefined, + [ATTRS.ERROR_MESSAGE]: isFailure ? reason ?? errorData?.message : undefined, + [ATTRS.CANCELED_REASON]: deployment.canceledReason ?? undefined, + [ATTRS.DURATION_TOTAL_MS]: durations.totalMs, + [ATTRS.DURATION_QUEUE_MS]: durations.queueMs, + [ATTRS.DURATION_INSTALL_MS]: durations.installMs, + [ATTRS.DURATION_BUILDING_MS]: durations.buildingMs, + [ATTRS.DURATION_DEPLOYING_MS]: durations.deployingMs, + }, + }, + ROOT_CONTEXT + ); + + // CANCELED is deliberately not an error: it is excluded from failure + // rates and tracked as its own volume. + if (isFailure) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: reason ?? errorData?.message, + }); + } + + span.end(terminalAt); + } catch (error) { + logger.debug("recordDeploymentLifecycle failed", { + deploymentFriendlyId: params.deployment.friendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + +/** + * Records a deployment's creation as a zero-duration `deployment.initialized` + * event, the funnel counterpart to `deployment.lifecycle`: an initialized + * deployment with no lifecycle event after a few hours is either stuck + * non-terminal or hit an emission bug. Never throws. + */ +export function recordDeploymentInitialized(params: { + deployment: Pick< + WorkerDeployment, + | "friendlyId" + | "version" + | "type" + | "status" + | "createdAt" + | "buildServerMetadata" + | "runtime" + | "cliVersion" + | "triggeredVia" + >; + environment: EnvironmentInfo; +}): void { + try { + const { deployment, environment } = params; + + const span = tracer.startSpan( + "deployment.initialized", + { + startTime: deployment.createdAt, + attributes: { + [SEMINTATTRS_FORCE_RECORDING]: true, + [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_REF]: environment.projectRef, + [ATTRS.ENV_ID]: environment.environmentId, + [ATTRS.ENV_TYPE]: environment.environmentType, + [ATTRS.DEPLOYMENT_ID]: deployment.friendlyId, + [ATTRS.VERSION]: deployment.version, + [ATTRS.STATUS]: deployment.status, + [ATTRS.BUILD_PATH]: deriveBuildPath(deployment.buildServerMetadata), + [ATTRS.WORKER_TYPE]: deployment.type, + [ATTRS.RUNTIME]: deployment.runtime ?? undefined, + [ATTRS.CLI_VERSION]: deployment.cliVersion ?? undefined, + [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, + }, + }, + ROOT_CONTEXT + ); + + span.end(deployment.createdAt); + } catch (error) { + logger.debug("recordDeploymentInitialized failed", { + deploymentFriendlyId: params.deployment.friendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + +function parseErrorData(errorData: unknown): { name?: string; message?: string } | undefined { + if (!errorData || typeof errorData !== "object") return undefined; + const record = errorData as Record; + return { + name: typeof record.name === "string" ? record.name : undefined, + message: typeof record.message === "string" ? record.message : undefined, + }; +} diff --git a/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts b/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts deleted file mode 100644 index e66a7a6a9f9..00000000000 --- a/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { SpanStatusCode } from "@opentelemetry/api"; -import { type WorkerDeploymentStatus } from "@trigger.dev/database"; -import { logger } from "~/services/logger.server"; -import { tracer } from "~/v3/tracer.server"; - -type TerminalDeploymentStatus = Extract< - WorkerDeploymentStatus, - "DEPLOYED" | "FAILED" | "TIMED_OUT" ->; - -/** - * Records a deployment's terminal status as a `deployment.outcome` span so - * deploy success/failure is queryable from traces (no DB read). Call after each - * terminal-status write. Org/project/env are best-effort; never throws. - */ -export function recordDeploymentOutcome(params: { - status: TerminalDeploymentStatus; - deploymentFriendlyId: string; - organizationId?: string; - projectId?: string; - environmentId?: string; - environmentType?: string; - reason?: string; -}): void { - try { - const span = tracer.startSpan("deployment.outcome", { - attributes: { - "$trigger.org.id": params.organizationId, - "$trigger.project.id": params.projectId, - "$trigger.env.id": params.environmentId, - "$trigger.env.type": params.environmentType, - "deployment.outcome.status": params.status, - "deployment.outcome.success": params.status === "DEPLOYED", - "deployment.outcome.deployment_id": params.deploymentFriendlyId, - "deployment.outcome.reason": params.reason, - }, - }); - - if (params.status !== "DEPLOYED") { - span.setStatus({ code: SpanStatusCode.ERROR, message: params.reason }); - } - - span.end(); - } catch (error) { - logger.debug("recordDeploymentOutcome failed", { - deploymentFriendlyId: params.deploymentFriendlyId, - error: error instanceof Error ? error.message : String(error), - }); - } -} diff --git a/apps/webapp/app/v3/services/timeoutDeployment.server.ts b/apps/webapp/app/v3/services/timeoutDeployment.server.ts index 5e417a7863b..4fe8e0178d1 100644 --- a/apps/webapp/app/v3/services/timeoutDeployment.server.ts +++ b/apps/webapp/app/v3/services/timeoutDeployment.server.ts @@ -5,7 +5,7 @@ import { commonWorker } from "../commonWorker.server"; import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { type PrismaClientOrTransaction } from "~/db.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server"; export class TimeoutDeploymentService extends BaseService { public async call(id: string, fromStatus: string, errorMessage: string) { @@ -38,25 +38,51 @@ export class TimeoutDeploymentService extends BaseService { return; } - const timedOutDeployment = await this._prisma.workerDeployment.update({ + const failedAt = new Date(); + const errorData = { message: errorMessage, name: "TimeoutError" }; + + // Guarded transition: keeps the fromStatus check atomic with the write, so + // a concurrent finalize/fail/cancel can't be overwritten by a late timeout + // (and exactly one caller emits the lifecycle event). + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: deployment.status, }, data: { status: "TIMED_OUT", - failedAt: new Date(), - errorData: { message: errorMessage, name: "TimeoutError" }, + failedAt, + errorData, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Deployment moved out of the expected state concurrently, skipping timeout", { + id: deployment.id, + fromStatus, + }); + return; + } + + const timedOutDeployment = { + ...deployment, + status: "TIMED_OUT" as const, + failedAt, + errorData, + buildEnvVars: null, + }; + + recordDeploymentLifecycle({ status: "TIMED_OUT", - deploymentFriendlyId: deployment.friendlyId, - organizationId: deployment.environment.project.organizationId, - projectId: deployment.environment.projectId, - environmentId: deployment.environmentId, - environmentType: deployment.environment.type, + deployment: timedOutDeployment, + environment: { + organizationId: deployment.environment.project.organizationId, + projectId: deployment.environment.projectId, + projectRef: deployment.environment.project.externalRef, + environmentId: deployment.environmentId, + environmentType: deployment.environment.type, + }, reason: errorMessage, }); diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index cbf9a937c03..b1a940a0d33 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -36,7 +36,9 @@ import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto"; import { BatchSpanProcessor, ParentBasedSampler, + type ReadableSpan, type Sampler, + type Span as SdkTraceSpan, SamplingDecision, type SamplingResult, SimpleSpanProcessor, @@ -69,7 +71,7 @@ import { metricsRegister } from "~/metrics.server"; import { collectDatabaseClientMetrics } from "~/utils/databaseMetrics.server"; import { performance } from "node:perf_hooks"; -const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; +export const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; export const DATASOURCE_CONTEXT_KEY = createContextKey("trigger.db.datasource"); @@ -89,6 +91,31 @@ class DatasourceAttributeSpanProcessor implements SpanProcessor { } } +// Mirrors spans whose name matches a prefix into a second exporter (e.g. the +// dedicated deployment-events dataset) without removing them from the main +// exporter's stream. +class SpanNamePrefixMirrorProcessor implements SpanProcessor { + constructor( + private readonly _inner: SpanProcessor, + private readonly _prefix: string + ) {} + + onStart(span: SdkTraceSpan, parentContext: Context): void { + this._inner.onStart(span, parentContext); + } + onEnd(span: ReadableSpan): void { + if (span.name.startsWith(this._prefix)) { + this._inner.onEnd(span); + } + } + shutdown(): Promise { + return this._inner.shutdown(); + } + forceFlush(): Promise { + return this._inner.forceFlush(); + } +} + class CustomWebappSampler implements Sampler { constructor(private readonly _baseSampler: Sampler) {} @@ -270,6 +297,30 @@ function setupTelemetry() { } } + if (env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL) { + const deploymentEventExporter = new OTLPTraceExporter({ + url: env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL, + timeoutMillis: 15_000, + headers: parseInternalDeploymentEventHeaders() ?? {}, + }); + + spanProcessors.push( + new SpanNamePrefixMirrorProcessor( + new BatchSpanProcessor(deploymentEventExporter, { + maxExportBatchSize: 64, + scheduledDelayMillis: 1000, + exportTimeoutMillis: 30000, + maxQueueSize: 2048, + }), + "deployment." + ) + ); + + console.log( + `🔦 Tracer: deployment-event exporter enabled to ${env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL}` + ); + } + const ratioSampler = new TraceIdRatioBasedSampler(samplingRate); const provider = new NodeTracerProvider({ @@ -341,6 +392,15 @@ function setupTelemetry() { instrumentations, }); + // closeServer only closes express and lets the process drain, so a flush + // here has time to run — without it every webapp shutdown drops the last + // batch of spans (up to 1s of scheduledDelayMillis backlog). + const flushOnShutdown = () => { + provider.forceFlush().catch(() => {}); + }; + process.once("SIGTERM", flushOnShutdown); + process.once("SIGINT", flushOnShutdown); + return { tracer: provider.getTracer("trigger.dev", "3.3.12"), logger: logs.getLogger("trigger.dev", "3.3.12"), @@ -874,6 +934,19 @@ function parseInternalTraceHeaders(): Record | undefined { } } +function parseInternalDeploymentEventHeaders(): Record | undefined { + try { + return env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS + ? (JSON.parse(env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS) as Record< + string, + string + >) + : undefined; + } catch { + return; + } +} + function parseInternalMetricsHeaders(): Record | undefined { try { return env.INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS diff --git a/apps/webapp/test/deploymentTelemetry.test.ts b/apps/webapp/test/deploymentTelemetry.test.ts new file mode 100644 index 00000000000..e673af44373 --- /dev/null +++ b/apps/webapp/test/deploymentTelemetry.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import { deriveBuildPath, deriveDeploymentDurations } from "~/v3/deploymentTelemetry"; + +describe("deriveBuildPath", () => { + it("classifies fromBundle native builds as local_bundle", () => { + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: true })).toBe("local_bundle"); + }); + + it("classifies native builds without fromBundle as native", () => { + expect(deriveBuildPath({ isNativeBuild: true })).toBe("native"); + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: false })).toBe("native"); + }); + + it("classifies everything else as depot", () => { + expect(deriveBuildPath(null)).toBe("depot"); + expect(deriveBuildPath(undefined)).toBe("depot"); + expect(deriveBuildPath({})).toBe("depot"); + expect(deriveBuildPath({ buildId: "depot-build-id" })).toBe("depot"); + expect(deriveBuildPath({ isNativeBuild: false })).toBe("depot"); + // fromBundle alone (skewed writer) must not count as local_bundle + expect(deriveBuildPath({ fromBundle: true })).toBe("depot"); + expect(deriveBuildPath("garbage")).toBe("depot"); + }); +}); + +describe("deriveDeploymentDurations", () => { + const t = (seconds: number) => new Date(1_700_000_000_000 + seconds * 1000); + + it("derives all phases for the full build-server chain", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(10), installedAt: t(40), builtAt: t(100) }, + t(130) + ); + + expect(durations).toEqual({ + totalMs: 130_000, + queueMs: 10_000, + installMs: 30_000, + buildingMs: 60_000, + deployingMs: 30_000, + }); + }); + + it("omits install and measures building from startedAt when installedAt is missing (depot)", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(0), installedAt: null, builtAt: t(90) }, + t(120) + ); + + expect(durations).toEqual({ + totalMs: 120_000, + queueMs: 0, + installMs: undefined, + buildingMs: 90_000, + deployingMs: 30_000, + }); + }); + + it("omits phases whose boundaries are missing (failed before building)", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(5), installedAt: null, builtAt: null }, + t(20) + ); + + expect(durations).toEqual({ + totalMs: 20_000, + queueMs: 5_000, + installMs: undefined, + buildingMs: undefined, + deployingMs: undefined, + }); + }); + + it("never returns negative durations on clock skew", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(10), startedAt: t(5), installedAt: null, builtAt: null }, + t(3) + ); + + expect(durations.totalMs).toBe(0); + expect(durations.queueMs).toBeUndefined(); + }); +}); diff --git a/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql new file mode 100644 index 00000000000..ba3697f38da --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql @@ -0,0 +1,2 @@ +-- Stamp the initiating CLI version on deployments at initialization +ALTER TABLE "public"."WorkerDeployment" ADD COLUMN IF NOT EXISTS "cliVersion" TEXT; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index a77890930b1..fd2a248eba4 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2265,6 +2265,9 @@ model WorkerDeployment { runtime String? runtimeVersion String? + /// CLI version that initiated the deploy (x-trigger-cli-version), stamped at + /// initialization so pre-index failures are attributable to a CLI version. + cliVersion String? imageReference String? imagePlatform String @default("linux/amd64") From 8244227af45b90e24c58cf06ef20bbc492bc2619 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 16:18:59 +0200 Subject: [PATCH 02/13] chore: format recordDeploymentLifecycle.server.ts --- apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts b/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts index 3a87267d291..f327e5ef1f8 100644 --- a/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts +++ b/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts @@ -94,7 +94,7 @@ export function recordDeploymentLifecycle(params: { [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, [ATTRS.COMMIT_SHA]: deployment.commitSHA ?? undefined, [ATTRS.ERROR_NAME]: isFailure ? errorData?.name : undefined, - [ATTRS.ERROR_MESSAGE]: isFailure ? reason ?? errorData?.message : undefined, + [ATTRS.ERROR_MESSAGE]: isFailure ? (reason ?? errorData?.message) : undefined, [ATTRS.CANCELED_REASON]: deployment.canceledReason ?? undefined, [ATTRS.DURATION_TOTAL_MS]: durations.totalMs, [ATTRS.DURATION_QUEUE_MS]: durations.queueMs, From cb09bb198ca2f843dab02d3c13eade5892a371b2 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 16:25:42 +0200 Subject: [PATCH 03/13] fix: bound the notIn status filter in failDeployment --- apps/webapp/app/v3/services/failDeployment.server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index 670eafb68fe..facf02a9d67 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -1,7 +1,7 @@ import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { BaseService } from "./baseService.server"; import { logger } from "~/services/logger.server"; -import { Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { boundedIn, Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { DeploymentService } from "./deployment.server"; @@ -49,7 +49,7 @@ export class FailDeploymentService extends BaseService { const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, - status: { notIn: FINAL_DEPLOYMENT_STATUSES }, + status: { notIn: boundedIn(FINAL_DEPLOYMENT_STATUSES) }, }, data: { status: "FAILED", From 6919caab59beab6cea5fbd316704b9c626ea4662 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 16:37:50 +0200 Subject: [PATCH 04/13] fix: return the post-update row from failDeployment --- .../app/v3/services/failDeployment.server.ts | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index facf02a9d67..95dd1f97c2b 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -67,13 +67,20 @@ export class FailDeploymentService extends BaseService { return; } - const failedDeployment = { - ...deployment, - status: "FAILED" as const, - failedAt, - errorData: params.error, - buildEnvVars: null, - }; + // Re-read after the guarded write: the row can gain phase timestamps + // between the initial read and the update, and callers expect the + // post-update row. + const failedDeployment = await this._prisma.workerDeployment.findFirst({ + where: { id: deployment.id }, + }); + + if (!failedDeployment) { + logger.error("Worker deployment disappeared after fail transition", { + id: deployment.id, + friendlyId, + }); + return; + } recordDeploymentLifecycle({ status: "FAILED", From 4038e6fe2dd924c63b39d3b60f7f79ee5933baa3 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 17:12:47 +0200 Subject: [PATCH 05/13] refactor: fold attribute docs into the telemetry constants, neverthrow cancel emission Replaces the standalone DEPLOYMENT_TELEMETRY_ATTRIBUTES.md with short comments on the DeploymentTelemetryAttributes keys, and chains the canceled-lifecycle emission through the cancel ResultAsync pipeline instead of a fire-and-forget promise. --- .../app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md | 44 ------------------- apps/webapp/app/v3/deploymentTelemetry.ts | 32 ++++++++++++-- .../app/v3/services/deployment.server.ts | 28 +++++++----- .../recordDeploymentLifecycle.server.ts | 4 +- 4 files changed, 48 insertions(+), 60 deletions(-) delete mode 100644 apps/webapp/app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md diff --git a/apps/webapp/app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md b/apps/webapp/app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md deleted file mode 100644 index 4ca68437892..00000000000 --- a/apps/webapp/app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md +++ /dev/null @@ -1,44 +0,0 @@ -# Deployment telemetry attributes - -`deploymentTelemetry.ts` is the single owner of these names. `deployment.lifecycle` -(one wide event per terminal transition, span backdated createdAt → terminal) and -`deployment.initialized` (zero-duration funnel event at creation) are emitted by -`services/recordDeploymentLifecycle.server.ts`. Axiom queries, dashboards, and -monitors reference these names — treat renames as breaking changes. - -| Attribute | Events | Values / notes | -| -------------------------------- | ---------------- | -------------------------------------------------------------------------- | -| `$trigger.org.id` | both | Organization id | -| `$trigger.project.id` | both | Project id | -| `$trigger.project.ref` | both | Project external ref (`proj_…`) | -| `$trigger.env.id` | both | Environment id | -| `$trigger.env.type` | both | `PRODUCTION` / `STAGING` / `PREVIEW` / `DEVELOPMENT` | -| `deployment.id` | both | Deployment friendly id — dedup key (`arg_max(_time, *) by deployment.id`) | -| `deployment.version` | both | Deployment version, e.g. `20260825.3` | -| `deployment.status` | both | lifecycle: terminal status; initialized: initial status (`PENDING`/`BUILDING`) | -| `deployment.success` | lifecycle | `status === "DEPLOYED"`. CANCELED is excluded from failure rates | -| `deployment.build_path` | both | `depot` / `native` / `local_bundle` (rare `--local-build` lands in `depot`) | -| `deployment.worker_type` | both | `V1` / `MANAGED` (run engine) | -| `deployment.runtime` | both | `node` / `node-22` / `bun` / … | -| `deployment.runtime_version` | lifecycle | Set at indexing; null for pre-index failures | -| `deployment.cli_version` | both | From `x-trigger-cli-version` at init; null for pre-column history | -| `deployment.triggered_via` | both | e.g. `cli`, GitHub/Vercel integrations | -| `deployment.commit_sha` | lifecycle | From git meta when present | -| `deployment.error.name` | lifecycle | Error class from `errorData` (`TimeoutError`, build errors, …) | -| `deployment.error.message` | lifecycle | Human-readable failure reason | -| `deployment.canceled_reason` | lifecycle | Only on CANCELED | -| `deployment.duration.total_ms` | lifecycle | createdAt → terminal (also the span's own duration) | -| `deployment.duration.queue_ms` | lifecycle | createdAt → startedAt; ≈0 when created directly in BUILDING (depot) | -| `deployment.duration.install_ms` | lifecycle | startedAt → installedAt; build-server paths only (depot never sets it) | -| `deployment.duration.building_ms`| lifecycle | (installedAt ?? startedAt) → builtAt | -| `deployment.duration.deploying_ms`| lifecycle | builtAt → terminal; for depot dominated by the server-side registry push | - -The span's `_time` is the deployment's **createdAt**, so a TIMED_OUT event lands -backdated by up to the full deploy timeout (~23 min at current defaults) — monitors -must use windows longer than the max timeout or they will systematically miss the -stuck deployments they exist to catch. - -Phase durations are omitted (not zero) when a boundary timestamp is missing — -timestamp chains are path-shaped. Compare only shared phases across build paths; -`total_ms` excludes local-bundle's pre-init client work (esbuild + upload) until -the CLI passes client timings. diff --git a/apps/webapp/app/v3/deploymentTelemetry.ts b/apps/webapp/app/v3/deploymentTelemetry.ts index d4a519d2b96..21915c76a49 100644 --- a/apps/webapp/app/v3/deploymentTelemetry.ts +++ b/apps/webapp/app/v3/deploymentTelemetry.ts @@ -1,33 +1,57 @@ import { BuildServerMetadata } from "@trigger.dev/core/v3"; -// Attribute names for the deployment telemetry events (see -// DEPLOYMENT_TELEMETRY_ATTRIBUTES.md next to this file). This module is the single owner of these names — Axiom -// queries, dashboards, and monitors reference them, so treat renames as -// breaking changes. +/** + * Attribute names for the `deployment.lifecycle` and `deployment.initialized` + * telemetry events (emitted by services/recordDeploymentLifecycle.server.ts). + * This module is the single owner of these names — external queries, + * dashboards, and monitors reference them, so treat renames as breaking. + * + * Query gotchas: dedup with `arg_max(_time, *) by deployment.id` (job retries + * can double-emit); the span's `_time` is the deployment's createdAt, so a + * TIMED_OUT event lands backdated by up to the full deploy timeout — monitor + * windows must exceed it; phase durations are omitted (not zero) when a + * boundary timestamp is missing, and `total_ms` excludes local-bundle's + * pre-init client work (esbuild + upload) until the CLI reports timings. + */ export const DeploymentTelemetryAttributes = { ORG_ID: "$trigger.org.id", PROJECT_ID: "$trigger.project.id", + // Project external ref ("proj_…") PROJECT_REF: "$trigger.project.ref", ENV_ID: "$trigger.env.id", + // PRODUCTION / STAGING / PREVIEW / DEVELOPMENT ENV_TYPE: "$trigger.env.type", + // Deployment friendly id — the dedup key DEPLOYMENT_ID: "deployment.id", VERSION: "deployment.version", + // lifecycle: terminal status; initialized: initial status (PENDING/BUILDING) STATUS: "deployment.status", + // status === DEPLOYED; CANCELED is excluded from failure rates SUCCESS: "deployment.success", + // depot / native / local_bundle (see deriveBuildPath) BUILD_PATH: "deployment.build_path", + // V1 / MANAGED (run engine) WORKER_TYPE: "deployment.worker_type", RUNTIME: "deployment.runtime", + // Set at indexing; null for pre-index failures RUNTIME_VERSION: "deployment.runtime_version", + // From x-trigger-cli-version at init; null for pre-column history CLI_VERSION: "deployment.cli_version", TRIGGERED_VIA: "deployment.triggered_via", COMMIT_SHA: "deployment.commit_sha", + // error.* only on FAILED/TIMED_OUT; CANCELED uses canceled_reason ERROR_NAME: "deployment.error.name", ERROR_MESSAGE: "deployment.error.message", CANCELED_REASON: "deployment.canceled_reason", + // createdAt → terminal (also the span's own duration) DURATION_TOTAL_MS: "deployment.duration.total_ms", + // createdAt → startedAt; ≈0 when created directly in BUILDING (depot) DURATION_QUEUE_MS: "deployment.duration.queue_ms", + // startedAt → installedAt; build-server paths only (depot never sets it) DURATION_INSTALL_MS: "deployment.duration.install_ms", + // (installedAt ?? startedAt) → builtAt DURATION_BUILDING_MS: "deployment.duration.building_ms", + // builtAt → terminal; for depot dominated by the server-side registry push DURATION_DEPLOYING_MS: "deployment.duration.deploying_ms", } as const; diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index 1fdbb36fd25..f674999fa8c 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -239,8 +239,6 @@ export class DeploymentService extends BaseService { if (result.count === 0) { return errAsync({ type: "deployment_cannot_be_cancelled" as const }); } - // Fire-and-forget: telemetry must never affect the cancel result. - void this.#recordCanceledLifecycle(deployment.id); return okAsync({ deployment }); }); @@ -253,6 +251,14 @@ export class DeploymentService extends BaseService { return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(cancelDeployment) + .andThen(({ deployment }) => + this.#recordCanceledLifecycle(deployment.id) + .orElse((error) => { + logger.error("Failed to record canceled deployment lifecycle", { error }); + return okAsync(undefined); + }) + .map(() => ({ deployment })) + ) .andThen(({ deployment }) => this.appendToEventLog(deployment.environment.project, deployment, [ { @@ -479,9 +485,9 @@ export class DeploymentService extends BaseService { // The cancel path only carries a narrow row selection, so re-fetch the full // row (post-update, status already CANCELED) for the lifecycle event. - async #recordCanceledLifecycle(deploymentId: string) { - try { - const canceled = await this._prisma.workerDeployment.findFirst({ + #recordCanceledLifecycle(deploymentId: string) { + return fromPromise( + this._prisma.workerDeployment.findFirst({ where: { id: deploymentId }, include: { environment: { @@ -492,8 +498,12 @@ export class DeploymentService extends BaseService { }, }, }, - }); - + }), + (error) => ({ + type: "other" as const, + cause: error, + }) + ).map((canceled) => { if (!canceled || canceled.status !== "CANCELED") return; recordDeploymentLifecycle({ @@ -507,9 +517,7 @@ export class DeploymentService extends BaseService { environmentType: canceled.environment.type, }, }); - } catch (error) { - logger.error("Failed to record canceled deployment lifecycle", { deploymentId, error }); - } + }); } private getDeployment(environmentId: string, friendlyId: string) { diff --git a/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts b/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts index f327e5ef1f8..0a5b70e0e37 100644 --- a/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts +++ b/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts @@ -49,8 +49,8 @@ type EnvironmentInfo = { * `deployment.lifecycle` span, backdated to span the deployment's real * lifetime (createdAt → terminal) and carrying per-phase durations as * attributes. This is THE per-deployment analytics event: build-path - * comparison dashboards and monitors are built on it (see - * ../deploymentTelemetry.ts and DEPLOYMENT_TELEMETRY_ATTRIBUTES.md for the attribute contract). + * comparison dashboards and monitors are built on it (the attribute contract + * lives in ../deploymentTelemetry.ts). * * Call exactly once per terminal transition, only after a guarded status * write confirmed this caller won the transition. Emitted on ROOT_CONTEXT From 9cd1c6cfbe5fdb9979ea5da2d14b3899e2c6deff Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 17:48:43 +0200 Subject: [PATCH 06/13] chore: trim inline comments to one-liners --- apps/webapp/app/env.server.ts | 4 +--- apps/webapp/app/v3/deploymentTelemetry.ts | 17 ++++++-------- .../app/v3/services/deployment.server.ts | 3 +-- .../app/v3/services/failDeployment.server.ts | 8 ++----- .../v3/services/finalizeDeployment.server.ts | 5 +---- .../recordDeploymentLifecycle.server.ts | 22 +++++++------------ .../v3/services/timeoutDeployment.server.ts | 4 +--- apps/webapp/app/v3/tracer.server.ts | 8 ++----- .../migration.sql | 1 - .../database/prisma/schema.prisma | 3 +-- 10 files changed, 24 insertions(+), 51 deletions(-) diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 1e43725d573..e1c701d37bd 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -936,9 +936,7 @@ const EnvironmentSchema = z INTERNAL_OTEL_LOG_EXPORTER_URL: z.string().optional(), - // Optional second OTLP trace exporter that receives only `deployment.*` - // spans (deployment lifecycle analytics), e.g. a dedicated long-retention - // Axiom dataset. The spans also still flow to the main trace exporter. + // Second trace exporter receiving only `deployment.*` spans; they still flow to the main one INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL: z.string().optional(), INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_URL: z.string().optional(), diff --git a/apps/webapp/app/v3/deploymentTelemetry.ts b/apps/webapp/app/v3/deploymentTelemetry.ts index 21915c76a49..eae4bbeebd9 100644 --- a/apps/webapp/app/v3/deploymentTelemetry.ts +++ b/apps/webapp/app/v3/deploymentTelemetry.ts @@ -58,11 +58,10 @@ export const DeploymentTelemetryAttributes = { export type DeploymentBuildPath = "local_bundle" | "native" | "depot"; /** - * Classifies which build path produced a deployment, from its persisted - * metadata. Everything that is not a native-build-server deployment falls into - * the depot bucket — including rare `--local-build` deploys, whose flag is not - * persisted. `externalBuildData` is NOT usable as a depot signal: init writes a - * placeholder (`"-"` fields) for every path. + * Everything that is not a native-build-server deployment falls into the depot + * bucket, including rare `--local-build` deploys (their flag is not persisted). + * `externalBuildData` is NOT a usable depot signal: init writes a placeholder + * for every path. */ export function deriveBuildPath(buildServerMetadata: unknown): DeploymentBuildPath { const metadata = BuildServerMetadata.safeParse(buildServerMetadata); @@ -90,11 +89,9 @@ export type DeploymentDurations = { }; /** - * Derives per-phase durations from the persisted timestamp chain - * (createdAt → startedAt → installedAt → builtAt → terminal). Chains are - * path-shaped: depot never sets installedAt (the /progress route is - * build-server-only) and PENDING-skipping deploys have queue ≈ 0 — each phase - * is emitted only when both of its boundary timestamps exist and are ordered. + * Timestamp chains are path-shaped (e.g. depot never sets installedAt), so + * each phase is derived only when both of its boundary timestamps exist and + * are ordered. */ export function deriveDeploymentDurations( timestamps: DeploymentTimestamps, diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index f674999fa8c..8d8f4488860 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -483,8 +483,7 @@ export class DeploymentService extends BaseService { ); } - // The cancel path only carries a narrow row selection, so re-fetch the full - // row (post-update, status already CANCELED) for the lifecycle event. + // The cancel chain only carries a narrow row selection, so re-fetch the full row #recordCanceledLifecycle(deploymentId: string) { return fromPromise( this._prisma.workerDeployment.findFirst({ diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index 95dd1f97c2b..5a97eea710a 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -43,9 +43,7 @@ export class FailDeploymentService extends BaseService { const failedAt = new Date(); - // Guarded transition: a concurrent finalize/timeout/cancel can win between - // the check above and this write; the predicate makes exactly one caller - // commit the terminal status (and emit the lifecycle event). + // Guarded: a concurrent terminal transition can win after the check above const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, @@ -67,9 +65,7 @@ export class FailDeploymentService extends BaseService { return; } - // Re-read after the guarded write: the row can gain phase timestamps - // between the initial read and the update, and callers expect the - // post-update row. + // Re-read: the row can gain phase timestamps between the read and the guarded write const failedDeployment = await this._prisma.workerDeployment.findFirst({ where: { id: deployment.id }, }); diff --git a/apps/webapp/app/v3/services/finalizeDeployment.server.ts b/apps/webapp/app/v3/services/finalizeDeployment.server.ts index acc9b30eb8d..d4ff97b8f6a 100644 --- a/apps/webapp/app/v3/services/finalizeDeployment.server.ts +++ b/apps/webapp/app/v3/services/finalizeDeployment.server.ts @@ -71,10 +71,7 @@ export class FinalizeDeploymentService extends BaseService { ? `${deployment.imageReference}@${imageDigest}` : deployment.imageReference; - // Guarded transition: a concurrent timeout/fail/cancel can win between the - // status check above and this write; the predicate makes exactly one caller - // commit the terminal status (and emit the lifecycle event). It also stops - // a late timeout from overwriting DEPLOYED. + // Guarded: stops a concurrent transition (e.g. a late timeout) from double-committing const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, diff --git a/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts b/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts index 0a5b70e0e37..830713c6487 100644 --- a/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts +++ b/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts @@ -46,15 +46,11 @@ type EnvironmentInfo = { /** * Records a deployment's terminal transition as a single wide - * `deployment.lifecycle` span, backdated to span the deployment's real - * lifetime (createdAt → terminal) and carrying per-phase durations as - * attributes. This is THE per-deployment analytics event: build-path - * comparison dashboards and monitors are built on it (the attribute contract - * lives in ../deploymentTelemetry.ts). - * - * Call exactly once per terminal transition, only after a guarded status - * write confirmed this caller won the transition. Emitted on ROOT_CONTEXT - * with forceRecording so the trace sampler can never drop it. Never throws. + * `deployment.lifecycle` span, backdated createdAt → terminal (attribute + * contract in ../deploymentTelemetry.ts). Call exactly once, only after a + * guarded status write confirmed this caller won the transition. Emitted on + * ROOT_CONTEXT with forceRecording so the sampler can never drop it; never + * throws. */ export function recordDeploymentLifecycle(params: { status: TerminalDeploymentStatus; @@ -106,8 +102,7 @@ export function recordDeploymentLifecycle(params: { ROOT_CONTEXT ); - // CANCELED is deliberately not an error: it is excluded from failure - // rates and tracked as its own volume. + // CANCELED is deliberately not an error: it stays out of failure rates if (isFailure) { span.setStatus({ code: SpanStatusCode.ERROR, @@ -126,9 +121,8 @@ export function recordDeploymentLifecycle(params: { /** * Records a deployment's creation as a zero-duration `deployment.initialized` - * event, the funnel counterpart to `deployment.lifecycle`: an initialized - * deployment with no lifecycle event after a few hours is either stuck - * non-terminal or hit an emission bug. Never throws. + * event — the funnel counterpart to `deployment.lifecycle` for detecting + * stuck deployments. Never throws. */ export function recordDeploymentInitialized(params: { deployment: Pick< diff --git a/apps/webapp/app/v3/services/timeoutDeployment.server.ts b/apps/webapp/app/v3/services/timeoutDeployment.server.ts index 4fe8e0178d1..fab1bb038b5 100644 --- a/apps/webapp/app/v3/services/timeoutDeployment.server.ts +++ b/apps/webapp/app/v3/services/timeoutDeployment.server.ts @@ -41,9 +41,7 @@ export class TimeoutDeploymentService extends BaseService { const failedAt = new Date(); const errorData = { message: errorMessage, name: "TimeoutError" }; - // Guarded transition: keeps the fromStatus check atomic with the write, so - // a concurrent finalize/fail/cancel can't be overwritten by a late timeout - // (and exactly one caller emits the lifecycle event). + // Guarded: keeps the fromStatus check atomic with the write const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index b1a940a0d33..7047d65d2b9 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -91,9 +91,7 @@ class DatasourceAttributeSpanProcessor implements SpanProcessor { } } -// Mirrors spans whose name matches a prefix into a second exporter (e.g. the -// dedicated deployment-events dataset) without removing them from the main -// exporter's stream. +// Mirrors name-prefixed spans into a second exporter; they still flow to the main one class SpanNamePrefixMirrorProcessor implements SpanProcessor { constructor( private readonly _inner: SpanProcessor, @@ -392,9 +390,7 @@ function setupTelemetry() { instrumentations, }); - // closeServer only closes express and lets the process drain, so a flush - // here has time to run — without it every webapp shutdown drops the last - // batch of spans (up to 1s of scheduledDelayMillis backlog). + // Without this flush every shutdown drops the last batch of spans const flushOnShutdown = () => { provider.forceFlush().catch(() => {}); }; diff --git a/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql index ba3697f38da..931ea947926 100644 --- a/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql +++ b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql @@ -1,2 +1 @@ --- Stamp the initiating CLI version on deployments at initialization ALTER TABLE "public"."WorkerDeployment" ADD COLUMN IF NOT EXISTS "cliVersion" TEXT; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index fd2a248eba4..a1423f340c0 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2265,8 +2265,7 @@ model WorkerDeployment { runtime String? runtimeVersion String? - /// CLI version that initiated the deploy (x-trigger-cli-version), stamped at - /// initialization so pre-index failures are attributable to a CLI version. + /// CLI version that initiated the deploy, stamped at initialization cliVersion String? imageReference String? From fc0d5fd1f687116f62ead5d87a299da330f0b9d2 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 18:05:05 +0200 Subject: [PATCH 07/13] refactor: rename the terminal telemetry event to deployment.finished --- apps/webapp/app/v3/deploymentTelemetry.ts | 6 +++--- .../createDeploymentBackgroundWorkerV4.server.ts | 4 ++-- apps/webapp/app/v3/services/deployment.server.ts | 10 +++++----- .../app/v3/services/failDeployment.server.ts | 4 ++-- .../app/v3/services/finalizeDeployment.server.ts | 4 ++-- .../app/v3/services/initializeDeployment.server.ts | 2 +- ...erver.ts => recordDeploymentFinished.server.ts} | 14 +++++++------- .../app/v3/services/timeoutDeployment.server.ts | 4 ++-- 8 files changed, 24 insertions(+), 24 deletions(-) rename apps/webapp/app/v3/services/{recordDeploymentLifecycle.server.ts => recordDeploymentFinished.server.ts} (94%) diff --git a/apps/webapp/app/v3/deploymentTelemetry.ts b/apps/webapp/app/v3/deploymentTelemetry.ts index eae4bbeebd9..947f9acf1e1 100644 --- a/apps/webapp/app/v3/deploymentTelemetry.ts +++ b/apps/webapp/app/v3/deploymentTelemetry.ts @@ -1,8 +1,8 @@ import { BuildServerMetadata } from "@trigger.dev/core/v3"; /** - * Attribute names for the `deployment.lifecycle` and `deployment.initialized` - * telemetry events (emitted by services/recordDeploymentLifecycle.server.ts). + * Attribute names for the `deployment.finished` and `deployment.initialized` + * telemetry events (emitted by services/recordDeploymentFinished.server.ts). * This module is the single owner of these names — external queries, * dashboards, and monitors reference them, so treat renames as breaking. * @@ -24,7 +24,7 @@ export const DeploymentTelemetryAttributes = { // Deployment friendly id — the dedup key DEPLOYMENT_ID: "deployment.id", VERSION: "deployment.version", - // lifecycle: terminal status; initialized: initial status (PENDING/BUILDING) + // finished: terminal status; initialized: initial status (PENDING/BUILDING) STATUS: "deployment.status", // status === DEPLOYED; CANCELED is excluded from failure rates SUCCESS: "deployment.success", diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index 60016d52f2c..7215c09c37c 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -18,7 +18,7 @@ import { } from "./createBackgroundWorker.server"; import { findOrCreateBackgroundWorker } from "./createDeploymentBackgroundWorkerV4/findOrCreateBackgroundWorker.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; -import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { env } from "~/env.server"; import { webhookPrisma } from "~/db.server"; @@ -335,7 +335,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { // BUILDING → DEPLOYING transition. await TimeoutDeploymentService.dequeue(deployment.id, this._prisma); - recordDeploymentLifecycle({ + recordDeploymentFinished({ status: "FAILED", deployment: { ...deployment, status: "FAILED", failedAt, errorData }, environment: { diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index 8d8f4488860..a639e6a7703 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -9,7 +9,7 @@ import { type DeploymentEvent, } from "@trigger.dev/core/v3"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; -import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { env } from "~/env.server"; import { createRemoteImageBuild } from "../remoteImageBuilder.server"; import { FINAL_DEPLOYMENT_STATUSES } from "./failDeployment.server"; @@ -252,9 +252,9 @@ export class DeploymentService extends BaseService { .andThen(validateDeployment) .andThen(cancelDeployment) .andThen(({ deployment }) => - this.#recordCanceledLifecycle(deployment.id) + this.#recordCanceledTelemetry(deployment.id) .orElse((error) => { - logger.error("Failed to record canceled deployment lifecycle", { error }); + logger.error("Failed to record canceled deployment telemetry", { error }); return okAsync(undefined); }) .map(() => ({ deployment })) @@ -484,7 +484,7 @@ export class DeploymentService extends BaseService { } // The cancel chain only carries a narrow row selection, so re-fetch the full row - #recordCanceledLifecycle(deploymentId: string) { + #recordCanceledTelemetry(deploymentId: string) { return fromPromise( this._prisma.workerDeployment.findFirst({ where: { id: deploymentId }, @@ -505,7 +505,7 @@ export class DeploymentService extends BaseService { ).map((canceled) => { if (!canceled || canceled.status !== "CANCELED") return; - recordDeploymentLifecycle({ + recordDeploymentFinished({ status: "CANCELED", deployment: canceled, environment: { diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index 5a97eea710a..534158308cb 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -5,7 +5,7 @@ import { boundedIn, Prisma, type WorkerDeploymentStatus } from "@trigger.dev/dat import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; export const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [ "CANCELED", @@ -78,7 +78,7 @@ export class FailDeploymentService extends BaseService { return; } - recordDeploymentLifecycle({ + recordDeploymentFinished({ status: "FAILED", deployment: failedDeployment, environment: { diff --git a/apps/webapp/app/v3/services/finalizeDeployment.server.ts b/apps/webapp/app/v3/services/finalizeDeployment.server.ts index d4ff97b8f6a..3ee7a1bebf0 100644 --- a/apps/webapp/app/v3/services/finalizeDeployment.server.ts +++ b/apps/webapp/app/v3/services/finalizeDeployment.server.ts @@ -10,7 +10,7 @@ import { projectPubSub } from "./projectPubSub.server"; import { FailDeploymentService } from "./failDeployment.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { engine } from "../runEngine.server"; import { tryCatch } from "@trigger.dev/core"; import { externalDeploymentCacheInstance } from "~/services/externalDeploymentCacheInstance.server"; @@ -101,7 +101,7 @@ export class FinalizeDeploymentService extends BaseService { buildEnvVars: null, }; - recordDeploymentLifecycle({ + recordDeploymentFinished({ status: "DEPLOYED", deployment: finalizedDeployment, environment: { diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index 7a182a8379b..56cf50e6879 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -16,7 +16,7 @@ import { getDeploymentImageRef } from "../getDeploymentImageRef.server"; import { tryCatch } from "@trigger.dev/core"; import { getRegistryConfig } from "../registryConfig.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentInitialized } from "./recordDeploymentLifecycle.server"; +import { recordDeploymentInitialized } from "./recordDeploymentFinished.server"; import { createDeploymentWithNextVersion } from "./initializeDeployment/createDeploymentWithNextVersion.server"; import { cancelSupersededDeployments, diff --git a/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts similarity index 94% rename from apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts rename to apps/webapp/app/v3/services/recordDeploymentFinished.server.ts index 830713c6487..72a872cb115 100644 --- a/apps/webapp/app/v3/services/recordDeploymentLifecycle.server.ts +++ b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts @@ -13,7 +13,7 @@ type TerminalDeploymentStatus = Extract< "DEPLOYED" | "FAILED" | "TIMED_OUT" | "CANCELED" >; -type LifecycleDeployment = Pick< +type FinishedDeployment = Pick< WorkerDeployment, | "friendlyId" | "version" @@ -46,15 +46,15 @@ type EnvironmentInfo = { /** * Records a deployment's terminal transition as a single wide - * `deployment.lifecycle` span, backdated createdAt → terminal (attribute + * `deployment.finished` span, backdated createdAt → terminal (attribute * contract in ../deploymentTelemetry.ts). Call exactly once, only after a * guarded status write confirmed this caller won the transition. Emitted on * ROOT_CONTEXT with forceRecording so the sampler can never drop it; never * throws. */ -export function recordDeploymentLifecycle(params: { +export function recordDeploymentFinished(params: { status: TerminalDeploymentStatus; - deployment: LifecycleDeployment; + deployment: FinishedDeployment; environment: EnvironmentInfo; reason?: string; }): void { @@ -68,7 +68,7 @@ export function recordDeploymentLifecycle(params: { const errorData = parseErrorData(deployment.errorData); const span = tracer.startSpan( - "deployment.lifecycle", + "deployment.finished", { startTime: deployment.createdAt, attributes: { @@ -112,7 +112,7 @@ export function recordDeploymentLifecycle(params: { span.end(terminalAt); } catch (error) { - logger.debug("recordDeploymentLifecycle failed", { + logger.debug("recordDeploymentFinished failed", { deploymentFriendlyId: params.deployment.friendlyId, error: error instanceof Error ? error.message : String(error), }); @@ -121,7 +121,7 @@ export function recordDeploymentLifecycle(params: { /** * Records a deployment's creation as a zero-duration `deployment.initialized` - * event — the funnel counterpart to `deployment.lifecycle` for detecting + * event — the funnel counterpart to `deployment.finished` for detecting * stuck deployments. Never throws. */ export function recordDeploymentInitialized(params: { diff --git a/apps/webapp/app/v3/services/timeoutDeployment.server.ts b/apps/webapp/app/v3/services/timeoutDeployment.server.ts index fab1bb038b5..63d81ba1930 100644 --- a/apps/webapp/app/v3/services/timeoutDeployment.server.ts +++ b/apps/webapp/app/v3/services/timeoutDeployment.server.ts @@ -5,7 +5,7 @@ import { commonWorker } from "../commonWorker.server"; import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { type PrismaClientOrTransaction } from "~/db.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; export class TimeoutDeploymentService extends BaseService { public async call(id: string, fromStatus: string, errorMessage: string) { @@ -71,7 +71,7 @@ export class TimeoutDeploymentService extends BaseService { buildEnvVars: null, }; - recordDeploymentLifecycle({ + recordDeploymentFinished({ status: "TIMED_OUT", deployment: timedOutDeployment, environment: { From 1af895a22e53ca74d5cca76638bcd1223354af5c Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 18:08:18 +0200 Subject: [PATCH 08/13] refactor: rename the local_bundle build path value to native_local_bundle --- apps/webapp/app/v3/deploymentTelemetry.ts | 6 +++--- apps/webapp/test/deploymentTelemetry.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/webapp/app/v3/deploymentTelemetry.ts b/apps/webapp/app/v3/deploymentTelemetry.ts index 947f9acf1e1..2dd0725f840 100644 --- a/apps/webapp/app/v3/deploymentTelemetry.ts +++ b/apps/webapp/app/v3/deploymentTelemetry.ts @@ -28,7 +28,7 @@ export const DeploymentTelemetryAttributes = { STATUS: "deployment.status", // status === DEPLOYED; CANCELED is excluded from failure rates SUCCESS: "deployment.success", - // depot / native / local_bundle (see deriveBuildPath) + // depot / native / native_local_bundle (see deriveBuildPath) BUILD_PATH: "deployment.build_path", // V1 / MANAGED (run engine) WORKER_TYPE: "deployment.worker_type", @@ -55,7 +55,7 @@ export const DeploymentTelemetryAttributes = { DURATION_DEPLOYING_MS: "deployment.duration.deploying_ms", } as const; -export type DeploymentBuildPath = "local_bundle" | "native" | "depot"; +export type DeploymentBuildPath = "native_local_bundle" | "native" | "depot"; /** * Everything that is not a native-build-server deployment falls into the depot @@ -67,7 +67,7 @@ export function deriveBuildPath(buildServerMetadata: unknown): DeploymentBuildPa const metadata = BuildServerMetadata.safeParse(buildServerMetadata); if (metadata.success && metadata.data.isNativeBuild) { - return metadata.data.fromBundle ? "local_bundle" : "native"; + return metadata.data.fromBundle ? "native_local_bundle" : "native"; } return "depot"; diff --git a/apps/webapp/test/deploymentTelemetry.test.ts b/apps/webapp/test/deploymentTelemetry.test.ts index e673af44373..9020f0c7102 100644 --- a/apps/webapp/test/deploymentTelemetry.test.ts +++ b/apps/webapp/test/deploymentTelemetry.test.ts @@ -2,8 +2,8 @@ import { describe, expect, it } from "vitest"; import { deriveBuildPath, deriveDeploymentDurations } from "~/v3/deploymentTelemetry"; describe("deriveBuildPath", () => { - it("classifies fromBundle native builds as local_bundle", () => { - expect(deriveBuildPath({ isNativeBuild: true, fromBundle: true })).toBe("local_bundle"); + it("classifies fromBundle native builds as native_local_bundle", () => { + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: true })).toBe("native_local_bundle"); }); it("classifies native builds without fromBundle as native", () => { @@ -17,7 +17,7 @@ describe("deriveBuildPath", () => { expect(deriveBuildPath({})).toBe("depot"); expect(deriveBuildPath({ buildId: "depot-build-id" })).toBe("depot"); expect(deriveBuildPath({ isNativeBuild: false })).toBe("depot"); - // fromBundle alone (skewed writer) must not count as local_bundle + // fromBundle alone (skewed writer) must not count as native_local_bundle expect(deriveBuildPath({ fromBundle: true })).toBe("depot"); expect(deriveBuildPath("garbage")).toBe("depot"); }); From d30575cad5e5e647d544661a50c0455bde059cb8 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 26 Aug 2026 10:25:13 +0200 Subject: [PATCH 09/13] refactor: use andTee/orTee for cancel-path side effects --- .../app/v3/services/deployment.server.ts | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index a639e6a7703..b0cdad889f0 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -251,15 +251,12 @@ export class DeploymentService extends BaseService { return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(cancelDeployment) - .andThen(({ deployment }) => - this.#recordCanceledTelemetry(deployment.id) - .orElse((error) => { - logger.error("Failed to record canceled deployment telemetry", { error }); - return okAsync(undefined); - }) - .map(() => ({ deployment })) + .andTee(({ deployment }) => + this.#recordCanceledTelemetry(deployment.id).orTee((error) => { + logger.error("Failed to record canceled deployment telemetry", { error }); + }) ) - .andThen(({ deployment }) => + .andTee(({ deployment }) => this.appendToEventLog(deployment.environment.project, deployment, [ { type: "finalized", @@ -268,14 +265,11 @@ export class DeploymentService extends BaseService { message: data?.canceledReason ?? undefined, }, }, - ]) - .orElse((error) => { - logger.error("Failed to append event to deployment event log", { error }); - return okAsync(deployment); - }) - .map(() => deployment) + ]).orTee((error) => { + logger.error("Failed to append event to deployment event log", { error }); + }) ) - .andThen(deleteTimeout) + .andThen(({ deployment }) => deleteTimeout(deployment)) .map(() => undefined); } From bf0ee665088af22025098fc7f88b3e24b5664ccc Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 26 Aug 2026 10:27:12 +0200 Subject: [PATCH 10/13] refactor: carry the full deployment row through the cancel chain instead of re-fetching --- .../app/v3/services/deployment.server.ts | 84 ++++++++----------- .../recordDeploymentFinished.server.ts | 6 +- 2 files changed, 39 insertions(+), 51 deletions(-) diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index b0cdad889f0..e8ac53cc475 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -196,10 +196,8 @@ export class DeploymentService extends BaseService { friendlyId: string, data?: Partial> ) { - const validateDeployment = ( - deployment: Pick & { - environment: { project: { externalRef: string } }; - } + const validateDeployment = >( + deployment: T ) => { if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) { logger.warn("Attempted cancelling deployment in a final state", { @@ -211,11 +209,7 @@ export class DeploymentService extends BaseService { return okAsync(deployment); }; - const cancelDeployment = ( - deployment: Pick & { - environment: { project: { externalRef: string } }; - } - ) => + const cancelDeployment = >(deployment: T) => fromPromise( this._prisma.workerDeployment.updateMany({ where: { @@ -252,8 +246,21 @@ export class DeploymentService extends BaseService { .andThen(validateDeployment) .andThen(cancelDeployment) .andTee(({ deployment }) => - this.#recordCanceledTelemetry(deployment.id).orTee((error) => { - logger.error("Failed to record canceled deployment telemetry", { error }); + recordDeploymentFinished({ + status: "CANCELED", + deployment: { + ...deployment, + status: "CANCELED", + canceledAt: new Date(), + canceledReason: data?.canceledReason ?? null, + }, + environment: { + organizationId: deployment.environment.project.organizationId, + projectId: deployment.environment.project.id, + projectRef: deployment.environment.project.externalRef, + environmentId: deployment.environment.id, + environmentType: deployment.environment.type, + }, }) ) .andTee(({ deployment }) => @@ -477,42 +484,6 @@ export class DeploymentService extends BaseService { ); } - // The cancel chain only carries a narrow row selection, so re-fetch the full row - #recordCanceledTelemetry(deploymentId: string) { - return fromPromise( - this._prisma.workerDeployment.findFirst({ - where: { id: deploymentId }, - include: { - environment: { - include: { - project: { - select: { id: true, organizationId: true, externalRef: true }, - }, - }, - }, - }, - }), - (error) => ({ - type: "other" as const, - cause: error, - }) - ).map((canceled) => { - if (!canceled || canceled.status !== "CANCELED") return; - - recordDeploymentFinished({ - status: "CANCELED", - deployment: canceled, - environment: { - organizationId: canceled.environment.project.organizationId, - projectId: canceled.environment.project.id, - projectRef: canceled.environment.project.externalRef, - environmentId: canceled.environmentId, - environmentType: canceled.environment.type, - }, - }); - }); - } - private getDeployment(environmentId: string, friendlyId: string) { return fromPromise( this._prisma.workerDeployment.findFirst({ @@ -523,6 +494,23 @@ export class DeploymentService extends BaseService { select: { status: true, id: true, + friendlyId: true, + version: true, + type: true, + createdAt: true, + startedAt: true, + installedAt: true, + builtAt: true, + deployedAt: true, + failedAt: true, + canceledAt: true, + canceledReason: true, + errorData: true, + runtime: true, + runtimeVersion: true, + cliVersion: true, + triggeredVia: true, + commitSHA: true, buildServerMetadata: true, imageReference: true, shortCode: true, @@ -530,6 +518,8 @@ export class DeploymentService extends BaseService { include: { project: { select: { + id: true, + organizationId: true, externalRef: true, }, }, diff --git a/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts index 72a872cb115..8d6727cac7f 100644 --- a/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts +++ b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts @@ -27,14 +27,13 @@ type FinishedDeployment = Pick< | "failedAt" | "canceledAt" | "canceledReason" - | "buildServerMetadata" | "errorData" | "runtime" | "runtimeVersion" | "cliVersion" | "triggeredVia" | "commitSHA" ->; +> & { buildServerMetadata: unknown }; type EnvironmentInfo = { organizationId?: string; @@ -132,11 +131,10 @@ export function recordDeploymentInitialized(params: { | "type" | "status" | "createdAt" - | "buildServerMetadata" | "runtime" | "cliVersion" | "triggeredVia" - >; + > & { buildServerMetadata: unknown }; environment: EnvironmentInfo; }): void { try { From cedd846266b942ef2b063648518cca5f7b98de2b Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 26 Aug 2026 10:40:17 +0200 Subject: [PATCH 11/13] fix: only accept version-shaped x-trigger-cli-version values --- apps/webapp/app/routes/api.v1.deployments.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index 9014343ccaf..798223fe5df 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -43,7 +43,7 @@ export async function action({ request, params }: ActionFunctionArgs) { try { const result = await service.call(authenticatedEnv, body.data, { - cliVersion: request.headers.get("x-trigger-cli-version") ?? undefined, + cliVersion: parseCliVersionHeader(request), }); const { deployment, imageRef } = result; @@ -77,6 +77,14 @@ export async function action({ request, params }: ActionFunctionArgs) { } } +// Client-controlled and persisted, so only accept version-shaped values +const CLI_VERSION_REGEX = /^[0-9A-Za-z.+-]{1,64}$/; + +function parseCliVersionHeader(request: Request): string | undefined { + const value = request.headers.get("x-trigger-cli-version"); + return value && CLI_VERSION_REGEX.test(value) ? value : undefined; +} + export const loader = createLoaderApiRoute( { searchParams: ApiDeploymentListSearchParams, From b515933a7e1700e210c4280e5d969b68b50b94f0 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 26 Aug 2026 10:51:00 +0200 Subject: [PATCH 12/13] fix(cli): send the CLI version header on all API requests The webapp stamps cliVersion at deployment initialization from x-trigger-cli-version, but the CLI only sent that header on two unrelated endpoints - getHeaders() now includes it everywhere. Also swap the webapp's regex check for a plain length cap. --- .changeset/violet-buses-tease.md | 5 +++++ apps/webapp/app/routes/api.v1.deployments.ts | 6 +++--- packages/cli-v3/src/apiClient.ts | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 .changeset/violet-buses-tease.md diff --git a/.changeset/violet-buses-tease.md b/.changeset/violet-buses-tease.md new file mode 100644 index 00000000000..191cff77683 --- /dev/null +++ b/.changeset/violet-buses-tease.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Send the CLI version header on all API requests so deployments are attributable to a CLI version diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index 798223fe5df..a783421f482 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -77,12 +77,12 @@ export async function action({ request, params }: ActionFunctionArgs) { } } -// Client-controlled and persisted, so only accept version-shaped values -const CLI_VERSION_REGEX = /^[0-9A-Za-z.+-]{1,64}$/; +// Client-controlled and persisted, so cap what we accept +const CLI_VERSION_MAX_LENGTH = 64; function parseCliVersionHeader(request: Request): string | undefined { const value = request.headers.get("x-trigger-cli-version"); - return value && CLI_VERSION_REGEX.test(value) ? value : undefined; + return value && value.length <= CLI_VERSION_MAX_LENGTH ? value : undefined; } export const loader = createLoaderApiRoute( diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index 8b9fd56eb1c..a7a07cf40eb 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -1016,6 +1016,7 @@ export class CliApiClient { Authorization: `Bearer ${this.accessToken}`, "Content-Type": "application/json", "x-trigger-source": this.source, + "x-trigger-cli-version": VERSION, ...this.getBranchHeader(), }; } From 104bcc9611472a784ab6d8e0c7130e87611e6282 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 26 Aug 2026 10:54:38 +0200 Subject: [PATCH 13/13] chore: relax the CLI version header length cap to 128 --- apps/webapp/app/routes/api.v1.deployments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index a783421f482..184fa996d97 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -78,7 +78,7 @@ export async function action({ request, params }: ActionFunctionArgs) { } // Client-controlled and persisted, so cap what we accept -const CLI_VERSION_MAX_LENGTH = 64; +const CLI_VERSION_MAX_LENGTH = 128; function parseCliVersionHeader(request: Request): string | undefined { const value = request.headers.get("x-trigger-cli-version");