fix: group transcript captions into readable sentences

apps/web/__tests__/integration/live-transcribe-diarization.test.ts

@@ -0,0 +1,124 @@
+import { Effect, Option } from "effect";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { createEmptyLiveTranscript } from "@/lib/live-transcribe-core";
+
+const mocks = vi.hoisted(() => ({
+	queue: vi.fn(),
+	objects: new Map<string, string>(),
+	writes: [] as string[],
+}));
+const video = {
+	id: "live-video",
+	ownerId: "live-owner",
+	source: { type: "desktopSegments" },
+	transcriptionStatus: null,
+	settings: null,
+};
+vi.mock("@cap/env", () => ({
+	serverEnv: () => ({ ASSEMBLY_API_KEY: "test-key" }),
+}));
+vi.mock("@cap/database/schema", () => ({
+	videos: {
+		id: "video-id",
+		ownerId: "owner-id",
+		metadata: "metadata",
+		updatedAt: "updated-at",
+	},
+	organizations: { id: "org-id", settings: "settings" },
+	users: { id: "user-id" },
+}));
+vi.mock("@cap/database", () => ({
+	db: () => ({
+		select: () => ({
+			from: () => ({
+				leftJoin: () => ({ where: async () => [{ video, orgSettings: null }] }),
+				where: async () => [video],
+			}),
+		}),
+		update: () => ({ set: () => ({ where: async () => [] }) }),
+	}),
+}));
+vi.mock("@cap/web-backend/src/Storage/index", () => ({
+	Storage: {
+		getAccessForVideo: () =>
+			Effect.succeed([
+				{
+					getObject: (key: string) =>
+						Effect.succeed(Option.fromNullable(mocks.objects.get(key))),
+					putObject: (key: string, value: string) =>
+						Effect.sync(() => {
+							mocks.writes.push(key);
+							mocks.objects.set(key, value);
+						}),
+				},
+			]),
+	},
+}));
+vi.mock("@/lib/video-storage", () => ({ decodeStorageVideo: () => ({}) }));
+vi.mock("@/lib/workflow-runtime", () => ({
+	runWorkflowPromise: Effect.runPromise,
+}));
+vi.mock("@/lib/transcribe", () => ({ transcribeVideo: mocks.queue }));
+vi.mock("@/lib/ai-generation-entitlement", () => ({
+	isAiGenerationEnabledForUser: () => false,
+}));
+
+const artifactKey = "live-owner/live-video/transcription.live.json";
+beforeEach(() => {
+	mocks.objects.clear();
+	mocks.writes.length = 0;
+	mocks.queue.mockResolvedValue({ success: true, message: "Queued" });
+	mocks.objects.set(
+		artifactKey,
+		JSON.stringify({
+			...createEmptyLiveTranscript("2026-09-08T00:00:00.000Z"),
+			lastAudioSegmentIndex: 2,
+			transcribedDurationMs: 4000,
+		}),
+	);
+	mocks.objects.set(
+		"live-owner/live-video/segments/manifest.json",
+		JSON.stringify({
+			version: 5,
+			video_init_uploaded: true,
+			audio_init_uploaded: true,
+			video_segments: [],
+			audio_segments: [
+				{ index: 1, duration: 2 },
+				{ index: 2, duration: 2 },
+			],
+			is_complete: true,
+		}),
+	);
+});
+
+describe("live recording diarization handoff", () => {
+	it("queues a full recording pass even when provisional chunks cover every segment", async () => {
+		const { liveTranscribeWorkflow } = await import(
+			"@/workflows/live-transcribe"
+		);
+		await liveTranscribeWorkflow({ videoId: video.id, userId: video.ownerId });
+		expect(mocks.queue).toHaveBeenCalledExactlyOnceWith(
+			video.id,
+			video.ownerId,
+			false,
+			{ earlyFromSegments: true },
+		);
+		expect(mocks.writes).toEqual([artifactKey]);
+		expect(JSON.parse(mocks.objects.get(artifactKey) ?? "{}").state).toBe(
+			"complete",
+		);
+	});
+	it("surfaces queue failures so the durable workflow retries the final transcription", async () => {
+		mocks.queue.mockResolvedValue({
+			success: false,
+			message: "Queue unavailable",
+		});
+		const { liveTranscribeWorkflow } = await import(
+			"@/workflows/live-transcribe"
+		);
+		await expect(
+			liveTranscribeWorkflow({ videoId: video.id, userId: video.ownerId }),
+		).rejects.toThrow("Queue unavailable");
+	});
+});

apps/web/__tests__/integration/transcribe-workflow.test.ts

@@ -206,6 +206,9 @@ describe("transcribeVideoWorkflow", () => {
 
 		expect(result.success).toBe(true);
 		expect(mocks.transcribe).toHaveBeenCalledTimes(1);
+		expect(mocks.transcribe).toHaveBeenCalledWith(
+			expect.objectContaining({ speaker_labels: true }),
+		);
 		expect(mocks.transcribe.mock.calls[0]?.[0]).toMatchObject({
 			disfluencies: true,
 			speech_models: ["universal-3-5-pro", "universal-2"],
@@ -265,6 +268,9 @@ describe("transcribeVideoWorkflow", () => {
 			message: "Video has no spoken audio - skipped transcription",
 		});
 		expect(mocks.transcribe).toHaveBeenCalledTimes(1);
+		expect(mocks.transcribe).toHaveBeenCalledWith(
+			expect.objectContaining({ speaker_labels: true }),
+		);
 		expect(mocks.updates).toContainEqual({ transcriptionStatus: "NO_AUDIO" });
 		expect(mocks.updates).not.toContainEqual({ transcriptionStatus: "ERROR" });
 		expect(mocks.startAiGeneration).not.toHaveBeenCalled();

apps/web/__tests__/unit/caption-cues.test.ts

@@ -20,9 +20,11 @@ describe("getActiveCaptionText", () => {
 	it("uses the latest active cue when cues overlap", () => {
 		const activeCues = createCueList([
 			{ startTime: 0, text: "First caption" },
-			{ startTime: 3.199, text: "<v Speaker>Second caption</v>" },
+			{ startTime: 3.199, text: "<v Speaker B>Second &amp; final caption</v>" },
 		]);
 
-		expect(getActiveCaptionText(activeCues)).toBe("Second caption");
+		expect(getActiveCaptionText(activeCues)).toBe(
+			"Speaker B: Second & final caption",
+		);
 	});
 });

apps/web/__tests__/unit/diarization.test.ts

@@ -0,0 +1,126 @@
+import { describe, expect, it } from "vitest";
+import { formatTranscriptAsVTT } from "@/app/s/[videoId]/_components/utils/transcript-utils";
+import {
+	createEditTranscript,
+	editTranscriptWordsToCaptionVtt,
+	groupEditTranscriptWords,
+	parseEditTranscript,
+	remapEditTranscriptThroughSpec,
+	serializeEditTranscript,
+} from "@/lib/edit-transcript";
+import { formatTranscriptAsParagraphs } from "@/lib/transcript-text";
+import {
+	formatVttCueText,
+	parseVTT,
+	parseVttCueText,
+	updateVttEntryText,
+} from "@/lib/transcript-vtt";
+
+const transcript = createEditTranscript(
+	{
+		words: [
+			{ text: "Hello", start: 100, end: 300, speaker: "A" },
+			{ text: "there", start: 300, end: 600, speaker: "A" },
+			{ text: "Hi", start: 650, end: 800, speaker: "B" },
+			{ text: "again", start: 850, end: 1000, speaker: "A" },
+			{ text: "unknown", start: 1100, end: 1300, speaker: null },
+		],
+	},
+	2000,
+);
+
+describe("speaker diarization", () => {
+	it("splits captions on every speaker transition without needing punctuation or silence", () => {
+		const cues = parseVTT(editTranscriptWordsToCaptionVtt(transcript.words));
+		expect(
+			cues.map(({ text, speaker, startTime, endTime }) => ({
+				text,
+				speaker,
+				startTime,
+				endTime,
+			})),
+		).toEqual([
+			{ text: "Hello there", speaker: "A", startTime: 0.1, endTime: 0.6 },
+			{ text: "Hi", speaker: "B", startTime: 0.65, endTime: 0.8 },
+			{ text: "again", speaker: "A", startTime: 0.85, endTime: 1 },
+			{ text: "unknown", speaker: null, startTime: 1.1, endTime: 1.3 },
+		]);
+	});
+
+	it("preserves labels through storage, video cuts, caption regeneration, and download", () => {
+		const stored = parseEditTranscript(serializeEditTranscript(transcript));
+		expect(stored).not.toBeNull();
+		if (!stored) throw new Error("Missing transcript");
+		const edited = remapEditTranscriptThroughSpec(stored, {
+			version: 1,
+			sourceDuration: 2,
+			keepRanges: [{ start: 0.6, end: 2 }],
+		});
+		const cues = parseVTT(editTranscriptWordsToCaptionVtt(edited.words));
+		expect(cues[0]).toMatchObject({
+			text: "Hi",
+			speaker: "B",
+			startTime: 0.05,
+		});
+		expect(parseVTT(formatTranscriptAsVTT(cues))).toEqual(cues);
+	});
+
+	it("shows separate editor groups and text paragraphs for speakers and unknown speech", () => {
+		expect(
+			groupEditTranscriptWords(transcript.words).map(
+				({ startIndex, endIndex }) => [startIndex, endIndex],
+			),
+		).toEqual([
+			[0, 1],
+			[2, 2],
+			[3, 3],
+			[4, 4],
+		]);
+		expect(
+			formatTranscriptAsParagraphs(
+				parseVTT(editTranscriptWordsToCaptionVtt(transcript.words)),
+			),
+		).toBe(
+			"Speaker A: Hello there\n\nSpeaker B: Hi\n\nSpeaker A: again\n\nunknown",
+		);
+	});
+
+	it("preserves the voice when editing spoken text and escapes markup", () => {
+		const vtt = editTranscriptWordsToCaptionVtt(transcript.words);
+		const updated = updateVttEntryText(vtt, 2, "Yes <script> & no");
+		expect(updated.updated).toBe(true);
+		expect(updated.content).toContain(
+			"<v Speaker B>Yes &lt;script&gt; &amp; no</v>",
+		);
+		expect(parseVTT(updated.content)[1]).toMatchObject({
+			speaker: "B",
+			text: "Yes <script> & no",
+			startTime: 0.65,
+			endTime: 0.8,
+		});
+	});
+
+	it("handles multiline voice cues, legacy plain cues, and escaped labels", () => {
+		const cues = parseVTT(
+			"WEBVTT\r\n\r\n1\r\n00:00:00.125 --> 00:00:01.500\r\n<v Speaker A>Hello\r\nthere</v>\r\n\r\n2\r\n00:00:01.500 --> 00:00:02.000\r\nLegacy text\r\n",
+		);
+		expect(cues[0]).toMatchObject({
+			text: "Hello there",
+			speaker: "A",
+			startTime: 0.125,
+			endTime: 1.5,
+		});
+		expect(cues[1]).toMatchObject({ text: "Legacy text", speaker: null });
+		expect(parseVttCueText(formatVttCueText("2 < 3 & 4 > 1", "A & B"))).toEqual(
+			{ text: "2 < 3 & 4 > 1", speaker: "A & B" },
+		);
+	});
+});
+
+it("keeps speaker metadata and literal text through the agent transcript API", async () => {
+	const { parseAgentVtt, renderAgentVtt } = await import("@/lib/agent-api");
+	const cues = [
+		{ startMs: 125, endMs: 500, text: "R&D < planning", speaker: "B" },
+	];
+	expect(parseAgentVtt(renderAgentVtt(cues))).toEqual(cues);
+});

apps/web/__tests__/unit/live-transcribe-core.test.ts

@@ -1,10 +1,8 @@
 import { describe, expect, it } from "vitest";
 import {
 	applyChunkToLiveTranscript,
-	canPromoteLiveTranscript,
 	createEmptyLiveTranscript,
 	isNoSpokenAudioError,
-	liveTranscriptToEditTranscript,
 	offsetChunkWords,
 	parseLiveTranscript,
 	planNextLiveChunk,
@@ -248,95 +246,6 @@ describe("live transcript artifact", () => {
 	});
 });
 
-describe("canPromoteLiveTranscript", () => {
-	const fullCoverage = (overrides = {}) => ({
-		...createEmptyLiveTranscript("2026-08-03T00:00:00.000Z"),
-		lastAudioSegmentIndex: 3,
-		transcribedDurationMs: 6000,
-		...overrides,
-	});
-	const completeManifest = {
-		...baseManifest,
-		audio_segments: [seg(1), seg(2), seg(3)],
-		is_complete: true,
-	};
-
-	it("promotes only full gap-free coverage", () => {
-		expect(canPromoteLiveTranscript(fullCoverage(), completeManifest)).toEqual({
-			ok: true,
-		});
-	});
-
-	it("declines incomplete manifests, partial coverage, and skipped chunks", () => {
-		expect(
-			canPromoteLiveTranscript(fullCoverage(), {
-				...completeManifest,
-				is_complete: false,
-			}).ok,
-		).toBe(false);
-		expect(
-			canPromoteLiveTranscript(
-				fullCoverage({ lastAudioSegmentIndex: 2 }),
-				completeManifest,
-			).ok,
-		).toBe(false);
-		expect(
-			canPromoteLiveTranscript(
-				fullCoverage({ hasGaps: true }),
-				completeManifest,
-			).ok,
-		).toBe(false);
-	});
-
-	it("declines manifests with segment index gaps", () => {
-		expect(
-			canPromoteLiveTranscript(fullCoverage({ lastAudioSegmentIndex: 4 }), {
-				...completeManifest,
-				audio_segments: [seg(1), seg(2), seg(4)],
-			}).ok,
-		).toBe(false);
-	});
-
-	it("declines recordings with no audio", () => {
-		expect(
-			canPromoteLiveTranscript(fullCoverage(), {
-				...completeManifest,
-				audio_segments: [],
-			}).ok,
-		).toBe(false);
-	});
-});
-
-describe("liveTranscriptToEditTranscript", () => {
-	it("shapes accumulated words as a canonical v3 edit transcript", () => {
-		const artifact = applyChunkToLiveTranscript(
-			createEmptyLiveTranscript("2026-08-03T00:00:00.000Z"),
-			{
-				startMs: 0,
-				durationMs: 4000,
-				lastAudioSegmentIndex: 2,
-				words: offsetChunkWords(
-					[{ text: "Hello", start: 10, end: 500 }],
-					0,
-					4000,
-				),
-				languageCode: "en",
-				nowIso: "2026-08-03T00:00:05.000Z",
-			},
-		);
-
-		const edit = liveTranscriptToEditTranscript(artifact, "universal-3-5-pro");
-		expect(edit).toMatchObject({
-			version: 3,
-			speechModelUsed: "universal-3-5-pro",
-			durationMs: 4000,
-			languageCode: "en",
-		});
-		expect(edit.words).toHaveLength(1);
-		expect(edit.words[0]).toMatchObject({ text: "Hello", startMs: 10 });
-	});
-});
-
 describe("isNoSpokenAudioError", () => {
 	it("recognizes speech-free chunks as valid empties, not failures", () => {
 		// exact message observed from the real API on a silent recording