diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts
index e8a3e555831..1f7d07ccff0 100644
--- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts
+++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts
@@ -6,6 +6,104 @@ import { sanitizeChatDisplayContent } from '@/app/workspace/[workspaceId]/home/c
import { scalingRatioOver4x } from '@/app/workspace/[workspaceId]/home/components/message-content/components/scaling-test-helpers'
describe('sanitizeChatDisplayContent', () => {
+ it.each(['source', 'workspace_resource'])(
+ 'unwraps %s JSON that mentions the other chip tag',
+ (name) => {
+ const otherTag = name === 'source' ? 'workspace_resource' : 'source'
+ const tag = `<${name}>${JSON.stringify({ title: `Use <${otherTag}>` })}${name}>`
+
+ expect(sanitizeChatDisplayContent(`\`${tag}\``)).toBe(tag)
+ }
+ )
+
+ it.each([2, 3, 4])('preserves a %i-backtick code span containing a chip', (length) => {
+ const delimiter = '`'.repeat(length)
+ const tag = '{"url":"https://example.com","title":"Use `config`"}'
+ const content = `${delimiter}${tag}${delimiter}`
+
+ expect(sanitizeChatDisplayContent(content)).toBe(content)
+ expect(sanitizeChatDisplayContent(`${delimiter}json\n\`${tag}\`\n${delimiter}`)).toBe(
+ `${delimiter}json\n\`${tag}\`\n${delimiter}`
+ )
+ expect(sanitizeChatDisplayContent(`${content} then \`${tag}\``)).toBe(`${content} then ${tag}`)
+ })
+
+ it('does not let an unmatched backtick run suppress later citations', () => {
+ const prefix = 'Use `` for two backticks.\n'
+ const tag = '{"url":"https://example.com"}'
+
+ expect(sanitizeChatDisplayContent(`${prefix}\`${tag}\``)).toBe(`${prefix}${tag}`)
+ })
+
+ it.each(['\n\n', '\r\n\r\n', '\n \t\n'])(
+ 'does not pair prose runs across paragraph break %j',
+ (separator) => {
+ const tag = '{"url":"https://example.com"}'
+ const before = `Use \`\` as a delimiter.${separator}`
+ const after = `${separator}Another \`\` marker.`
+
+ expect(sanitizeChatDisplayContent(`${before}\`${tag}\`${after}`)).toBe(
+ `${before}${tag}${after}`
+ )
+ }
+ )
+
+ it('preserves matched multi-backtick spans across a soft line break', () => {
+ const content = '``Literal\n`{"url":"https://example.com"}`\nexample``'
+
+ expect(sanitizeChatDisplayContent(content)).toBe(content)
+ })
+
+ it('does not treat blank lines inside chip JSON as paragraph breaks', () => {
+ const tag = '{\n\n"url":"https://example.com",\n\n"title":"Use `code`"\n}'
+
+ expect(sanitizeChatDisplayContent(`\`${tag}\``)).toBe(tag)
+ })
+
+ it.each(['```', '~~~'])(
+ 'preserves a %s fence closed by a longer run and unwraps citations after it',
+ (fence) => {
+ const tag = '{"url":"https://example.com"}'
+ const block = `${fence}json\n\`${tag}\`\n${fence}${fence[0]}\n`
+
+ expect(sanitizeChatDisplayContent(`${block}\`${tag}\``)).toBe(`${block}${tag}`)
+ }
+ )
+
+ it.each(['```', '~~~'])('leaves an unclosed %s streaming fence literal', (fence) => {
+ const content = `${fence}json\n\`{"url":"https://example.com"}\``
+
+ expect(sanitizeChatDisplayContent(content)).toBe(content)
+ })
+
+ it.each(['source', 'workspace_resource'])(
+ 'preserves tilde-fenced %s chips with backticks in the info string',
+ (name) => {
+ const tag = `<${name}>{"title":"Example"}${name}>`
+ const block = `~~~example \`code\`\n\`${tag}\`\n~~~\n`
+
+ expect(sanitizeChatDisplayContent(`${block}\`${tag}\``)).toBe(`${block}${tag}`)
+ }
+ )
+
+ it.each(['```', '~~~'])(
+ 'does not close a %s fence with a different character or a shorter run',
+ (fence) => {
+ const tag = '{"url":"https://example.com"}'
+ const otherFence = fence === '```' ? '~~~~' : '````'
+ const block = `${fence}${fence[0]}\n${otherFence}\n\`${tag}\`\n${fence}\n\`${tag}\`\n${fence}${fence[0]}\n`
+
+ expect(sanitizeChatDisplayContent(`${block}\`${tag}\``)).toBe(`${block}${tag}`)
+ }
+ )
+
+ it('does not open a backtick fence with backticks in its info string', () => {
+ const prefix = '```example `code`\n\n'
+ const tag = '{"url":"https://example.com"}'
+
+ expect(sanitizeChatDisplayContent(`${prefix}\`${tag}\``)).toBe(`${prefix}${tag}`)
+ })
+
it('unwraps workspace resource tags from inline code spans', () => {
const content =
'`I updated {"type":"workflow","id":"wf-1","title":"Workflow"}.`'
@@ -159,4 +257,24 @@ describe('sanitizeChatDisplayContent', () => {
'{"type":"file","path":"a.md","title":"a"} done'
)
})
+
+ it.each(['source', 'workspace_resource'])(
+ 'stays linear on repeated %s tags with unterminated JSON strings',
+ (name) => {
+ expect(
+ scalingRatioOver4x(sanitizeChatDisplayContent, (times) =>
+ `<${name}>{${String.fromCharCode(92, 34)}`.repeat(times)
+ )
+ ).toBeLessThan(8)
+ }
+ )
+
+ it.each(['"', '{"key":"'])(
+ 'stays linear on repeated quoted payload prefix %s',
+ (prefix) => {
+ expect(
+ scalingRatioOver4x(sanitizeChatDisplayContent, (times) => prefix.repeat(times))
+ ).toBeLessThan(8)
+ }
+ )
})
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts
index 867916bd863..0c9813cdf65 100644
--- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts
+++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts
@@ -1,66 +1,129 @@
const HIDDEN_INLINE_REFERENCE_PATTERN =
/`[^`\n]*(?:internal\/tool-results\/|internal\/blocktips\/|components\/integrations\/[^`\n]*README)[^`\n]*`/g
-/** JSON strings own their escaped quotes, backticks, and any quoted tag markers. */
-const JSON_STRING_SOURCE = String.raw`"(?:[^"\\\r\n]|\\[^\r\n])*"`
+/** JSON strings own their escaped quotes, backticks, and quoted tag markers. */
+const JSON_STRING_SOURCE = '"(?:\\\\(?:["\\\\/bfnrt]|u[0-9a-fA-F]{4})|[^"\\\\\\r\\n])*"'
-/**
- * Complete chip tags consume JSON strings atomically. Outside strings, a new
- * opener or backtick ends the candidate, so prose mentions cannot join into a
- * tag and repeated unclosed openers cannot repeatedly scan the same suffix.
- */
-const COMPLETE_TAG_SOURCE = `<(?workspace_resource|source)>\\s*\\{(?:${JSON_STRING_SOURCE}|[^"\`<])*?\\}\\s*\\k>`
+/** Unquoted openers, backticks, and invalid backslashes bound failed payload scans. */
+const COMPLETE_TAG_SOURCE = `<(?workspace_resource|source)>\\s*\\{(?:${JSON_STRING_SOURCE}|[^"\`<\\\\])*?\\}\\s*\\k>`
-const CHIP_OR_CODE_DELIMITER = new RegExp(`${COMPLETE_TAG_SOURCE}|\`|\n`, 'g')
+const INLINE_CHIP_OR_DELIMITER = new RegExp(`${COMPLETE_TAG_SOURCE}|\`+|\\n`, 'g')
+const CHIP_OR_PARAGRAPH_BREAK = new RegExp(`${COMPLETE_TAG_SOURCE}|\\n[\\t \\r]*\\n`, 'g')
-/**
- * Pair Markdown delimiters outside chip payloads in one forward pass. A pair
- * containing a chip is unwrapped; a lone delimiter is removed only when flush
- * against a chip. Neighbouring code spans and multiline fences keep their pairs.
- */
-export function sanitizeChatDisplayContent(content: string): string {
- const removedDelimiters: number[] = []
- let openingTick = -1
- let containsChip = false
- let adjacentToChip = false
- let lastChipEnd = -1
+interface OpenCodeSpan {
+ index: number
+ containsChip: boolean
+ touchesChip: boolean
+}
- for (const match of content.matchAll(CHIP_OR_CODE_DELIMITER)) {
- const index = match.index
- if (match.groups?.chipTag) {
- if (openingTick !== -1) {
- containsChip = true
- adjacentToChip ||= index === openingTick + 1
- }
- lastChipEnd = index + match[0].length
- continue
+/** Only matched multi-backtick runs are code; an unmatched run remains ordinary prose. */
+function unwrapInlineParagraph(content: string): string {
+ const remainingRuns = new Map()
+ for (const [value] of content.matchAll(INLINE_CHIP_OR_DELIMITER)) {
+ if (value.startsWith('`') && value.length > 1) {
+ remainingRuns.set(value.length, (remainingRuns.get(value.length) ?? 0) + 1)
}
+ }
+ const removedDelimiters: number[] = []
+ let openSpan: OpenCodeSpan | null = null
+ let previousChipEnd = -1
+ let protectedRunLength: number | null = null
- if (match[0] === '\n') {
- if (openingTick !== -1 && adjacentToChip) removedDelimiters.push(openingTick)
- openingTick = -1
- lastChipEnd = -1
- continue
- }
+ const finishLine = () => {
+ if (openSpan?.touchesChip) removedDelimiters.push(openSpan.index)
+ openSpan = null
+ }
- if (openingTick === -1) {
- openingTick = index
- containsChip = false
- adjacentToChip = lastChipEnd === index
+ for (const token of content.matchAll(INLINE_CHIP_OR_DELIMITER)) {
+ const [value] = token
+ const index = token.index
+ if (value === '\n') {
+ finishLine()
+ previousChipEnd = -1
+ } else if (value.startsWith('`')) {
+ if (value.length > 1) {
+ remainingRuns.set(value.length, (remainingRuns.get(value.length) ?? 1) - 1)
+ }
+ if (protectedRunLength !== null) {
+ if (value.length === protectedRunLength) protectedRunLength = null
+ continue
+ }
+ if (value.length > 1) {
+ if (!openSpan && remainingRuns.get(value.length)) protectedRunLength = value.length
+ continue
+ }
+ if (openSpan) {
+ if (openSpan.containsChip) removedDelimiters.push(openSpan.index, index)
+ openSpan = null
+ } else {
+ openSpan = { index, containsChip: false, touchesChip: previousChipEnd === index }
+ }
} else {
- if (containsChip) removedDelimiters.push(openingTick, index)
- openingTick = -1
+ if (openSpan) {
+ openSpan.containsChip = true
+ openSpan.touchesChip ||= index === openSpan.index + 1
+ }
+ previousChipEnd = index + value.length
}
}
-
- if (openingTick !== -1 && adjacentToChip) removedDelimiters.push(openingTick)
+ finishLine()
const parts: string[] = []
- let start = 0
+ let cursor = 0
for (const index of removedDelimiters) {
- parts.push(content.slice(start, index))
- start = index + 1
+ parts.push(content.slice(cursor, index))
+ cursor = index + 1
+ }
+ parts.push(content.slice(cursor))
+ return parts.join('')
+}
+
+/** Paragraph breaks end inline spans, but blank lines inside chip JSON belong to the payload. */
+function unwrapInlineChips(content: string): string {
+ const parts: string[] = []
+ let cursor = 0
+
+ for (const match of content.matchAll(CHIP_OR_PARAGRAPH_BREAK)) {
+ if (!match[0].startsWith('\n')) continue
+ parts.push(unwrapInlineParagraph(content.slice(cursor, match.index)), match[0])
+ cursor = match.index + match[0].length
+ }
+ parts.push(unwrapInlineParagraph(content.slice(cursor)))
+ return parts.join('')
+}
+
+/** Fenced blocks are literal, including unclosed streaming fences and longer closing runs. */
+export function sanitizeChatDisplayContent(content: string): string {
+ const parts: string[] = []
+ let cursor = 0
+ let fenceStart: number | null = null
+ let fence = ''
+
+ for (const line of content.matchAll(/^ {0,3}(`{3,}|~{3,})([^\n]*)(?:\n|$)/gm)) {
+ const [, delimiter, info] = line
+ if (fenceStart === null) {
+ if (delimiter[0] === '`' && info.includes('`')) continue
+ fenceStart = line.index
+ fence = delimiter
+ } else if (
+ delimiter[0] === fence[0] &&
+ delimiter.length >= fence.length &&
+ /^[\t \r]*$/.test(info)
+ ) {
+ const end = line.index + line[0].length
+ parts.push(
+ unwrapInlineChips(content.slice(cursor, fenceStart)),
+ content.slice(fenceStart, end)
+ )
+ cursor = end
+ fenceStart = null
+ }
+ }
+
+ if (fenceStart === null) {
+ parts.push(unwrapInlineChips(content.slice(cursor)))
+ } else {
+ parts.push(unwrapInlineChips(content.slice(cursor, fenceStart)), content.slice(fenceStart))
}
- parts.push(content.slice(start))
return parts.join('').replace(HIDDEN_INLINE_REFERENCE_PATTERN, '')
}
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/scaling-test-helpers.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/scaling-test-helpers.ts
index 7d2dab3b83e..921de059228 100644
--- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/scaling-test-helpers.ts
+++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/scaling-test-helpers.ts
@@ -27,13 +27,16 @@ function fastest(run: (content: string) => void, content: string): number {
* through at the single size it happens to sample. Quadratic costs ~16x for 4x
* the input; linear costs ~4x.
*/
-export function scalingRatioOver4x(run: (content: string) => void): number {
+export function scalingRatioOver4x(
+ run: (content: string) => void,
+ buildContent: (times: number) => string = buildRepeatedTagMentions
+): number {
// Warm up first — the JIT would otherwise charge the whole compile to the
// small sample and flatter the ratio.
- fastest(run, buildRepeatedTagMentions(2_000))
+ fastest(run, buildContent(2_000))
- const small = fastest(run, buildRepeatedTagMentions(2_000))
- const large = fastest(run, buildRepeatedTagMentions(8_000))
+ const small = fastest(run, buildContent(2_000))
+ const large = fastest(run, buildContent(8_000))
return large / small
}