From 9fc991a724b023d15efc50151fb56b617b7c81e8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 15:36:19 -0700 Subject: [PATCH 1/3] feat(knowledge): crawl Slack per member on Sim Search through each person's own Slack user token --- .../permission-scoped-listing.test.ts | 43 ++-- apps/sim/connectors/slack/meta.ts | 11 + apps/sim/connectors/slack/slack.test.ts | 191 +++++++++++++++++ apps/sim/connectors/slack/slack.ts | 201 ++++++++++++------ .../standard-oauth-provider.ts | 25 ++- .../connectors/member-access.test.ts | 59 +++++ .../lib/knowledge/connectors/member-access.ts | 14 +- .../connectors/member-provisioning.test.ts | 75 ++++++- .../connectors/member-provisioning.ts | 26 ++- 9 files changed, 539 insertions(+), 106 deletions(-) create mode 100644 apps/sim/connectors/slack/slack.test.ts diff --git a/apps/sim/connectors/permission-scoped-listing.test.ts b/apps/sim/connectors/permission-scoped-listing.test.ts index 1e66c9b5f33..2b29560a943 100644 --- a/apps/sim/connectors/permission-scoped-listing.test.ts +++ b/apps/sim/connectors/permission-scoped-listing.test.ts @@ -3,12 +3,29 @@ */ import { describe, expect, it } from 'vitest' import { getManagedOAuthConnectorPolicy } from '@/lib/auth/connectors/managed-oauth' +import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry' import { + type CredentialGroupProvider, + getCredentialGroupProviderFromProviderId, getCredentialGroupProviderService, - getCredentialGroupStandardOAuthProviderFromProviderId, + isCredentialGroupStandardOAuthProvider, } from '@/lib/credential-groups/providers' +import { SLACK_MANAGED_USER_SCOPES } from '@/lib/credential-groups/slack-managed-user-scopes' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' +/** + * The scopes an option of the provider requests from every member: the + * provider's service scopes plus its managed policy's additions for a standard + * OAuth provider, and the fixed user-token policy for Slack. + */ +function optionScopesFor(provider: CredentialGroupProvider): string[] { + if (!isCredentialGroupStandardOAuthProvider(provider)) return [...SLACK_MANAGED_USER_SCOPES] + const service = getCredentialGroupProviderService(provider) + const policy = getManagedOAuthConnectorPolicy(service.providerId) + expect(policy).toBeDefined() + return [...new Set([...service.scopes, ...(policy?.additionalScopes ?? [])])] +} + const permissionScoped = Object.values(CONNECTOR_META_REGISTRY).filter( (meta) => meta.permissionScopedListing !== undefined ) @@ -50,32 +67,22 @@ describe('permission-scoped connector listings', () => { 'outlook', 'salesforce', 'sharepoint', + 'slack', 'zoom', ]) }) it.each(permissionScoped.map((meta) => [meta.id, meta] as const))( - '%s authenticates through a managed OAuth provider whose option scopes cover its read scopes', + '%s authenticates through a Credential Group provider whose option scopes cover its read scopes', (_id, meta) => { expect(meta.auth.mode).toBe('oauth') if (meta.auth.mode !== 'oauth') return - const policy = getManagedOAuthConnectorPolicy(meta.auth.provider) - expect(policy).toBeDefined() - if (!policy) return - - const groupProvider = getCredentialGroupStandardOAuthProviderFromProviderId( - meta.auth.provider - ) - expect(groupProvider).toBeDefined() - - const optionScopes = [ - ...new Set([ - ...getCredentialGroupProviderService(groupProvider).scopes, - ...policy.additionalScopes, - ]), - ] - expect(policy.hasRequiredScopes(optionScopes, meta.auth.requiredScopes ?? [])).toBe(true) + const provider = getCredentialGroupProviderFromProviderId(meta.auth.provider) + const adapter = getCredentialGroupProviderAdapter(provider) + expect( + adapter.hasRequiredScopes(optionScopesFor(provider), meta.auth.requiredScopes ?? []) + ).toBe(true) } ) diff --git a/apps/sim/connectors/slack/meta.ts b/apps/sim/connectors/slack/meta.ts index 5bd5ebaf9b3..3a398fd0b0a 100644 --- a/apps/sim/connectors/slack/meta.ts +++ b/apps/sim/connectors/slack/meta.ts @@ -22,6 +22,17 @@ export const slackConnectorMeta: ConnectorMeta = { ], }, + /** + * `conversations.list` under a person's own token returns the public + * channels of their workspace and the private channels they belong to, + * exactly what they may read, so one member's crawl is their access. The + * channel selection is a cap: it would hide part of a member's corpus, and + * the per-member crawl indexes every channel the member can see instead. + * `maxMessages` bounds each channel document's window, not which channels + * are listed, so it is not a cap. + */ + permissionScopedListing: { capFieldIds: ['channel'] }, + configFields: [ { id: 'channelSelector', diff --git a/apps/sim/connectors/slack/slack.test.ts b/apps/sim/connectors/slack/slack.test.ts new file mode 100644 index 00000000000..2112b6e66ca --- /dev/null +++ b/apps/sim/connectors/slack/slack.test.ts @@ -0,0 +1,191 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { slackConnectorMeta } from '@/connectors/slack/meta' +import { slackConnector } from '@/connectors/slack/slack' +import { CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' + +const GENERAL = { + id: 'C0GENERAL', + name: 'general', + topic: { value: 'Company-wide announcements' }, + purpose: { value: '' }, +} +const PLATFORM = { id: 'G0PLATFORM', name: 'platform', topic: { value: '' } } + +const MESSAGES = [ + { type: 'message', user: 'U2', text: 'Shipping today', ts: '1700000200.000100' }, + { type: 'message', user: 'U1', text: 'Morning', ts: '1700000100.000100' }, + { type: 'message', subtype: 'channel_join', user: 'U1', text: 'joined', ts: '1700000000.000100' }, +] + +function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) +} + +const requestedUrls: string[] = [] +const fetchMock = vi.fn<(input: string | URL | Request, init?: RequestInit) => Promise>() + +/** Channels returned by `conversations.list`; per-test overridable. */ +let listedChannels: Record[] = [GENERAL, PLATFORM] +/** `next_cursor` returned by `conversations.list`; per-test overridable. */ +let listNextCursor = '' +/** Messages returned by `conversations.history`; per-test overridable. */ +let history: Record[] = MESSAGES +/** Whether `conversations.info` reports the channel as missing; per-test overridable. */ +let channelMissing = false + +beforeEach(() => { + requestedUrls.length = 0 + listedChannels = [GENERAL, PLATFORM] + listNextCursor = '' + history = MESSAGES + channelMissing = false + fetchMock.mockReset() + fetchMock.mockImplementation(async (input) => { + const url = new URL(String(input)) + requestedUrls.push(`${url.pathname}?${url.searchParams.toString()}`) + switch (url.pathname) { + case '/api/auth.test': + return jsonResponse({ ok: true, team_id: 'T0TEAM' }) + case '/api/conversations.list': + return jsonResponse({ + ok: true, + channels: listedChannels, + response_metadata: { next_cursor: listNextCursor }, + }) + case '/api/conversations.info': + return channelMissing + ? jsonResponse({ ok: false, error: 'channel_not_found' }) + : jsonResponse({ ok: true, channel: GENERAL }) + case '/api/conversations.history': + return jsonResponse({ ok: true, messages: history, response_metadata: {} }) + case '/api/users.info': { + const id = url.searchParams.get('user') + return jsonResponse({ ok: true, user: { id, name: id, real_name: `Person ${id}` } }) + } + default: + return jsonResponse({ ok: false, error: 'unknown_method' }) + } + }) + vi.stubGlobal('fetch', fetchMock) +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +const requested = (method: string) => requestedUrls.filter((url) => url.includes(`/${method}?`)) + +describe('slack connector meta', () => { + it('crawls per member with the channel selection as the only listing cap', () => { + expect(slackConnectorMeta.permissionScopedListing).toEqual({ capFieldIds: ['channel'] }) + }) +}) + +describe('listDocuments', () => { + it('lists configured channels as deferred stubs without reading their history', async () => { + const syncContext: Record = { syncRunId: 'run-1' } + const result = await slackConnector.listDocuments( + 'token', + { channel: ['C0GENERAL'] }, + undefined, + syncContext + ) + + expect(result.hasMore).toBe(false) + expect(result.documents).toEqual([ + expect.objectContaining({ + externalId: 'C0GENERAL', + title: '#general', + content: '', + contentDeferred: true, + estimatedBytes: CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, + contentHash: 'slack-listing:C0GENERAL:run-1', + sourceUrl: 'https://app.slack.com/client/T0TEAM/C0GENERAL', + metadata: expect.objectContaining({ channelName: 'general' }), + }), + ]) + expect(requested('conversations.history')).toHaveLength(0) + }) + + it('lists every readable channel when none is configured, paging through the cursor', async () => { + listNextCursor = 'page-2' + const syncContext: Record = { + syncRunId: 'run-1', + ...PER_MEMBER_LISTING_CONTEXT, + } + const first = await slackConnector.listDocuments( + 'token', + { channel: 0 }, + undefined, + syncContext + ) + + expect(first.documents.map((doc) => doc.externalId)).toEqual(['C0GENERAL', 'G0PLATFORM']) + expect(first).toMatchObject({ hasMore: true, nextCursor: 'page-2' }) + expect(requested('conversations.list')[0]).toContain('types=public_channel%2Cprivate_channel') + expect(requested('conversations.list')[0]).toContain('exclude_archived=true') + + listNextCursor = '' + listedChannels = [] + const second = await slackConnector.listDocuments( + 'token', + { channel: 0 }, + 'page-2', + syncContext + ) + expect(second).toEqual({ documents: [], nextCursor: undefined, hasMore: false }) + expect(requested('conversations.list')[1]).toContain('cursor=page-2') + }) + + it('gives every member of one run the same stub for a channel', async () => { + const ada = await slackConnector.listDocuments('ada', {}, undefined, { syncRunId: 'run-7' }) + const bob = await slackConnector.listDocuments('bob', {}, undefined, { syncRunId: 'run-7' }) + expect(ada.documents[0].contentHash).toBe(bob.documents[0].contentHash) + }) + + it('changes the stub between runs so each run re-reads the channel', async () => { + const first = await slackConnector.listDocuments('token', {}, undefined, {}) + const second = await slackConnector.listDocuments('token', {}, undefined, {}) + expect(first.documents[0].contentHash).not.toBe(second.documents[0].contentHash) + }) +}) + +describe('getDocument', () => { + it('builds the transcript under a header with the real content hash', async () => { + const doc = await slackConnector.getDocument('token', {}, 'C0GENERAL', {}) + + expect(doc).toMatchObject({ + externalId: 'C0GENERAL', + title: '#general', + contentHash: 'slack-v3:C0GENERAL:1700000000.000100:1700000200.000100:3:noedit:noreply:0', + metadata: expect.objectContaining({ channelName: 'general', messageCount: 2 }), + }) + expect(doc?.content).toBe( + [ + 'Channel: #general', + 'Topic: Company-wide announcements', + '', + '[2023-11-14T22:15:00.000Z] Person U1: Morning', + '[2023-11-14T22:16:40.000Z] Person U2: Shipping today', + ].join('\n') + ) + }) + + it('keeps a channel with no messages as a live document', async () => { + history = [] + const doc = await slackConnector.getDocument('token', {}, 'C0GENERAL', {}) + expect(doc?.content).toBe('Channel: #general\nTopic: Company-wide announcements\n') + expect(doc?.metadata?.messageCount).toBe(0) + }) + + it('returns null only for a channel Slack no longer knows', async () => { + channelMissing = true + await expect(slackConnector.getDocument('token', {}, 'C0GONE', {})).resolves.toBeNull() + }) +}) diff --git a/apps/sim/connectors/slack/slack.ts b/apps/sim/connectors/slack/slack.ts index 00ea03c55be..7d61b5f4f7f 100644 --- a/apps/sim/connectors/slack/slack.ts +++ b/apps/sim/connectors/slack/slack.ts @@ -1,14 +1,26 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_MESSAGES, slackConnectorMeta } from '@/connectors/slack/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + BoundedLines, + CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('SlackConnector') const SLACK_API_BASE = 'https://slack.com/api' const MESSAGES_PER_PAGE = 200 +/** Page size for `conversations.list`; Slack recommends staying well under its 1000 maximum. */ +const CHANNELS_PER_PAGE = 200 +/** The conversation kinds a channel listing walks; DMs need scopes the connector does not request. */ +const LISTED_CHANNEL_TYPES = 'public_channel,private_channel' +/** `syncContext` key holding this run's listing token when the engine supplies no run id. */ +const LISTING_TOKEN_KEY = '_slackListingToken' /** * Message subtypes that carry no user-authored text (channel events, bot @@ -373,14 +385,14 @@ function walkBlockText(node: unknown, out: string[]): void { * Each entry: "[ISO timestamp] username: message text" (text may span lines * when the message has rich attachment/block content). */ -async function formatMessages( +async function appendMessages( accessToken: string, + lines: BoundedLines, messages: SlackMessage[], syncContext?: Record -): Promise { - const lines: string[] = [] - - // Process in reverse so oldest messages come first +): Promise { + let appended = 0 + /** Slack returns newest first; the transcript reads oldest first. */ const chronological = [...messages].reverse() for (const msg of chronological) { @@ -400,10 +412,11 @@ async function formatMessages( ? await resolveUserName(accessToken, msg.user, syncContext) : msg.username || 'unknown' - lines.push(`[${timestamp}] ${userName}: ${content}`) + if (!lines.push(`[${timestamp}] ${userName}: ${content}`)) break + appended += 1 } - return lines.join('\n') + return appended } /** @@ -488,8 +501,79 @@ async function resolveTeamId( } } +function channelUrl(channel: SlackChannel, teamId: string | undefined): string { + return teamId + ? `https://app.slack.com/client/${teamId}/${channel.id}` + : `https://app.slack.com/client/${channel.id}` +} + +/** + * A token that is stable for one sync run and different on the next. A channel + * listing carries no signal of new messages (`conversations.list` returns no + * last-message timestamp, and reading one per channel would cost a Tier 3 call + * per channel per listing), so every listed channel is hydrated each run and + * the real hash `getDocument` computes decides whether anything is re-indexed: + * an unchanged channel costs one history page and no embedding. The member + * engine sets `syncRunId` on every member's context, so members listing the + * same channel agree on its stub. + */ +function listingToken(syncContext?: Record): string { + const runId = syncContext?.syncRunId + if (typeof runId === 'string' && runId) return runId + const cached = syncContext?.[LISTING_TOKEN_KEY] + if (typeof cached === 'string') return cached + const token = generateId() + if (syncContext) syncContext[LISTING_TOKEN_KEY] = token + return token +} + +/** The deferred listing stub for a channel; its transcript is fetched in `getDocument`. */ +function channelToStub( + channel: SlackChannel, + teamId: string | undefined, + syncContext?: Record +): ExternalDocument { + return { + externalId: channel.id, + title: `#${channel.name}`, + content: '', + contentDeferred: true, + estimatedBytes: CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, + mimeType: 'text/plain', + sourceUrl: channelUrl(channel, teamId), + contentHash: `slack-listing:${channel.id}:${listingToken(syncContext)}`, + metadata: { + channelName: channel.name, + topic: channel.topic?.value, + purpose: channel.purpose?.value, + }, + } +} + +/** One page of every unarchived channel the token can read. */ +async function listChannelsPage( + accessToken: string, + cursor: string | undefined +): Promise<{ channels: SlackChannel[]; nextCursor: string | undefined }> { + const params: Record = { + types: LISTED_CHANNEL_TYPES, + limit: String(CHANNELS_PER_PAGE), + exclude_archived: 'true', + } + if (cursor) params.cursor = cursor + const data = await slackApiGet('conversations.list', accessToken, params) + const responseMeta = data.response_metadata as { next_cursor?: string } | undefined + return { + channels: (data.channels as SlackChannel[]) || [], + nextCursor: responseMeta?.next_cursor || undefined, + } +} + /** - * Builds a channel document payload shared by `listDocuments` and `getDocument`. + * Builds a channel's document: a header naming the channel, its topic and its + * purpose, then the newest `maxMessages` messages oldest first. The header + * means a channel with no messages in the window is still a live document, so + * the sync engine never mistakes an empty channel for a deleted one. * * The `contentHash` is derived from stable Slack metadata — channel ID, the * newest message `ts`, and the message count — rather than the formatted text. @@ -517,8 +601,14 @@ async function buildSlackChannelDocument( maxMessages ) - const content = await formatMessages(accessToken, messages, syncContext) - const messageCount = messages.length + const lines = new BoundedLines() + lines.push(`Channel: #${channel.name}`) + const topic = channel.topic?.value?.trim() + if (topic) lines.push(`Topic: ${topic}`) + const purpose = channel.purpose?.value?.trim() + if (purpose) lines.push(`Purpose: ${purpose}`) + lines.push('') + const messageCount = await appendMessages(accessToken, lines, messages, syncContext) /** * Edit/thread fingerprint: max(edited.ts) and max(latest_reply) across the @@ -540,39 +630,49 @@ async function buildSlackChannelDocument( * `latest_reply` alone misses reply edits and deletes. Folding `reply_count` * in catches deletes (count drops) but still cannot detect reply edits * without fetching `conversations.replies` for each parent. + * + * The `slack-v3` prefix forces a one-time re-index of channels indexed + * before the document gained its header and size ceiling; `slack-v2` did the + * same when attachment and Block Kit content started being extracted. + * Per-message `ts` and the window are unchanged by either, so without the + * bump the hash would match and the richer content would never be embedded. */ - /** - * `slack-v2` prefix forces a one-time re-sync for channels indexed before - * we started extracting attachment + Block Kit content from bot messages. - * Per-message `ts` and `messageCount` are unchanged, so without the version - * bump the hash would match and richer content would not be re-embedded. - */ - const contentHash = `slack-v2:${channel.id}:${oldestTs ?? 'empty'}:${lastActivityTs ?? 'empty'}:${messageCount}:${maxEditTs || 'noedit'}:${maxReplyTs || 'noreply'}:${totalReplies}` + const contentHash = `slack-v3:${channel.id}:${oldestTs ?? 'empty'}:${lastActivityTs ?? 'empty'}:${messages.length}:${maxEditTs || 'noedit'}:${maxReplyTs || 'noreply'}:${totalReplies}` - return { content, contentHash, messageCount, lastActivityTs } + return { content: lines.join(), contentHash, messageCount, lastActivityTs } } export const slackConnector: ConnectorConfig = { ...slackConnectorMeta, + /** + * Lists the configured channels, or, when none are configured, every channel + * the token can read. A members-mode crawl clears the channel selection, so + * each member's listing is their whole view of the workspace. Listing stubs + * are deferred: the transcript is fetched in `getDocument`, once per channel + * per run, however many members list it. + */ listDocuments: async ( accessToken: string, sourceConfig: Record, - _cursor?: string, + cursor?: string, syncContext?: Record ): Promise => { const channelInputs = parseMultiValue(sourceConfig.channel) + const teamId = await resolveTeamId(accessToken, syncContext) + if (channelInputs.length === 0) { - throw new Error('At least one channel is required') + logger.info('Listing every readable Slack channel', { hasCursor: Boolean(cursor) }) + const page = await listChannelsPage(accessToken, cursor) + return { + documents: page.channels.map((channel) => channelToStub(channel, teamId, syncContext)), + nextCursor: page.nextCursor, + hasMore: page.nextCursor !== undefined, + } } - const maxMessages = resolveMaxMessages(sourceConfig.maxMessages) - - logger.info('Syncing Slack channels', { channels: channelInputs, maxMessages }) - - const teamId = await resolveTeamId(accessToken, syncContext) + logger.info('Listing configured Slack channels', { channels: channelInputs }) const documents: ExternalDocument[] = [] - for (const channelInput of channelInputs) { const channel = await resolveChannel(accessToken, channelInput) if (!channel) { @@ -585,45 +685,15 @@ export const slackConnector: ConnectorConfig = { */ throw new Error(`Channel not found: ${channelInput}`) } - - const { content, contentHash, messageCount, lastActivityTs } = - await buildSlackChannelDocument(accessToken, channel, maxMessages, syncContext) - if (!content.trim()) { - logger.info(`No messages found in channel: #${channel.name}`) - continue - } - - const sourceUrl = teamId - ? `https://app.slack.com/client/${teamId}/${channel.id}` - : `https://app.slack.com/client/${channel.id}` - - documents.push({ - externalId: channel.id, - title: `#${channel.name}`, - content, - mimeType: 'text/plain', - sourceUrl, - contentHash, - metadata: { - channelName: channel.name, - messageCount, - lastActivity: lastActivityTs ? formatSlackTimestamp(lastActivityTs) : undefined, - topic: channel.topic?.value, - purpose: channel.purpose?.value, - }, - }) + documents.push(channelToStub(channel, teamId, syncContext)) } /** - * All channels are processed in one call — the multi-select UI keeps the - * count small, and each channel is an independent document with its own - * `externalId` and `contentHash`, so the sync engine treats them as - * independent documents. + * Configured channels are listed in one call — the multi-select UI keeps + * the count small, and each channel is an independent document with its + * own `externalId` and `contentHash`. */ - return { - documents, - hasMore: false, - } + return { documents, hasMore: false } }, getDocument: async ( @@ -640,19 +710,14 @@ export const slackConnector: ConnectorConfig = { const { content, contentHash, messageCount, lastActivityTs } = await buildSlackChannelDocument(accessToken, channel, maxMessages, syncContext) - if (!content.trim()) return null - const teamId = await resolveTeamId(accessToken, syncContext) - const sourceUrl = teamId - ? `https://app.slack.com/client/${teamId}/${channel.id}` - : `https://app.slack.com/client/${channel.id}` return { externalId: channel.id, title: `#${channel.name}`, content, mimeType: 'text/plain', - sourceUrl, + sourceUrl: channelUrl(channel, teamId), contentHash, metadata: { channelName: channel.name, diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.ts index a87fdae1ed3..43e84509a09 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.ts @@ -8,7 +8,9 @@ import { } from 'better-auth/oauth2' import { type ConnectorProviderConfig, + getManagedOAuthConnectorPolicy, getManagedOAuthConnectorProviderConfig, + type ManagedOAuthConnectorConfig, } from '@/lib/auth/connectors/managed-oauth' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { credentialGroupOAuthNonceMatches } from '@/lib/credential-groups/oauth-state' @@ -134,6 +136,24 @@ async function resolveOAuthEndpoints( } } +/** + * The provider's scope policy on its own. Comparing scopes needs no OAuth + * client, so a saved option can be validated against a connector wherever the + * client is not configured, which `getCurrentProvider` would refuse. + */ +function getScopePolicy( + provider: CredentialGroupStandardOAuthProvider +): ManagedOAuthConnectorConfig { + const service = getCredentialGroupProviderService(provider) + const policy = getManagedOAuthConnectorPolicy(service.providerId) + if (!policy) { + throw new CredentialGroupProviderConfigurationError( + `Managed ${service.name} authorization is not configured` + ) + } + return policy +} + function getCurrentProvider( provider: CredentialGroupStandardOAuthProvider ): CurrentStandardOAuthProvider { @@ -357,10 +377,7 @@ export function createStandardOAuthCredentialGroupProviderAdapter( } }, hasRequiredScopes(grantedScopes, requiredScopes) { - return getCurrentProvider(provider).connector.managedOAuth.hasRequiredScopes( - grantedScopes, - requiredScopes - ) + return getScopePolicy(provider).hasRequiredScopes(grantedScopes, requiredScopes) }, async refreshToken(refreshToken) { return refreshOAuthToken(getCurrentProvider(provider).policy.providerId, refreshToken) diff --git a/apps/sim/lib/knowledge/connectors/member-access.test.ts b/apps/sim/lib/knowledge/connectors/member-access.test.ts index 276b201ec65..625464a005b 100644 --- a/apps/sim/lib/knowledge/connectors/member-access.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-access.test.ts @@ -53,6 +53,7 @@ vi.mock('@/lib/credentials/managed-oauth', () => ({ import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' import { CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT } from '@/lib/credential-groups/limits' +import { SLACK_MANAGED_USER_SCOPES } from '@/lib/credential-groups/slack-managed-user-scopes' import { findListingCapViolation, grantKnowledgeConnectorCredentialAccess, @@ -480,6 +481,64 @@ describe('knowledge connector member access', () => { ).toEqual({ ok: true, option: driveOption }) }) + describe('a Slack option, whose members authorize through the workspace custom app', () => { + const slackMeta = { + name: 'Slack', + auth: { + mode: 'oauth' as const, + provider: 'slack' as const, + requiredScopes: ['channels:read', 'channels:history', 'groups:read', 'groups:history'], + }, + permissionScopedListing: { capFieldIds: ['channel'] }, + configFields: [{ id: 'channel', title: 'Channels', type: 'short-input' as const }], + } + const slackOption = { + ...driveOption, + id: 'option-slack', + provider: 'slack', + label: 'Slack', + authorizationAppId: 'slack:app', + requiredScopes: [...SLACK_MANAGED_USER_SCOPES], + } + const slackGroup = { status: 'active' as const, options: [slackOption] } + + it('accepts the binding through the Slack scope policy', () => { + expect( + validateKnowledgeConnectorMembersBinding({ + connectorMeta: slackMeta, + group: slackGroup, + credentialGroupOptionId: 'option-slack', + sourceConfig: { maxMessages: '500' }, + }) + ).toEqual({ ok: true, option: slackOption }) + }) + + it('rejects a channel selection as a listing cap', () => { + const result = validateKnowledgeConnectorMembersBinding({ + connectorMeta: slackMeta, + group: slackGroup, + credentialGroupOptionId: 'option-slack', + sourceConfig: { channel: ['general'] }, + }) + expect(result.ok).toBe(false) + if (!result.ok) expect(result.message).toContain('Channels cannot be set') + }) + + it('rejects an option missing a history scope', () => { + const result = validateKnowledgeConnectorMembersBinding({ + connectorMeta: slackMeta, + group: { + ...slackGroup, + options: [{ ...slackOption, requiredScopes: ['channels:read', 'groups:read'] }], + }, + credentialGroupOptionId: 'option-slack', + sourceConfig: {}, + }) + expect(result.ok).toBe(false) + if (!result.ok) expect(result.message).toContain('every permission') + }) + }) + it.each([ [ 'a connector whose listing is not permission scoped', diff --git a/apps/sim/lib/knowledge/connectors/member-access.ts b/apps/sim/lib/knowledge/connectors/member-access.ts index 4dcae8586fc..65cf062d76f 100644 --- a/apps/sim/lib/knowledge/connectors/member-access.ts +++ b/apps/sim/lib/knowledge/connectors/member-access.ts @@ -1,7 +1,6 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import type { CredentialGroupOptionConfig } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { getManagedOAuthConnectorPolicy } from '@/lib/auth/connectors/managed-oauth' import { OrchestrationError } from '@/lib/core/orchestration/types' import { type CredentialGroupKnowledgeConnectorAccess, @@ -19,6 +18,7 @@ import { loadManagedCredentialGroupBinding, } from '@/lib/credential-groups/credentials' import { CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT } from '@/lib/credential-groups/limits' +import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry' import { getCredentialGroupProviderId, isCredentialGroupProvider, @@ -453,16 +453,8 @@ export function validateKnowledgeConnectorMembersBinding(input: { message: `Credential option collects ${option.provider} accounts, but ${connectorMeta.name} needs ${connectorMeta.auth.provider}`, } } - const scopePolicy = getManagedOAuthConnectorPolicy(connectorMeta.auth.provider) - if (!scopePolicy) { - return { - ok: false, - message: `${connectorMeta.auth.provider} is not a managed OAuth provider`, - } - } - if ( - !scopePolicy.hasRequiredScopes(option.requiredScopes, connectorMeta.auth.requiredScopes ?? []) - ) { + const adapter = getCredentialGroupProviderAdapter(option.provider) + if (!adapter.hasRequiredScopes(option.requiredScopes, connectorMeta.auth.requiredScopes ?? [])) { return { ok: false, message: `Credential option does not request every permission ${connectorMeta.name} needs to read the source`, diff --git a/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts b/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts index a7aa1891de9..d3f8e5590ec 100644 --- a/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/credential-groups/enrollments', () => ({ createCredentialGroupInvitationLink: vi.fn(), @@ -20,12 +20,85 @@ vi.mock('@/lib/credential-groups/service', () => ({ })) vi.mock('@/lib/workspaces/permissions/utils', () => ({ getUsersWithPermissions: vi.fn() })) +import { createCredentialGroup, listCredentialGroups } from '@/lib/credential-groups/service' import { chooseSharedMembersBinding, deriveViewerConnectorMembership, pickProvisionedGroupName, + provisionKnowledgeConnectorMembersBinding, } from '@/lib/knowledge/connectors/member-provisioning' +describe('provisionKnowledgeConnectorMembersBinding', () => { + const slackMeta = { name: 'Slack', auth: { mode: 'oauth' as const, provider: 'slack' } } + const gmailMeta = { name: 'Gmail', auth: { mode: 'oauth' as const, provider: 'google-email' } } + const slackOption = (id: string, configurationStatus = 'ready') => ({ + id, + provider: 'slack', + label: 'Slack', + required: true, + status: 'active', + slackBotCredentialId: 'cred-bot', + configurationStatus, + }) + const group = (id: string, options: unknown[]) => ({ id, name: id, status: 'active', options }) + const provision = (meta: typeof slackMeta) => + provisionKnowledgeConnectorMembersBinding({ + workspaceId: 'ws-1', + connectorMeta: meta, + userId: 'user-1', + }) + + beforeEach(() => { + vi.mocked(listCredentialGroups).mockReset() + vi.mocked(createCredentialGroup).mockReset() + }) + + it('adopts the one Slack option an admin has already set up', async () => { + vi.mocked(listCredentialGroups).mockResolvedValue([group('g-1', [slackOption('o-1')])] as never) + + await expect(provision(slackMeta)).resolves.toEqual({ + credentialGroupId: 'g-1', + credentialGroupOptionId: 'o-1', + }) + expect(createCredentialGroup).not.toHaveBeenCalled() + }) + + it('points at Settings when no ready Slack option exists, since it cannot create one', async () => { + vi.mocked(listCredentialGroups).mockResolvedValue([ + group('g-1', [slackOption('o-1', 'not_configured')]), + ] as never) + + await expect(provision(slackMeta)).rejects.toThrow('in Settings') + expect(createCredentialGroup).not.toHaveBeenCalled() + }) + + it('leaves two Slack options for the admin to choose between', async () => { + vi.mocked(listCredentialGroups).mockResolvedValue([ + group('g-1', [slackOption('o-1')]), + group('g-2', [slackOption('o-2')]), + ] as never) + + await expect(provision(slackMeta)).rejects.toThrow('choose which one') + }) + + it('still creates a group for a standard OAuth provider', async () => { + vi.mocked(listCredentialGroups).mockResolvedValue([]) + vi.mocked(createCredentialGroup).mockResolvedValue({ + id: 'g-new', + options: [{ id: 'o-new' }], + } as never) + + await expect(provision(gmailMeta)).resolves.toEqual({ + credentialGroupId: 'g-new', + credentialGroupOptionId: 'o-new', + }) + expect(createCredentialGroup).toHaveBeenCalledWith('ws-1', 'user-1', { + name: 'Gmail', + options: [{ provider: 'gmail', label: 'Gmail', required: true }], + }) + }) +}) + describe('pickProvisionedGroupName', () => { it('names the group after the connector and steps past taken names', () => { expect(pickProvisionedGroupName('Google Drive', [])).toBe('Google Drive') diff --git a/apps/sim/lib/knowledge/connectors/member-provisioning.ts b/apps/sim/lib/knowledge/connectors/member-provisioning.ts index 6d5e93a9d20..3b9fa1ec37e 100644 --- a/apps/sim/lib/knowledge/connectors/member-provisioning.ts +++ b/apps/sim/lib/knowledge/connectors/member-provisioning.ts @@ -17,9 +17,11 @@ import { inviteCredentialGroupEnrollment, } from '@/lib/credential-groups/enrollments' import { + type CredentialGroupProvider, + getCredentialGroupProviderFromProviderId, getCredentialGroupProviderId, - getCredentialGroupStandardOAuthProviderFromProviderId, isCredentialGroupProvider, + isCredentialGroupStandardOAuthProvider, } from '@/lib/credential-groups/providers' import { createCredentialGroup, listCredentialGroups } from '@/lib/credential-groups/service' import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' @@ -113,9 +115,9 @@ export async function provisionKnowledgeConnectorMembersBinding(input: { throw new OrchestrationError('validation', 'Only an OAuth connector can sync per member') } const providerId = connectorMeta.auth.provider - let provider: ReturnType + let provider: CredentialGroupProvider try { - provider = getCredentialGroupStandardOAuthProviderFromProviderId(providerId) + provider = getCredentialGroupProviderFromProviderId(providerId) } catch { throw new OrchestrationError( 'validation', @@ -128,7 +130,7 @@ export async function provisionKnowledgeConnectorMembersBinding(input: { for (const group of groups) { if (group.status !== 'active') continue for (const option of group.options) { - if (option.status !== 'active') continue + if (option.status !== 'active' || option.configurationStatus !== 'ready') continue if (!isCredentialGroupProvider(option.provider)) continue if (getCredentialGroupProviderId(option.provider) !== providerId) continue candidates.push({ credentialGroupId: group.id, credentialGroupOptionId: option.id }) @@ -149,6 +151,22 @@ export async function provisionKnowledgeConnectorMembersBinding(input: { ) } + if (!isCredentialGroupStandardOAuthProvider(provider)) { + /** + * A Slack option authorizes through the workspace's own Slack app, which + * only an admin can configure in Settings, so no group can be created + * here. The one option already set up for it is what the connector was + * meant to crawl through; anything else needs the admin's choice. + */ + if (candidates.length === 1) return candidates[0] + throw new OrchestrationError( + 'validation', + candidates.length === 0 + ? `Add a ${connectorMeta.name} option to a Credential Group in Settings, using your own ${connectorMeta.name} app, then connect again` + : `Several Credential Groups collect ${connectorMeta.name} accounts; choose which one this connector syncs through` + ) + } + const name = pickProvisionedGroupName( connectorMeta.name, groups.map((group) => group.name) From d9956152d779f4d5f56ef200a30bdb8967706603 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 15:44:35 -0700 Subject: [PATCH 2/3] fix(connectors): keep the newest messages when a chat transcript reaches the size limit --- .../sim/connectors/google-chat/google-chat.ts | 21 +++-- apps/sim/connectors/slack/slack.ts | 26 +++--- apps/sim/connectors/utils.test.ts | 33 +++++++ apps/sim/connectors/utils.ts | 91 +++++++++++++++---- 4 files changed, 133 insertions(+), 38 deletions(-) diff --git a/apps/sim/connectors/google-chat/google-chat.ts b/apps/sim/connectors/google-chat/google-chat.ts index 1d694ab6c35..107e2acea72 100644 --- a/apps/sim/connectors/google-chat/google-chat.ts +++ b/apps/sim/connectors/google-chat/google-chat.ts @@ -328,24 +328,27 @@ function formatSpaceContent( space: Space, messages: ChatMessage[] ): { content: string; messageCount: number } { - const parts = new BoundedLines() - parts.push(`Space: ${spaceTitle(space)}`) + /** The newest messages survive when the window does not fit; the header always does. */ + const parts = new BoundedLines(CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, 'last') + parts.pin(`Space: ${spaceTitle(space)}`) const description = space.spaceDetails?.description?.trim() - if (description) parts.push(`Description: ${description}`) + if (description) parts.pin(`Description: ${description}`) const guidelines = space.spaceDetails?.guidelines?.trim() - if (guidelines) parts.push(`Guidelines: ${guidelines}`) + if (guidelines) parts.pin(`Guidelines: ${guidelines}`) - let messageCount = 0 + let headed = false for (const message of messages) { const text = message.text?.trim() || message.fallbackText?.trim() if (!text) continue - if (messageCount === 0) parts.push('', '--- Messages ---') + if (!headed) { + parts.pin('', '--- Messages ---') + headed = true + } const timestamp = message.createTime ?? '' - if (!parts.push(`[${timestamp}] ${senderLabel(message.sender)}: ${text}`)) break - messageCount += 1 + parts.push(`[${timestamp}] ${senderLabel(message.sender)}: ${text}`) } - return { content: parts.join(), messageCount } + return { content: parts.join(), messageCount: parts.count } } export const googleChatConnector: ConnectorConfig = { diff --git a/apps/sim/connectors/slack/slack.ts b/apps/sim/connectors/slack/slack.ts index 7d61b5f4f7f..9954f7ac60b 100644 --- a/apps/sim/connectors/slack/slack.ts +++ b/apps/sim/connectors/slack/slack.ts @@ -385,13 +385,17 @@ function walkBlockText(node: unknown, out: string[]): void { * Each entry: "[ISO timestamp] username: message text" (text may span lines * when the message has rich attachment/block content). */ +/** + * Appends the messages to the transcript oldest first. The transcript keeps + * its newest messages when the window does not fit, so a message is skipped + * only when it cannot fit on its own. + */ async function appendMessages( accessToken: string, lines: BoundedLines, messages: SlackMessage[], syncContext?: Record -): Promise { - let appended = 0 +): Promise { /** Slack returns newest first; the transcript reads oldest first. */ const chronological = [...messages].reverse() @@ -412,11 +416,8 @@ async function appendMessages( ? await resolveUserName(accessToken, msg.user, syncContext) : msg.username || 'unknown' - if (!lines.push(`[${timestamp}] ${userName}: ${content}`)) break - appended += 1 + lines.push(`[${timestamp}] ${userName}: ${content}`) } - - return appended } /** @@ -601,14 +602,15 @@ async function buildSlackChannelDocument( maxMessages ) - const lines = new BoundedLines() - lines.push(`Channel: #${channel.name}`) + const lines = new BoundedLines(CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, 'last') + lines.pin(`Channel: #${channel.name}`) const topic = channel.topic?.value?.trim() - if (topic) lines.push(`Topic: ${topic}`) + if (topic) lines.pin(`Topic: ${topic}`) const purpose = channel.purpose?.value?.trim() - if (purpose) lines.push(`Purpose: ${purpose}`) - lines.push('') - const messageCount = await appendMessages(accessToken, lines, messages, syncContext) + if (purpose) lines.pin(`Purpose: ${purpose}`) + lines.pin('') + await appendMessages(accessToken, lines, messages, syncContext) + const messageCount = lines.count /** * Edit/thread fingerprint: max(edited.ts) and max(latest_reply) across the diff --git a/apps/sim/connectors/utils.test.ts b/apps/sim/connectors/utils.test.ts index b433006d63c..e023bc2887e 100644 --- a/apps/sim/connectors/utils.test.ts +++ b/apps/sim/connectors/utils.test.ts @@ -1662,4 +1662,37 @@ describe('BoundedLines', () => { expect(lines.push('éé')).toBe(true) expect(lines.push('é')).toBe(false) }) + + describe('keeping the last records', () => { + it('lets the oldest records go so the newest fit, under a header that stays', () => { + const lines = new BoundedLines(24, 'last') + lines.pin('# room') + expect(lines.push('one')).toBe(true) + expect(lines.push('two')).toBe(true) + expect(lines.push('three')).toBe(true) + expect(lines.push('four')).toBe(true) + expect(lines.count).toBe(3) + expect(lines.join()).toBe( + '# room\n[Truncated: earlier text was left out to fit the size limit]\ntwo\nthree\nfour' + ) + }) + + it('refuses only a record that cannot fit on its own and carries on', () => { + const lines = new BoundedLines(12, 'last') + expect(lines.push('a very long record')).toBe(false) + expect(lines.push('short')).toBe(true) + expect(lines.push('next')).toBe(true) + expect(lines.count).toBe(2) + expect(lines.join()).toBe( + '[Truncated: earlier text was left out to fit the size limit]\nshort\nnext' + ) + }) + + it('joins the header and records plainly when everything fits', () => { + const lines = new BoundedLines(64, 'last') + lines.pin('# room', '') + lines.push('hello') + expect(lines.join()).toBe('# room\n\nhello') + }) + }) }) diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index e3bc5485113..412c58b0753 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -806,39 +806,96 @@ export function isSkippableMicrosoftGraphFolderError( */ export const CONNECTOR_TEXT_DOCUMENT_MAX_BYTES = 12 * 1024 * 1024 -const TRUNCATION_NOTICE = '[Truncated: the indexed text reached the size limit]' +const TRAILING_TRUNCATION_NOTICE = '[Truncated: the indexed text reached the size limit]' +const LEADING_TRUNCATION_NOTICE = '[Truncated: earlier text was left out to fit the size limit]' /** - * Accumulates newline-joined text under a byte ceiling. A record is appended - * whole or not at all, so a truncated document never ends mid-message, and the - * output carries a notice when something was left out. + * Which end of the stream survives when it does not fit: `first` keeps what + * was pushed first and refuses the rest (a mail thread, whose root message is + * the context), `last` keeps what was pushed last and lets older records go + * (a chat transcript, whose newest messages are the ones people search for). + */ +export type BoundedLinesKeep = 'first' | 'last' + +interface BoundedRecord { + lines: string[] + bytes: number +} + +function byteSize(lines: readonly string[]): number { + let size = 0 + for (const line of lines) size += Buffer.byteLength(line, 'utf8') + 1 + return size +} + +/** + * Accumulates newline-joined text under a byte ceiling. A record is kept + * whole or not at all, so a truncated document never ends mid-message, and + * the output carries a notice where something was left out. */ export class BoundedLines { - private readonly lines: string[] = [] - private bytes = 0 + private readonly pinned: string[] = [] + private readonly records: BoundedRecord[] = [] + private pinnedBytes = 0 + private recordBytes = 0 private truncated = false - constructor(private readonly maxBytes = CONNECTOR_TEXT_DOCUMENT_MAX_BYTES) {} + constructor( + private readonly maxBytes = CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, + private readonly keep: BoundedLinesKeep = 'first' + ) {} + + /** Lines that open the document and are never let go, such as its header; counted against the ceiling. */ + pin(...lines: string[]): void { + this.pinned.push(...lines) + this.pinnedBytes += byteSize(lines) + } + + /** Records currently kept. */ + get count(): number { + return this.records.length + } /** - * Appends the lines together when they fit; otherwise marks the document - * truncated, appends nothing, and returns false so the caller stops. + * Appends the lines as one record and returns whether it was kept. Keeping + * the first, a record that does not fit is refused, and so is every later + * one, so a caller can stop. Keeping the last, a record is refused only when + * it cannot fit on its own, and appending it lets the oldest records go + * until the rest fits, so a caller carries on. */ push(...lines: string[]): boolean { - if (this.truncated) return false - let size = 0 - for (const line of lines) size += Buffer.byteLength(line, 'utf8') + 1 - if (this.bytes + size > this.maxBytes) { + const bytes = byteSize(lines) + if (this.keep === 'first') { + if (this.truncated) return false + if (this.pinnedBytes + this.recordBytes + bytes > this.maxBytes) { + this.truncated = true + return false + } + this.records.push({ lines, bytes }) + this.recordBytes += bytes + return true + } + if (this.pinnedBytes + bytes > this.maxBytes) { this.truncated = true return false } - this.lines.push(...lines) - this.bytes += size + this.records.push({ lines, bytes }) + this.recordBytes += bytes + while (this.pinnedBytes + this.recordBytes > this.maxBytes) { + const oldest = this.records.shift() + if (!oldest) break + this.recordBytes -= oldest.bytes + this.truncated = true + } return true } - /** Joins the accepted lines, ending with the truncation notice when a push was refused. */ + /** Joins the kept lines, with the truncation notice where records were left out. */ join(): string { - return this.truncated ? [...this.lines, TRUNCATION_NOTICE].join('\n') : this.lines.join('\n') + const body = this.records.flatMap((record) => record.lines) + if (!this.truncated) return [...this.pinned, ...body].join('\n') + return this.keep === 'first' + ? [...this.pinned, ...body, TRAILING_TRUNCATION_NOTICE].join('\n') + : [...this.pinned, LEADING_TRUNCATION_NOTICE, ...body].join('\n') } } From 89bb7a006b87b1b7700f07145a63930a116a315e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 2 Sep 2026 15:52:07 -0700 Subject: [PATCH 3/3] fix(slack): fold the channel header into the content hash so a rename or topic edit re-indexes --- apps/sim/connectors/slack/slack.test.ts | 25 +++++++++++++++++++++++-- apps/sim/connectors/slack/slack.ts | 22 ++++++++++++++++------ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/apps/sim/connectors/slack/slack.test.ts b/apps/sim/connectors/slack/slack.test.ts index 2112b6e66ca..addfdf527e6 100644 --- a/apps/sim/connectors/slack/slack.test.ts +++ b/apps/sim/connectors/slack/slack.test.ts @@ -38,6 +38,8 @@ let listNextCursor = '' let history: Record[] = MESSAGES /** Whether `conversations.info` reports the channel as missing; per-test overridable. */ let channelMissing = false +/** The channel `conversations.info` returns; per-test overridable. */ +let infoChannel: Record = GENERAL beforeEach(() => { requestedUrls.length = 0 @@ -45,6 +47,7 @@ beforeEach(() => { listNextCursor = '' history = MESSAGES channelMissing = false + infoChannel = GENERAL fetchMock.mockReset() fetchMock.mockImplementation(async (input) => { const url = new URL(String(input)) @@ -61,7 +64,7 @@ beforeEach(() => { case '/api/conversations.info': return channelMissing ? jsonResponse({ ok: false, error: 'channel_not_found' }) - : jsonResponse({ ok: true, channel: GENERAL }) + : jsonResponse({ ok: true, channel: infoChannel }) case '/api/conversations.history': return jsonResponse({ ok: true, messages: history, response_metadata: {} }) case '/api/users.info': { @@ -163,7 +166,9 @@ describe('getDocument', () => { expect(doc).toMatchObject({ externalId: 'C0GENERAL', title: '#general', - contentHash: 'slack-v3:C0GENERAL:1700000000.000100:1700000200.000100:3:noedit:noreply:0', + contentHash: expect.stringMatching( + /^slack-v3:C0GENERAL:[0-9a-f]{16}:1700000000\.000100:1700000200\.000100:3:noedit:noreply:0$/ + ), metadata: expect.objectContaining({ channelName: 'general', messageCount: 2 }), }) expect(doc?.content).toBe( @@ -177,6 +182,22 @@ describe('getDocument', () => { ) }) + it('moves the hash when the header changes without any message changing', async () => { + const before = await slackConnector.getDocument('token', {}, 'C0GENERAL', {}) + + infoChannel = { ...GENERAL, name: 'general-renamed' } + const renamed = await slackConnector.getDocument('token', {}, 'C0GENERAL', {}) + expect(renamed?.contentHash).not.toBe(before?.contentHash) + + infoChannel = { ...GENERAL, topic: { value: 'A new topic' } } + const retopiced = await slackConnector.getDocument('token', {}, 'C0GENERAL', {}) + expect(retopiced?.contentHash).not.toBe(before?.contentHash) + + infoChannel = GENERAL + const again = await slackConnector.getDocument('token', {}, 'C0GENERAL', {}) + expect(again?.contentHash).toBe(before?.contentHash) + }) + it('keeps a channel with no messages as a live document', async () => { history = [] const doc = await slackConnector.getDocument('token', {}, 'C0GENERAL', {}) diff --git a/apps/sim/connectors/slack/slack.ts b/apps/sim/connectors/slack/slack.ts index 9954f7ac60b..dd819b10e42 100644 --- a/apps/sim/connectors/slack/slack.ts +++ b/apps/sim/connectors/slack/slack.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' @@ -602,13 +603,15 @@ async function buildSlackChannelDocument( maxMessages ) - const lines = new BoundedLines(CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, 'last') - lines.pin(`Channel: #${channel.name}`) + const header = [`Channel: #${channel.name}`] const topic = channel.topic?.value?.trim() - if (topic) lines.pin(`Topic: ${topic}`) + if (topic) header.push(`Topic: ${topic}`) const purpose = channel.purpose?.value?.trim() - if (purpose) lines.pin(`Purpose: ${purpose}`) - lines.pin('') + if (purpose) header.push(`Purpose: ${purpose}`) + + /** The newest messages survive when the window does not fit; the header always does. */ + const lines = new BoundedLines(CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, 'last') + lines.pin(...header, '') await appendMessages(accessToken, lines, messages, syncContext) const messageCount = lines.count @@ -633,13 +636,20 @@ async function buildSlackChannelDocument( * in catches deletes (count drops) but still cannot detect reply edits * without fetching `conversations.replies` for each parent. * + * The header is digested into the hash because it is part of the document: + * renaming a channel or editing its topic changes the indexed text without + * touching a single message, and the sync engine drops a refresh whose hash + * matches the stored one. A digest keeps the hash bounded and free of the + * delimiter collisions raw topic text would bring. + * * The `slack-v3` prefix forces a one-time re-index of channels indexed * before the document gained its header and size ceiling; `slack-v2` did the * same when attachment and Block Kit content started being extracted. * Per-message `ts` and the window are unchanged by either, so without the * bump the hash would match and the richer content would never be embedded. */ - const contentHash = `slack-v3:${channel.id}:${oldestTs ?? 'empty'}:${lastActivityTs ?? 'empty'}:${messages.length}:${maxEditTs || 'noedit'}:${maxReplyTs || 'noreply'}:${totalReplies}` + const headerDigest = createHash('sha256').update(header.join('\n')).digest('hex').slice(0, 16) + const contentHash = `slack-v3:${channel.id}:${headerDigest}:${oldestTs ?? 'empty'}:${lastActivityTs ?? 'empty'}:${messages.length}:${maxEditTs || 'noedit'}:${maxReplyTs || 'noreply'}:${totalReplies}` return { content: lines.join(), contentHash, messageCount, lastActivityTs } }