From ef0985745350658dfcfea71018b62232e4d87b86 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 18 Sep 2026 19:01:22 -0700 Subject: [PATCH] improvement(knowledge): skip redundant document resurrection writes --- .../listing-continuation.integration.ts | 88 +++++++++++++++++++ .../knowledge/connectors/sync-content-pass.ts | 1 + 2 files changed, 89 insertions(+) diff --git a/apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts b/apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts index c24e2476960..804fefa58ee 100644 --- a/apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts @@ -166,6 +166,94 @@ describe('durable source and member cycles in PostgreSQL', () => { }) } + it('restores rediscovered tombstones without rewriting already-live documents during resurrection', async () => { + const connectorId = generateId() + const runId = generateId() + const [connector] = await db + .insert(knowledgeConnector) + .values({ + id: connectorId, + knowledgeBaseId: ids.knowledgeBaseId, + connectorType: 'google_drive', + status: 'syncing', + syncLockToken: runId, + accessMode: 'admin', + sourceConfig: {}, + }) + .returning() + const deletedAt = new Date(Date.now() - 60_000) + const documents = ['live', 'tombstoned'].map(sourceDoc) + await db.insert(document).values( + documents.map((item) => ({ + id: generateId(), + knowledgeBaseId: ids.knowledgeBaseId, + connectorId, + externalId: item.externalId, + filename: item.title, + fileUrl: '', + storageKey: `kb/${item.externalId}.txt`, + fileSize: 10, + mimeType: item.mimeType, + contentHash: item.contentHash, + processingStatus: 'completed', + deletedAt: item.externalId === 'tombstoned' ? deletedAt : null, + })) + ) + let liveVersionBeforeResurrection: string | undefined + const syncResult = result() + const pass = await runConnectorContentPass({ + connectorId, + connector, + connectorConfig: { + ...CONNECTOR_REGISTRY.google_drive, + listDocuments: async () => ({ documents, hasMore: false }), + }, + sourceConfig: {}, + syncContext: {}, + kbOwner: { userId: ids.aliceId, workspaceId: ids.workspaceId }, + billingAttribution: billing, + result: syncResult, + lease: createContentSyncLease(connectorId, runId), + leaseKind: 'content', + runId, + fingerprint: listingFingerprint({ source: 'resurrection' }), + documentAccess: 'admin', + getAccessToken: async () => 'fixture', + hydration: { getDocument: async () => null }, + forceRehydrate: false, + deadlineAt: Date.now() + 60_000, + onPage: async () => { + const rows = await db + .select({ + externalId: document.externalId, + deletedAt: document.deletedAt, + version: sql`xmin::text`, + }) + .from(document) + .where(eq(document.connectorId, connectorId)) + liveVersionBeforeResurrection = rows.find((row) => row.externalId === 'live')!.version + expect(rows.find((row) => row.externalId === 'tombstoned')!.deletedAt).toEqual(deletedAt) + return undefined + }, + }) + const rows = await db + .select({ + externalId: document.externalId, + deletedAt: document.deletedAt, + version: sql`xmin::text`, + }) + .from(document) + .where(eq(document.connectorId, connectorId)) + expect(pass.complete).toBe(true) + expect(syncResult).toEqual({ ...result(), docsUnchanged: 2 }) + expect(rows).toHaveLength(2) + expect(rows.every((row) => row.deletedAt === null)).toBe(true) + expect(liveVersionBeforeResurrection).toBeDefined() + expect(rows.find((row) => row.externalId === 'live')!.version).toBe( + liveVersionBeforeResurrection + ) + }) + it('refreshes verified existing permissions while repairing changed bodies, without indexing discoveries or reconciling absence', async () => { const connectorId = generateId() const runId = generateId() diff --git a/apps/sim/lib/knowledge/connectors/sync-content-pass.ts b/apps/sim/lib/knowledge/connectors/sync-content-pass.ts index 68b482446d1..8532a82cb8b 100644 --- a/apps/sim/lib/knowledge/connectors/sync-content-pass.ts +++ b/apps/sim/lib/knowledge/connectors/sync-content-pass.ts @@ -290,6 +290,7 @@ export async function runConnectorContentPass(input: ContentPassInput) { and( eq(document.connectorId, input.connectorId), inArray(document.externalId, verified.slice(offset, offset + 500)), + isNotNull(document.deletedAt), isNotNull(document.contentHash), isNull(document.archivedAt) )