mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-22 04:26:23 +08:00
feat(server): impl doc gc (#15282)
#### PR Dependency Tree * **PR #15282** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added automated document cleanup to reconcile missing workspace docs, delete related stored data, and recover if the doc returns. * Added effect-based follow-up reconciliation for search indexing, Copilot embeddings, and comment attachment cleanup with explicit acknowledgements. * **Bug Fixes** * Deleted-document references now persist as dangling references rather than disappearing. * Improved document deletion flow to enforce permissions and ensure authorized deletions succeed. * **Tests** * Expanded coverage for cleanup recovery, indexing/embedding reconciliation, permissions, and reference semantics. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
@@ -28,7 +28,7 @@ import {
|
||||
WorkspaceModel,
|
||||
WorkspaceRole,
|
||||
} from '../../models';
|
||||
import type { LlmToolCallbackRequest } from '../../native';
|
||||
import { addDocToRootDoc, type LlmToolCallbackRequest } from '../../native';
|
||||
import { CopilotModule } from '../../plugins/copilot';
|
||||
import { CopilotContextService } from '../../plugins/copilot/context';
|
||||
import { CopilotContextResolver } from '../../plugins/copilot/context/resolver';
|
||||
@@ -260,6 +260,80 @@ test.after.always(async t => {
|
||||
await t.context.module?.close();
|
||||
});
|
||||
|
||||
test('document cleanup reconciles missing and restored copilot state before ack', async t => {
|
||||
const { db, jobs, models, module, workspace } = t.context;
|
||||
const queue = module.get(JobQueue);
|
||||
const deleteEmbedding = Sinon.spy(
|
||||
models.copilotContext,
|
||||
'purgeWorkspaceEmbedding'
|
||||
);
|
||||
const scheduleEmbedding = Sinon.stub(
|
||||
jobs,
|
||||
'addDocEmbeddingQueueFromEvent'
|
||||
).resolves();
|
||||
|
||||
for (const [docId, restored, cleanupVersion] of [
|
||||
['missing-doc', false, 'missing-version'],
|
||||
['restored-doc', true, 'restored-version'],
|
||||
] as const) {
|
||||
const ws = await workspace.create(userId);
|
||||
const root = addDocToRootDoc(Buffer.from([0, 0]), docId, docId);
|
||||
const missingRoot = addDocToRootDoc(
|
||||
Buffer.from([0, 0]),
|
||||
'live-doc',
|
||||
'Live'
|
||||
);
|
||||
await db.snapshot.create({
|
||||
data: {
|
||||
workspaceId: ws.id,
|
||||
id: ws.id,
|
||||
blob: restored ? root : missingRoot,
|
||||
state: Buffer.from([0, 0]),
|
||||
updatedAt: new Date(),
|
||||
createdAt: new Date(),
|
||||
},
|
||||
});
|
||||
if (restored) {
|
||||
await db.snapshot.create({
|
||||
data: {
|
||||
workspaceId: ws.id,
|
||||
id: docId,
|
||||
blob: addDocToRootDoc(Buffer.from([0, 0]), 'content', 'Content'),
|
||||
state: Buffer.from([0, 0]),
|
||||
updatedAt: new Date(),
|
||||
createdAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
await jobs.reconcileDocumentCleanup({
|
||||
workspaceId: ws.id,
|
||||
docId,
|
||||
cleanupVersion,
|
||||
});
|
||||
|
||||
if (restored) {
|
||||
t.true(scheduleEmbedding.calledOnceWith({ workspaceId: ws.id, docId }));
|
||||
t.false(deleteEmbedding.called);
|
||||
} else {
|
||||
t.true(deleteEmbedding.calledOnceWith(ws.id, docId));
|
||||
t.false(scheduleEmbedding.called);
|
||||
}
|
||||
const { payload } = await module.queue.waitFor(
|
||||
'backendRuntime.ackDocumentCleanupEffect'
|
||||
);
|
||||
t.deepEqual(payload, {
|
||||
workspaceId: ws.id,
|
||||
docId,
|
||||
cleanupVersion,
|
||||
effect: 'copilot',
|
||||
});
|
||||
deleteEmbedding.resetHistory();
|
||||
scheduleEmbedding.resetHistory();
|
||||
(queue.add as Sinon.SinonStub).resetHistory();
|
||||
}
|
||||
});
|
||||
|
||||
test('MCP credentials stay bound to their endpoint, workspace, and profile', async t => {
|
||||
const { db, mcpCredentials, mcpProvider, models, workspace } = t.context;
|
||||
const ws = await workspace.create(userId);
|
||||
|
||||
@@ -671,7 +671,7 @@ test('active users metric should dedupe multiple sockets for one user', async t
|
||||
test('workspace sync delete-doc should enforce doc permissions', async t => {
|
||||
const db = app.get(PrismaClient);
|
||||
const models = app.get(Models);
|
||||
const { user: owner } = await login(app);
|
||||
const { user: owner, cookieHeader: ownerCookieHeader } = await login(app);
|
||||
const { user: collaborator, cookieHeader } = await login(app);
|
||||
const workspace = await models.workspace.create(owner.id);
|
||||
const docId = 'private-doc';
|
||||
@@ -692,9 +692,10 @@ test('workspace sync delete-doc should enforce doc permissions', async t => {
|
||||
});
|
||||
|
||||
const socket = createClient(url, cookieHeader);
|
||||
const ownerSocket = createClient(url, ownerCookieHeader);
|
||||
|
||||
try {
|
||||
await waitForConnect(socket);
|
||||
await Promise.all([waitForConnect(socket), waitForConnect(ownerSocket)]);
|
||||
|
||||
const join = unwrapResponse(
|
||||
t,
|
||||
@@ -719,8 +720,37 @@ test('workspace sync delete-doc should enforce doc permissions', async t => {
|
||||
})
|
||||
);
|
||||
t.true(error.message.includes('Doc.Delete'));
|
||||
|
||||
const ownerJoin = unwrapResponse(
|
||||
t,
|
||||
await emitWithAck<{ clientId: string; success: boolean }>(
|
||||
ownerSocket,
|
||||
'space:join',
|
||||
{
|
||||
spaceType: 'workspace',
|
||||
spaceId: workspace.id,
|
||||
clientVersion: '0.26.0',
|
||||
}
|
||||
)
|
||||
);
|
||||
t.true(ownerJoin.success);
|
||||
unwrapResponse(
|
||||
t,
|
||||
await emitWithAck(ownerSocket, 'space:delete-doc', {
|
||||
spaceType: 'workspace',
|
||||
spaceId: workspace.id,
|
||||
docId,
|
||||
})
|
||||
);
|
||||
t.is(
|
||||
await db.snapshot.count({
|
||||
where: { workspaceId: workspace.id, id: docId },
|
||||
}),
|
||||
1
|
||||
);
|
||||
} finally {
|
||||
socket.disconnect();
|
||||
ownerSocket.disconnect();
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -137,8 +137,8 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
|
||||
}));
|
||||
}
|
||||
|
||||
async deleteDoc(workspaceId: string, docId: string) {
|
||||
await this.models.doc.delete(workspaceId, docId);
|
||||
async deleteDoc(_workspaceId: string, _docId: string) {
|
||||
return;
|
||||
}
|
||||
|
||||
async deleteSpace(workspaceId: string) {
|
||||
|
||||
@@ -258,6 +258,33 @@ export class StorageRuntimeProvider
|
||||
);
|
||||
}
|
||||
|
||||
async reconcileWorkspaceDocuments(workspaceId: string) {
|
||||
return await this.measured('reconcileWorkspaceDocuments', rt =>
|
||||
rt.reconcileWorkspaceDocuments(workspaceId)
|
||||
);
|
||||
}
|
||||
|
||||
async executeDocumentCleanupCandidates(
|
||||
workspaceId: string | null | undefined,
|
||||
gracePeriodDays: number,
|
||||
limit: number
|
||||
) {
|
||||
return await this.measured('executeDocumentCleanupCandidates', rt =>
|
||||
rt.executeDocumentCleanupCandidates(workspaceId, gracePeriodDays, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async ackDocumentCleanupEffect(
|
||||
workspaceId: string,
|
||||
docId: string,
|
||||
cleanupVersion: string,
|
||||
effect: 'search' | 'copilot'
|
||||
) {
|
||||
return await this.measured('ackDocumentCleanupEffect', rt =>
|
||||
rt.ackDocumentCleanupEffect(workspaceId, docId, cleanupVersion, effect)
|
||||
);
|
||||
}
|
||||
|
||||
async planUnreferencedWorkspaceBlobs(
|
||||
workspaceId: string,
|
||||
gracePeriodDays: number,
|
||||
|
||||
@@ -6,10 +6,13 @@ import { StorageBlobJob } from '../blob-job';
|
||||
interface Context {
|
||||
runtime: {
|
||||
health: Sinon.SinonStub;
|
||||
reconcileWorkspaceDocuments: Sinon.SinonStub;
|
||||
backfillMissingBlobMetadata: Sinon.SinonStub;
|
||||
rebuildWorkspaceDocBlobRefs: Sinon.SinonStub;
|
||||
planUnreferencedWorkspaceBlobs: Sinon.SinonStub;
|
||||
executeBlobCleanupCandidates: Sinon.SinonStub;
|
||||
executeDocumentCleanupCandidates: Sinon.SinonStub;
|
||||
ackDocumentCleanupEffect: Sinon.SinonStub;
|
||||
};
|
||||
event: {
|
||||
emitAsync: Sinon.SinonStub;
|
||||
@@ -35,10 +38,18 @@ test.beforeEach(t => {
|
||||
providerConfigured: true,
|
||||
provider: 'fs',
|
||||
}),
|
||||
reconcileWorkspaceDocuments: Sinon.stub().resolves({
|
||||
scannedDocs: 1,
|
||||
marked: 0,
|
||||
reset: 0,
|
||||
recovered: 0,
|
||||
}),
|
||||
backfillMissingBlobMetadata: Sinon.stub(),
|
||||
rebuildWorkspaceDocBlobRefs: Sinon.stub(),
|
||||
planUnreferencedWorkspaceBlobs: Sinon.stub(),
|
||||
executeBlobCleanupCandidates: Sinon.stub(),
|
||||
executeDocumentCleanupCandidates: Sinon.stub(),
|
||||
ackDocumentCleanupEffect: Sinon.stub(),
|
||||
};
|
||||
t.context.event = {
|
||||
emitAsync: Sinon.stub().resolves(undefined),
|
||||
@@ -47,7 +58,18 @@ test.beforeEach(t => {
|
||||
add: Sinon.stub().resolves(undefined),
|
||||
};
|
||||
t.context.db = {
|
||||
$queryRaw: Sinon.stub(),
|
||||
$queryRaw: Sinon.stub().resolves([
|
||||
{
|
||||
marked: 0n,
|
||||
failed: 0n,
|
||||
effectsPending: 0n,
|
||||
failedWorkspaceCheckpoints: 0n,
|
||||
rootFailureCheckpoints: 0n,
|
||||
staleProjectionBlocks: 0n,
|
||||
oldestFailedSeconds: null,
|
||||
oldestEffectsPendingSeconds: null,
|
||||
},
|
||||
]),
|
||||
workspace: {
|
||||
findMany: Sinon.stub(),
|
||||
},
|
||||
@@ -129,46 +151,6 @@ for (const scenario of objectStorageRequiredCases) {
|
||||
});
|
||||
}
|
||||
|
||||
test('doc blob refs sweep continues after one workspace fails', async t => {
|
||||
t.context.db.workspace.findMany.resolves([
|
||||
{ id: 'workspace-1', sid: 1 },
|
||||
{ id: 'workspace-2', sid: 2 },
|
||||
]);
|
||||
t.context.runtime.rebuildWorkspaceDocBlobRefs
|
||||
.onFirstCall()
|
||||
.rejects(new Error('bad root doc'))
|
||||
.onSecondCall()
|
||||
.resolves({
|
||||
scannedDocs: 1,
|
||||
parsedDocs: 1,
|
||||
refsWritten: 0,
|
||||
refsDeleted: 0,
|
||||
failedDocs: 0,
|
||||
nextCursor: null,
|
||||
});
|
||||
|
||||
await t.context.job.rebuildWorkspaceDocBlobRefsBySid({
|
||||
workspaceLimit: 2,
|
||||
docLimit: 100,
|
||||
});
|
||||
|
||||
t.is(t.context.runtime.rebuildWorkspaceDocBlobRefs.callCount, 2);
|
||||
t.deepEqual(t.context.runtime.rebuildWorkspaceDocBlobRefs.firstCall.args, [
|
||||
'workspace-1',
|
||||
100,
|
||||
]);
|
||||
t.deepEqual(t.context.runtime.rebuildWorkspaceDocBlobRefs.secondCall.args, [
|
||||
'workspace-2',
|
||||
100,
|
||||
]);
|
||||
t.true(
|
||||
t.context.queue.add.calledWith(
|
||||
'backendRuntime.rebuildWorkspaceDocBlobRefsBySid',
|
||||
{ lastSid: 2, workspaceLimit: 2, docLimit: 100 }
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
test('blob cleanup planning drains each workspace cursor before continuing', async t => {
|
||||
t.context.db.workspace.findMany.resolves([
|
||||
{ id: 'workspace-1', sid: 1 },
|
||||
@@ -229,6 +211,13 @@ test('blob cleanup planning drains each workspace cursor before continuing', asy
|
||||
test('daily blob cleanup execution uses a fixed job id', async t => {
|
||||
await t.context.job.dailyBlobCleanupExecution();
|
||||
|
||||
t.true(
|
||||
t.context.queue.add.calledWith(
|
||||
'backendRuntime.executeDocumentCleanupCandidates',
|
||||
{},
|
||||
{ jobId: 'daily-backend-runtime-document-cleanup-execution' }
|
||||
)
|
||||
);
|
||||
t.true(
|
||||
t.context.queue.add.calledWith(
|
||||
'backendRuntime.executeBlobCleanupCandidatesByMarkedRuns',
|
||||
@@ -238,6 +227,132 @@ test('daily blob cleanup execution uses a fixed job id', async t => {
|
||||
);
|
||||
});
|
||||
|
||||
test('daily storage reconciliation uses a fixed job id', async t => {
|
||||
await t.context.job.dailyStorageReconciliation();
|
||||
|
||||
t.true(
|
||||
t.context.queue.add.calledWith(
|
||||
'backendRuntime.reconcileWorkspaceStorageBySid',
|
||||
{},
|
||||
{ jobId: 'daily-backend-runtime-storage-reconciliation' }
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
test('storage reconciliation orders document retention before blob cleanup', async t => {
|
||||
t.context.db.workspace.findMany.resolves([{ id: 'workspace-1', sid: 1 }]);
|
||||
t.context.runtime.rebuildWorkspaceDocBlobRefs.resolves({
|
||||
scannedDocs: 1,
|
||||
parsedDocs: 1,
|
||||
refsWritten: 1,
|
||||
refsDeleted: 0,
|
||||
failedDocs: 0,
|
||||
nextCursor: null,
|
||||
});
|
||||
t.context.runtime.planUnreferencedWorkspaceBlobs.resolves({
|
||||
runId: 'run-1',
|
||||
scannedBlobs: 1,
|
||||
candidatesMarked: 0,
|
||||
nextCursor: null,
|
||||
});
|
||||
|
||||
await t.context.job.reconcileWorkspaceStorageBySid({ workspaceLimit: 10 });
|
||||
|
||||
Sinon.assert.callOrder(
|
||||
t.context.runtime.reconcileWorkspaceDocuments,
|
||||
t.context.runtime.rebuildWorkspaceDocBlobRefs,
|
||||
t.context.runtime.planUnreferencedWorkspaceBlobs
|
||||
);
|
||||
t.pass();
|
||||
});
|
||||
|
||||
test('storage reconciliation still refreshes document retention without object storage', async t => {
|
||||
t.context.runtime.health.resolves({
|
||||
databaseConnected: true,
|
||||
providerConfigured: true,
|
||||
provider: undefined,
|
||||
});
|
||||
t.context.db.workspace.findMany.resolves([{ id: 'workspace-1', sid: 1 }]);
|
||||
t.context.runtime.rebuildWorkspaceDocBlobRefs.resolves({
|
||||
scannedDocs: 1,
|
||||
parsedDocs: 1,
|
||||
refsWritten: 0,
|
||||
refsDeleted: 0,
|
||||
failedDocs: 0,
|
||||
nextCursor: null,
|
||||
});
|
||||
|
||||
await t.context.job.reconcileWorkspaceStorageBySid({});
|
||||
|
||||
t.true(t.context.runtime.reconcileWorkspaceDocuments.calledOnce);
|
||||
t.true(t.context.runtime.rebuildWorkspaceDocBlobRefs.calledOnce);
|
||||
t.false(t.context.runtime.planUnreferencedWorkspaceBlobs.called);
|
||||
});
|
||||
|
||||
test('document cleanup dispatches independent stable search and copilot effects', async t => {
|
||||
t.context.runtime.executeDocumentCleanupCandidates.resolves({
|
||||
scannedCandidates: 1,
|
||||
serializationRetries: 0,
|
||||
executed: 1,
|
||||
recovered: 0,
|
||||
reset: 0,
|
||||
failed: 0,
|
||||
deletedRows: 3,
|
||||
effects: [
|
||||
{
|
||||
workspaceId: 'workspace-1',
|
||||
docId: 'doc-1',
|
||||
cleanupVersion: 'version-1',
|
||||
commentObjectsDone: true,
|
||||
searchDone: false,
|
||||
copilotDone: false,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await t.context.job.executeDocumentCleanupCandidates({});
|
||||
|
||||
t.true(
|
||||
t.context.queue.add.calledWith(
|
||||
'indexer.reconcileDocumentCleanup',
|
||||
Sinon.match({ docId: 'doc-1' }),
|
||||
{
|
||||
jobId: 'document-cleanup:search:workspace-1:doc-1:version-1',
|
||||
}
|
||||
)
|
||||
);
|
||||
t.true(
|
||||
t.context.queue.add.calledWith(
|
||||
'copilot.embedding.reconcileDocumentCleanup',
|
||||
Sinon.match({ docId: 'doc-1' }),
|
||||
{
|
||||
jobId: 'document-cleanup:copilot:workspace-1:doc-1:version-1',
|
||||
}
|
||||
)
|
||||
);
|
||||
t.true(
|
||||
t.context.event.emitAsync.calledWith('workspace.blobs.updated', {
|
||||
workspaceId: 'workspace-1',
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
test('document cleanup effect ack delegates to storage runtime', async t => {
|
||||
await t.context.job.ackDocumentCleanupEffect({
|
||||
workspaceId: 'workspace-1',
|
||||
docId: 'doc-1',
|
||||
cleanupVersion: 'version-1',
|
||||
effect: 'search',
|
||||
});
|
||||
|
||||
t.deepEqual(t.context.runtime.ackDocumentCleanupEffect.firstCall.args, [
|
||||
'workspace-1',
|
||||
'doc-1',
|
||||
'version-1',
|
||||
'search',
|
||||
]);
|
||||
});
|
||||
|
||||
test('blob cleanup execution sweep drains marked runs and continues by page', async t => {
|
||||
t.context.db.$queryRaw
|
||||
.onFirstCall()
|
||||
|
||||
@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
import { EventBus, JobQueue, OnJob } from '../../base';
|
||||
import { EventBus, JobQueue, metrics, OnJob } from '../../base';
|
||||
import { StorageRuntimeProvider } from '../storage-runtime';
|
||||
|
||||
// Queue keys are persisted API; keep the legacy backendRuntime.* names while
|
||||
@@ -18,15 +18,22 @@ declare global {
|
||||
workspaceLimit?: number;
|
||||
objectLimit?: number;
|
||||
};
|
||||
'backendRuntime.rebuildWorkspaceDocBlobRefs': {
|
||||
workspaceId: string;
|
||||
limit?: number;
|
||||
};
|
||||
'backendRuntime.rebuildWorkspaceDocBlobRefsBySid': {
|
||||
'backendRuntime.reconcileWorkspaceStorageBySid': {
|
||||
lastSid?: number;
|
||||
workspaceLimit?: number;
|
||||
docLimit?: number;
|
||||
};
|
||||
'backendRuntime.executeDocumentCleanupCandidates': {
|
||||
workspaceId?: string;
|
||||
gracePeriodDays?: number;
|
||||
limit?: number;
|
||||
};
|
||||
'backendRuntime.ackDocumentCleanupEffect': {
|
||||
workspaceId: string;
|
||||
docId: string;
|
||||
cleanupVersion: string;
|
||||
effect: 'search' | 'copilot';
|
||||
};
|
||||
'backendRuntime.planUnreferencedWorkspaceBlobs': {
|
||||
workspaceId: string;
|
||||
gracePeriodDays?: number;
|
||||
@@ -89,25 +96,6 @@ export class StorageBlobJob {
|
||||
});
|
||||
}
|
||||
|
||||
async enqueueRebuildWorkspaceDocBlobRefs(workspaceId: string, limit = 1000) {
|
||||
await this.queue.add('backendRuntime.rebuildWorkspaceDocBlobRefs', {
|
||||
workspaceId,
|
||||
limit,
|
||||
});
|
||||
}
|
||||
|
||||
async enqueueRebuildWorkspaceDocBlobRefsBySid(
|
||||
lastSid = 0,
|
||||
workspaceLimit = 100,
|
||||
docLimit = 1000
|
||||
) {
|
||||
await this.queue.add('backendRuntime.rebuildWorkspaceDocBlobRefsBySid', {
|
||||
lastSid,
|
||||
workspaceLimit,
|
||||
docLimit,
|
||||
});
|
||||
}
|
||||
|
||||
@OnJob('backendRuntime.backfillMissingBlobMetadataBySid')
|
||||
async backfillMissingBlobMetadataBySid({
|
||||
lastSid = 0,
|
||||
@@ -211,25 +199,21 @@ export class StorageBlobJob {
|
||||
}
|
||||
|
||||
@Cron(CronExpression.EVERY_DAY_AT_2AM)
|
||||
async dailyDocBlobRefsRebuild() {
|
||||
async dailyStorageReconciliation() {
|
||||
await this.queue.add(
|
||||
'backendRuntime.rebuildWorkspaceDocBlobRefsBySid',
|
||||
'backendRuntime.reconcileWorkspaceStorageBySid',
|
||||
{},
|
||||
{ jobId: 'daily-backend-runtime-doc-blob-refs-rebuild' }
|
||||
);
|
||||
}
|
||||
|
||||
@Cron(CronExpression.EVERY_DAY_AT_3AM)
|
||||
async dailyBlobCleanupPlanning() {
|
||||
await this.queue.add(
|
||||
'backendRuntime.planUnreferencedWorkspaceBlobsBySid',
|
||||
{},
|
||||
{ jobId: 'daily-backend-runtime-blob-cleanup-planning' }
|
||||
{ jobId: 'daily-backend-runtime-storage-reconciliation' }
|
||||
);
|
||||
}
|
||||
|
||||
@Cron(CronExpression.EVERY_DAY_AT_4AM)
|
||||
async dailyBlobCleanupExecution() {
|
||||
await this.queue.add(
|
||||
'backendRuntime.executeDocumentCleanupCandidates',
|
||||
{},
|
||||
{ jobId: 'daily-backend-runtime-document-cleanup-execution' }
|
||||
);
|
||||
await this.queue.add(
|
||||
'backendRuntime.executeBlobCleanupCandidatesByMarkedRuns',
|
||||
{},
|
||||
@@ -249,44 +233,39 @@ export class StorageBlobJob {
|
||||
await this.drainBlobMetadataBackfill(workspaceId, limit);
|
||||
}
|
||||
|
||||
@OnJob('backendRuntime.rebuildWorkspaceDocBlobRefs')
|
||||
async rebuildWorkspaceDocBlobRefs({
|
||||
workspaceId,
|
||||
limit = 1000,
|
||||
}: Jobs['backendRuntime.rebuildWorkspaceDocBlobRefs']) {
|
||||
await this.drainWorkspaceDocBlobRefs(workspaceId, limit);
|
||||
}
|
||||
|
||||
@OnJob('backendRuntime.rebuildWorkspaceDocBlobRefsBySid')
|
||||
async rebuildWorkspaceDocBlobRefsBySid({
|
||||
@OnJob('backendRuntime.reconcileWorkspaceStorageBySid')
|
||||
async reconcileWorkspaceStorageBySid({
|
||||
lastSid = 0,
|
||||
workspaceLimit = 100,
|
||||
docLimit = 1000,
|
||||
}: Jobs['backendRuntime.rebuildWorkspaceDocBlobRefsBySid']) {
|
||||
}: Jobs['backendRuntime.reconcileWorkspaceStorageBySid']) {
|
||||
const workspaces = await this.db.workspace.findMany({
|
||||
where: {
|
||||
sid: {
|
||||
gt: lastSid,
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
sid: 'asc',
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
sid: true,
|
||||
},
|
||||
where: { sid: { gt: lastSid } },
|
||||
orderBy: { sid: 'asc' },
|
||||
select: { id: true, sid: true },
|
||||
take: workspaceLimit,
|
||||
});
|
||||
const objectStorageConfigured = await this.hasObjectStorage(
|
||||
'storage reconciliation blob cleanup planning'
|
||||
);
|
||||
|
||||
for (const workspace of workspaces) {
|
||||
try {
|
||||
await this.rt.reconcileWorkspaceDocuments(workspace.id);
|
||||
await this.drainWorkspaceDocBlobRefs(workspace.id, docLimit, {
|
||||
sid: workspace.sid,
|
||||
});
|
||||
if (objectStorageConfigured) {
|
||||
await this.drainBlobCleanupPlanning(workspace.id, 30, 1000, {
|
||||
sid: workspace.sid,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
metrics.storage
|
||||
.counter('document_cleanup_workspace_failure_total')
|
||||
.add(1);
|
||||
this.logger.error(
|
||||
`doc blob refs rebuild failed workspace=${workspace.id} sid=${workspace.sid}`,
|
||||
`storage reconciliation failed workspace=${workspace.id} sid=${workspace.sid}`,
|
||||
err
|
||||
);
|
||||
}
|
||||
@@ -294,14 +273,71 @@ export class StorageBlobJob {
|
||||
|
||||
const nextSid = workspaces.at(-1)?.sid;
|
||||
if (nextSid !== undefined && workspaces.length === workspaceLimit) {
|
||||
await this.enqueueRebuildWorkspaceDocBlobRefsBySid(
|
||||
nextSid,
|
||||
await this.queue.add('backendRuntime.reconcileWorkspaceStorageBySid', {
|
||||
lastSid: nextSid,
|
||||
workspaceLimit,
|
||||
docLimit
|
||||
);
|
||||
docLimit,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@OnJob('backendRuntime.executeDocumentCleanupCandidates')
|
||||
async executeDocumentCleanupCandidates({
|
||||
workspaceId,
|
||||
gracePeriodDays = 30,
|
||||
limit = 100,
|
||||
}: Jobs['backendRuntime.executeDocumentCleanupCandidates']) {
|
||||
const result = await this.rt.executeDocumentCleanupCandidates(
|
||||
workspaceId,
|
||||
gracePeriodDays,
|
||||
limit
|
||||
);
|
||||
metrics.storage
|
||||
.counter('document_cleanup_serialization_retry_total')
|
||||
.add(result.serializationRetries);
|
||||
metrics.storage
|
||||
.counter('document_cleanup_execute_failure_total')
|
||||
.add(result.failed);
|
||||
for (const effect of result.effects) {
|
||||
if (!effect.searchDone) {
|
||||
await this.queue.add('indexer.reconcileDocumentCleanup', effect, {
|
||||
jobId: `document-cleanup:search:${effect.workspaceId}:${effect.docId}:${effect.cleanupVersion}`,
|
||||
});
|
||||
}
|
||||
if (!effect.copilotDone) {
|
||||
await this.queue.add(
|
||||
'copilot.embedding.reconcileDocumentCleanup',
|
||||
effect,
|
||||
{
|
||||
jobId: `document-cleanup:copilot:${effect.workspaceId}:${effect.docId}:${effect.cleanupVersion}`,
|
||||
}
|
||||
);
|
||||
}
|
||||
if (effect.commentObjectsDone) {
|
||||
await this.event.emitAsync('workspace.blobs.updated', {
|
||||
workspaceId: effect.workspaceId,
|
||||
});
|
||||
}
|
||||
}
|
||||
await this.recordDocumentCleanupHealth();
|
||||
return result;
|
||||
}
|
||||
|
||||
@OnJob('backendRuntime.ackDocumentCleanupEffect')
|
||||
async ackDocumentCleanupEffect({
|
||||
workspaceId,
|
||||
docId,
|
||||
cleanupVersion,
|
||||
effect,
|
||||
}: Jobs['backendRuntime.ackDocumentCleanupEffect']) {
|
||||
await this.rt.ackDocumentCleanupEffect(
|
||||
workspaceId,
|
||||
docId,
|
||||
cleanupVersion,
|
||||
effect
|
||||
);
|
||||
}
|
||||
|
||||
@OnJob('backendRuntime.planUnreferencedWorkspaceBlobs')
|
||||
async planUnreferencedWorkspaceBlobs({
|
||||
workspaceId,
|
||||
@@ -545,6 +581,82 @@ export class StorageBlobJob {
|
||||
return rows.map(row => row.runId);
|
||||
}
|
||||
|
||||
private async recordDocumentCleanupHealth() {
|
||||
const [health] = await this.db.$queryRaw<
|
||||
{
|
||||
marked: bigint;
|
||||
failed: bigint;
|
||||
effectsPending: bigint;
|
||||
failedWorkspaceCheckpoints: bigint;
|
||||
rootFailureCheckpoints: bigint;
|
||||
staleProjectionBlocks: bigint;
|
||||
oldestFailedSeconds: number | null;
|
||||
oldestEffectsPendingSeconds: number | null;
|
||||
}[]
|
||||
>`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE status = 'marked') AS marked,
|
||||
COUNT(*) FILTER (WHERE status = 'failed') AS failed,
|
||||
COUNT(*) FILTER (WHERE status = 'effects_pending') AS "effectsPending",
|
||||
EXTRACT(EPOCH FROM CURRENT_TIMESTAMP -
|
||||
MIN(updated_at) FILTER (WHERE status = 'failed'))::double precision AS "oldestFailedSeconds",
|
||||
EXTRACT(EPOCH FROM CURRENT_TIMESTAMP -
|
||||
MIN(updated_at) FILTER (WHERE status = 'effects_pending'))::double precision AS "oldestEffectsPendingSeconds",
|
||||
(SELECT COUNT(*) FROM storage_reconciliation_checkpoints
|
||||
WHERE kind = 'document_cleanup' AND status = 'failed') AS "failedWorkspaceCheckpoints",
|
||||
(SELECT COUNT(*) FROM storage_reconciliation_checkpoints
|
||||
WHERE kind = 'document_cleanup' AND status = 'failed'
|
||||
AND metadata->>'failureKind' = 'root') AS "rootFailureCheckpoints",
|
||||
(SELECT COUNT(*) FROM storage_reconciliation_runs
|
||||
WHERE kind = 'blob_cleanup_plan'
|
||||
AND started_at >= CURRENT_TIMESTAMP - INTERVAL '1 day'
|
||||
AND jsonb_array_length(COALESCE(metadata->'staleOrFailedProjectionWorkspaces', '[]')) > 0
|
||||
) AS "staleProjectionBlocks"
|
||||
FROM document_cleanup_candidates
|
||||
`;
|
||||
if (!health) {
|
||||
return;
|
||||
}
|
||||
for (const [name, value] of [
|
||||
['document_cleanup_marked', health.marked],
|
||||
['document_cleanup_failed', health.failed],
|
||||
['document_cleanup_effects_pending', health.effectsPending],
|
||||
[
|
||||
'document_cleanup_failed_workspace_checkpoints',
|
||||
health.failedWorkspaceCheckpoints,
|
||||
],
|
||||
[
|
||||
'document_cleanup_root_failure_checkpoints',
|
||||
health.rootFailureCheckpoints,
|
||||
],
|
||||
[
|
||||
'document_cleanup_stale_projection_blocks',
|
||||
health.staleProjectionBlocks,
|
||||
],
|
||||
] as const) {
|
||||
metrics.storage.gauge(name).record(Number(value));
|
||||
}
|
||||
metrics.storage
|
||||
.gauge('document_cleanup_oldest_failed_seconds')
|
||||
.record(health.oldestFailedSeconds ?? 0);
|
||||
metrics.storage
|
||||
.gauge('document_cleanup_oldest_effects_pending_seconds')
|
||||
.record(health.oldestEffectsPendingSeconds ?? 0);
|
||||
|
||||
if (
|
||||
health.failedWorkspaceCheckpoints > 0n ||
|
||||
health.rootFailureCheckpoints > 0n ||
|
||||
health.staleProjectionBlocks > 0n ||
|
||||
(health.oldestFailedSeconds ?? 0) >= 86_400 ||
|
||||
(health.oldestEffectsPendingSeconds ?? 0) >= 86_400 ||
|
||||
health.marked >= 10_000n
|
||||
) {
|
||||
this.logger.warn(
|
||||
`document cleanup health marked=${health.marked} failed=${health.failed} effectsPending=${health.effectsPending} failedWorkspaceCheckpoints=${health.failedWorkspaceCheckpoints} rootFailureCheckpoints=${health.rootFailureCheckpoints} staleProjectionBlocks=${health.staleProjectionBlocks} oldestFailedSeconds=${health.oldestFailedSeconds ?? 0} oldestEffectsPendingSeconds=${health.oldestEffectsPendingSeconds ?? 0}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async hasMarkedBlobCleanupCandidates() {
|
||||
const rows = await this.db.$queryRaw<{ exists: boolean }[]>`
|
||||
SELECT EXISTS(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import test from 'ava';
|
||||
import { omit } from 'lodash-es';
|
||||
import * as Y from 'yjs';
|
||||
|
||||
import { createModule } from '../../../__tests__/create-module';
|
||||
import { Mockers } from '../../../__tests__/mocks';
|
||||
@@ -38,6 +39,86 @@ test('can read all doc ids from workspace snapshot', async t => {
|
||||
t.snapshot(docIds);
|
||||
});
|
||||
|
||||
test('merged root updates retain trash and exclude permanently removed docs', async t => {
|
||||
const rootDoc = await models.doc.get(workspace.id, workspace.id);
|
||||
t.truthy(rootDoc);
|
||||
|
||||
const root = new Y.Doc();
|
||||
Y.applyUpdate(root, rootDoc!.blob);
|
||||
const pending = new Y.Doc();
|
||||
Y.applyUpdate(pending, Y.encodeStateAsUpdate(root));
|
||||
const trashMeta = new Y.Map<unknown>();
|
||||
trashMeta.set('id', 'trash-doc');
|
||||
trashMeta.set('title', 'Trash');
|
||||
trashMeta.set('trash', true);
|
||||
(pending.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>).push([
|
||||
trashMeta,
|
||||
]);
|
||||
const pendingUpdate = Y.encodeStateAsUpdate(
|
||||
pending,
|
||||
Y.encodeStateVector(root)
|
||||
);
|
||||
const merged = Y.mergeUpdates([rootDoc!.blob, pendingUpdate]);
|
||||
|
||||
t.false(readAllDocIdsFromWorkspaceSnapshot(merged).includes('trash-doc'));
|
||||
t.true(
|
||||
readAllDocIdsFromWorkspaceSnapshot(merged, true).includes('trash-doc')
|
||||
);
|
||||
|
||||
const removed = new Y.Doc();
|
||||
Y.applyUpdate(removed, merged);
|
||||
const pages = removed.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>;
|
||||
const target = pages
|
||||
.toArray()
|
||||
.findIndex(meta => meta instanceof Y.Map && meta.get('id') === 'trash-doc');
|
||||
pages.delete(target, 1);
|
||||
t.false(
|
||||
readAllDocIdsFromWorkspaceSnapshot(
|
||||
Y.encodeStateAsUpdate(removed),
|
||||
true
|
||||
).includes('trash-doc')
|
||||
);
|
||||
});
|
||||
|
||||
test('nested concurrent meta edits do not restore a deleted entry', t => {
|
||||
const initial = new Y.Doc();
|
||||
const pages = new Y.Array<Y.Map<unknown>>();
|
||||
const meta = new Y.Map<unknown>();
|
||||
meta.set('id', 'doc-1');
|
||||
meta.set('title', 'Initial');
|
||||
pages.push([meta]);
|
||||
initial.getMap('meta').set('pages', pages);
|
||||
const state = Y.encodeStateAsUpdate(initial);
|
||||
|
||||
const deletingClient = new Y.Doc();
|
||||
const editingClient = new Y.Doc();
|
||||
Y.applyUpdate(deletingClient, state);
|
||||
Y.applyUpdate(editingClient, state);
|
||||
(
|
||||
deletingClient.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>
|
||||
).delete(0, 1);
|
||||
(editingClient.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>)
|
||||
.get(0)
|
||||
.set('title', 'Offline edit');
|
||||
|
||||
const merged = new Y.Doc();
|
||||
Y.applyUpdate(merged, Y.encodeStateAsUpdate(deletingClient));
|
||||
Y.applyUpdate(merged, Y.encodeStateAsUpdate(editingClient));
|
||||
t.is(
|
||||
(merged.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>).length,
|
||||
0
|
||||
);
|
||||
|
||||
(editingClient.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>).push([
|
||||
meta.clone(),
|
||||
]);
|
||||
Y.applyUpdate(merged, Y.encodeStateAsUpdate(editingClient));
|
||||
t.is(
|
||||
(merged.getMap('meta').get('pages') as Y.Array<Y.Map<unknown>>).length,
|
||||
1
|
||||
);
|
||||
});
|
||||
|
||||
test('can read all blocks from doc snapshot', async t => {
|
||||
const rootDoc = await models.doc.get(workspace.id, workspace.id);
|
||||
t.truthy(rootDoc);
|
||||
|
||||
@@ -45,8 +45,11 @@ export function parsePageDoc(
|
||||
);
|
||||
}
|
||||
|
||||
export function readAllDocIdsFromWorkspaceSnapshot(snapshot: Uint8Array) {
|
||||
return readAllDocIdsFromRootDoc(Buffer.from(snapshot), false);
|
||||
export function readAllDocIdsFromWorkspaceSnapshot(
|
||||
snapshot: Uint8Array,
|
||||
includeTrash = false
|
||||
) {
|
||||
return readAllDocIdsFromRootDoc(Buffer.from(snapshot), includeTrash);
|
||||
}
|
||||
|
||||
function safeParseJson<T>(str: string): T | undefined {
|
||||
|
||||
@@ -337,10 +337,14 @@ export class CopilotContextModel extends BaseModel {
|
||||
}
|
||||
|
||||
async deleteWorkspaceEmbedding(workspaceId: string, docId: string) {
|
||||
await this.purgeWorkspaceEmbedding(workspaceId, docId);
|
||||
await this.fulfillEmptyEmbedding(workspaceId, docId);
|
||||
}
|
||||
|
||||
async purgeWorkspaceEmbedding(workspaceId: string, docId: string) {
|
||||
await this.db.aiWorkspaceEmbedding.deleteMany({
|
||||
where: { workspaceId, docId },
|
||||
});
|
||||
await this.fulfillEmptyEmbedding(workspaceId, docId);
|
||||
}
|
||||
|
||||
async matchWorkspaceEmbedding(
|
||||
|
||||
@@ -205,20 +205,49 @@ export class CopilotEmbeddingJob {
|
||||
);
|
||||
}
|
||||
|
||||
@OnJob('copilot.embedding.deleteDoc')
|
||||
async deleteDocEmbeddingQueueFromEvent(
|
||||
doc: Jobs['copilot.embedding.deleteDoc']
|
||||
) {
|
||||
private async deleteDocEmbedding(doc: {
|
||||
workspaceId: string;
|
||||
docId: string;
|
||||
}) {
|
||||
await this.queue.remove(
|
||||
`workspace:embedding:${doc.workspaceId}:${doc.docId}`,
|
||||
'copilot.embedding.docs'
|
||||
);
|
||||
await this.models.copilotContext.deleteWorkspaceEmbedding(
|
||||
await this.models.copilotContext.purgeWorkspaceEmbedding(
|
||||
doc.workspaceId,
|
||||
doc.docId
|
||||
);
|
||||
}
|
||||
|
||||
@OnJob('copilot.embedding.reconcileDocumentCleanup')
|
||||
async reconcileDocumentCleanup({
|
||||
workspaceId,
|
||||
docId,
|
||||
cleanupVersion,
|
||||
}: Jobs['copilot.embedding.reconcileDocumentCleanup']) {
|
||||
const root = await this.doc.getDoc(workspaceId, workspaceId);
|
||||
if (!root) {
|
||||
throw new Error(`workspace root ${workspaceId} not found`);
|
||||
}
|
||||
const live = readAllDocIdsFromWorkspaceSnapshot(root.bin, true).includes(
|
||||
docId
|
||||
);
|
||||
if (live) {
|
||||
if (!(await this.doc.getDoc(workspaceId, docId))) {
|
||||
throw new Error(`restored document ${workspaceId}/${docId} not found`);
|
||||
}
|
||||
await this.addDocEmbeddingQueueFromEvent({ workspaceId, docId });
|
||||
} else {
|
||||
await this.deleteDocEmbedding({ workspaceId, docId });
|
||||
}
|
||||
await this.queue.add('backendRuntime.ackDocumentCleanupEffect', {
|
||||
workspaceId,
|
||||
docId,
|
||||
cleanupVersion,
|
||||
effect: 'copilot',
|
||||
});
|
||||
}
|
||||
|
||||
private async readCopilotBlob(
|
||||
userId: string,
|
||||
workspaceId: string,
|
||||
|
||||
@@ -68,9 +68,10 @@ declare global {
|
||||
docId: string;
|
||||
};
|
||||
|
||||
'copilot.embedding.deleteDoc': {
|
||||
'copilot.embedding.reconcileDocumentCleanup': {
|
||||
workspaceId: string;
|
||||
docId: string;
|
||||
cleanupVersion: string;
|
||||
};
|
||||
|
||||
'copilot.embedding.files': {
|
||||
|
||||
@@ -39,10 +39,6 @@ declare global {
|
||||
'copilot.session.generateTitle': {
|
||||
sessionId: string;
|
||||
};
|
||||
'copilot.session.deleteDoc': {
|
||||
workspaceId: string;
|
||||
docId: string;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -512,23 +508,6 @@ export class ChatSessionService {
|
||||
return null;
|
||||
}
|
||||
|
||||
@OnJob('copilot.session.deleteDoc')
|
||||
async deleteDocSessions(doc: Jobs['copilot.session.deleteDoc']) {
|
||||
const sessionIds = await this.models.copilotSession
|
||||
.list({
|
||||
userId: undefined,
|
||||
workspaceId: doc.workspaceId,
|
||||
docId: doc.docId,
|
||||
})
|
||||
.then(s => s.map(s => [s.userId, s.id]));
|
||||
for (const [userId, sessionId] of sessionIds) {
|
||||
await this.models.copilotSession.update(
|
||||
{ userId, sessionId, docId: null },
|
||||
true
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@OnJob('copilot.session.generateTitle')
|
||||
async generateSessionTitle(job: Jobs['copilot.session.generateTitle']) {
|
||||
const { sessionId } = job;
|
||||
|
||||
@@ -6,14 +6,17 @@ import Sinon from 'sinon';
|
||||
|
||||
import { createModule } from '../../../__tests__/create-module';
|
||||
import { Mockers } from '../../../__tests__/mocks';
|
||||
import { JOB_SIGNAL } from '../../../base';
|
||||
import { Config, JOB_SIGNAL } from '../../../base';
|
||||
import { ConfigModule } from '../../../base/config';
|
||||
import { ServerConfigModule } from '../../../core/config';
|
||||
import { DocReader } from '../../../core/doc';
|
||||
import { Models } from '../../../models';
|
||||
import { addDocToRootDoc } from '../../../native';
|
||||
import { SearchProviderFactory } from '../factory';
|
||||
import { IndexerModule, IndexerService } from '../index';
|
||||
import { IndexerJob } from '../job';
|
||||
import { ManticoresearchProvider } from '../providers';
|
||||
import { blockSQL, docSQL, SearchTable } from '../tables';
|
||||
|
||||
const module = await createModule({
|
||||
imports: [
|
||||
@@ -32,6 +35,8 @@ const indexerJob = module.get(IndexerJob);
|
||||
const searchProviderFactory = module.get(SearchProviderFactory);
|
||||
const manticoresearch = module.get(ManticoresearchProvider);
|
||||
const models = module.get(Models);
|
||||
const docReader = module.get(DocReader);
|
||||
const config = module.get(Config);
|
||||
|
||||
const user = await module.create(Mockers.User);
|
||||
const workspace = await module.create(Mockers.Workspace, {
|
||||
@@ -39,6 +44,11 @@ const workspace = await module.create(Mockers.Workspace, {
|
||||
owner: user,
|
||||
});
|
||||
|
||||
test.before(async () => {
|
||||
await manticoresearch.recreateTable(SearchTable.block, blockSQL);
|
||||
await manticoresearch.recreateTable(SearchTable.doc, docSQL);
|
||||
});
|
||||
|
||||
test.after.always(async () => {
|
||||
await module.close();
|
||||
});
|
||||
@@ -105,7 +115,7 @@ test('should not sync existing doc', async t => {
|
||||
t.is(module.queue.count('indexer.indexDoc'), count);
|
||||
});
|
||||
|
||||
test('should delete doc from indexer when docId is not in workspace', async t => {
|
||||
test('should delete dangling indexed docs absent from the root live set', async t => {
|
||||
const count = module.queue.count('indexer.deleteDoc');
|
||||
mock.method(indexerService, 'listDocIds', async () => {
|
||||
return ['mock-doc-id1', 'mock-doc-id2'];
|
||||
@@ -121,6 +131,104 @@ test('should delete doc from indexer when docId is not in workspace', async t =>
|
||||
t.is(module.queue.count('indexer.deleteDoc'), count + 2);
|
||||
});
|
||||
|
||||
test('document cleanup reconcile deletes missing search state before ack', async t => {
|
||||
const deleteSpy = Sinon.spy(indexerService, 'deleteDoc');
|
||||
const indexSpy = Sinon.spy(indexerService, 'indexDoc');
|
||||
const cleanupWorkspace = await module.create(Mockers.Workspace, {
|
||||
owner: user,
|
||||
});
|
||||
await module.create(Mockers.DocSnapshot, {
|
||||
workspaceId: cleanupWorkspace.id,
|
||||
docId: cleanupWorkspace.id,
|
||||
user,
|
||||
blob: addDocToRootDoc(Buffer.from([0, 0]), 'live-doc', 'Live'),
|
||||
});
|
||||
|
||||
await indexerJob.reconcileDocumentCleanup({
|
||||
workspaceId: cleanupWorkspace.id,
|
||||
docId: 'missing-doc',
|
||||
cleanupVersion: 'version-1',
|
||||
});
|
||||
|
||||
t.true(deleteSpy.calledOnceWith(cleanupWorkspace.id, 'missing-doc'));
|
||||
t.false(indexSpy.called);
|
||||
const { payload } = await module.queue.waitFor(
|
||||
'backendRuntime.ackDocumentCleanupEffect'
|
||||
);
|
||||
t.deepEqual(payload, {
|
||||
workspaceId: cleanupWorkspace.id,
|
||||
docId: 'missing-doc',
|
||||
cleanupVersion: 'version-1',
|
||||
effect: 'search',
|
||||
});
|
||||
});
|
||||
|
||||
test('document cleanup reconcile reindexes restored doc before ack', async t => {
|
||||
const deleteSpy = Sinon.spy(indexerService, 'deleteDoc');
|
||||
const indexSpy = Sinon.spy(indexerService, 'indexDoc');
|
||||
const cleanupWorkspace = await module.create(Mockers.Workspace, {
|
||||
owner: user,
|
||||
});
|
||||
await module.create(Mockers.DocSnapshot, {
|
||||
workspaceId: cleanupWorkspace.id,
|
||||
docId: cleanupWorkspace.id,
|
||||
user,
|
||||
blob: addDocToRootDoc(Buffer.from([0, 0]), 'restored-doc', 'Restored'),
|
||||
});
|
||||
await module.create(Mockers.DocSnapshot, {
|
||||
workspaceId: cleanupWorkspace.id,
|
||||
docId: 'restored-doc',
|
||||
user,
|
||||
});
|
||||
const getDocSpy = Sinon.spy(docReader, 'getDoc');
|
||||
|
||||
await indexerJob.reconcileDocumentCleanup({
|
||||
workspaceId: cleanupWorkspace.id,
|
||||
docId: 'restored-doc',
|
||||
cleanupVersion: 'version-2',
|
||||
});
|
||||
|
||||
t.true(indexSpy.calledOnceWith(cleanupWorkspace.id, 'restored-doc'));
|
||||
t.false(deleteSpy.called);
|
||||
t.true(getDocSpy.calledWith(cleanupWorkspace.id, cleanupWorkspace.id));
|
||||
t.true(getDocSpy.calledWith(cleanupWorkspace.id, 'restored-doc'));
|
||||
const { payload } = await module.queue.waitFor(
|
||||
'backendRuntime.ackDocumentCleanupEffect'
|
||||
);
|
||||
t.deepEqual(payload, {
|
||||
workspaceId: cleanupWorkspace.id,
|
||||
docId: 'restored-doc',
|
||||
cleanupVersion: 'version-2',
|
||||
effect: 'search',
|
||||
});
|
||||
});
|
||||
|
||||
test('document cleanup reconcile only acknowledges when indexer is disabled', async t => {
|
||||
Sinon.stub(config.indexer, 'enabled').value(false);
|
||||
const deleteSpy = Sinon.spy(indexerService, 'deleteDoc');
|
||||
const indexSpy = Sinon.spy(indexerService, 'indexDoc');
|
||||
const getDocSpy = Sinon.spy(docReader, 'getDoc');
|
||||
|
||||
await indexerJob.reconcileDocumentCleanup({
|
||||
workspaceId: workspace.id,
|
||||
docId: 'disabled-doc',
|
||||
cleanupVersion: 'version-disabled',
|
||||
});
|
||||
|
||||
t.false(deleteSpy.called);
|
||||
t.false(indexSpy.called);
|
||||
t.false(getDocSpy.called);
|
||||
const { payload } = await module.queue.waitFor(
|
||||
'backendRuntime.ackDocumentCleanupEffect'
|
||||
);
|
||||
t.deepEqual(payload, {
|
||||
workspaceId: workspace.id,
|
||||
docId: 'disabled-doc',
|
||||
cleanupVersion: 'version-disabled',
|
||||
effect: 'search',
|
||||
});
|
||||
});
|
||||
|
||||
test('should handle indexer.deleteWorkspace job', async t => {
|
||||
const spy = Sinon.spy(indexerService, 'deleteWorkspace');
|
||||
|
||||
|
||||
@@ -1887,12 +1887,9 @@ test('should delete doc work', async t => {
|
||||
t.is(result4.nodes.length, 1);
|
||||
t.deepEqual(result4.nodes[0].fields.docId, [docId2]);
|
||||
|
||||
const count = module.queue.count('copilot.embedding.deleteDoc');
|
||||
|
||||
await indexerService.deleteDoc(workspaceId, docId1, {
|
||||
refresh: true,
|
||||
});
|
||||
t.is(module.queue.count('copilot.embedding.deleteDoc'), count + 1);
|
||||
|
||||
// make sure the docId1 is deleted
|
||||
result1 = await indexerService.search({
|
||||
|
||||
@@ -3,6 +3,7 @@ import './config';
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { ServerConfigModule } from '../../core/config';
|
||||
import { DocStorageModule } from '../../core/doc';
|
||||
import { PermissionModule } from '../../core/permission';
|
||||
import { QuotaServiceModule } from '../../core/quota';
|
||||
import { IndexerEvent } from './event';
|
||||
@@ -13,7 +14,12 @@ import { IndexerResolver } from './resolver';
|
||||
import { IndexerService } from './service';
|
||||
|
||||
@Module({
|
||||
imports: [ServerConfigModule, PermissionModule, QuotaServiceModule],
|
||||
imports: [
|
||||
ServerConfigModule,
|
||||
DocStorageModule,
|
||||
PermissionModule,
|
||||
QuotaServiceModule,
|
||||
],
|
||||
providers: [
|
||||
IndexerResolver,
|
||||
IndexerService,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { Config, JOB_SIGNAL, JobQueue, OnJob } from '../../base';
|
||||
import { DocReader } from '../../core/doc';
|
||||
import { readAllDocIdsFromWorkspaceSnapshot } from '../../core/utils/blocksuite';
|
||||
import { Models } from '../../models';
|
||||
import { IndexerService } from './service';
|
||||
@@ -24,6 +25,11 @@ declare global {
|
||||
'indexer.autoIndexWorkspaces': {
|
||||
lastIndexedWorkspaceSid?: number;
|
||||
};
|
||||
'indexer.reconcileDocumentCleanup': {
|
||||
workspaceId: string;
|
||||
docId: string;
|
||||
cleanupVersion: string;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,7 +41,8 @@ export class IndexerJob {
|
||||
private readonly models: Models,
|
||||
private readonly service: IndexerService,
|
||||
private readonly queue: JobQueue,
|
||||
private readonly config: Config
|
||||
private readonly config: Config,
|
||||
private readonly doc: DocReader
|
||||
) {}
|
||||
|
||||
@OnJob('indexer.indexDoc')
|
||||
@@ -66,6 +73,39 @@ export class IndexerJob {
|
||||
await this.service.deleteDoc(workspaceId, docId);
|
||||
}
|
||||
|
||||
@OnJob('indexer.reconcileDocumentCleanup')
|
||||
async reconcileDocumentCleanup({
|
||||
workspaceId,
|
||||
docId,
|
||||
cleanupVersion,
|
||||
}: Jobs['indexer.reconcileDocumentCleanup']) {
|
||||
if (this.config.indexer.enabled) {
|
||||
const root = await this.doc.getDoc(workspaceId, workspaceId);
|
||||
if (!root) {
|
||||
throw new Error(`workspace root ${workspaceId} not found`);
|
||||
}
|
||||
const live = readAllDocIdsFromWorkspaceSnapshot(root.bin, true).includes(
|
||||
docId
|
||||
);
|
||||
if (live) {
|
||||
if (!(await this.doc.getDoc(workspaceId, docId))) {
|
||||
throw new Error(
|
||||
`restored document ${workspaceId}/${docId} not found`
|
||||
);
|
||||
}
|
||||
await this.service.indexDoc(workspaceId, docId);
|
||||
} else {
|
||||
await this.service.deleteDoc(workspaceId, docId);
|
||||
}
|
||||
}
|
||||
await this.queue.add('backendRuntime.ackDocumentCleanupEffect', {
|
||||
workspaceId,
|
||||
docId,
|
||||
cleanupVersion,
|
||||
effect: 'search',
|
||||
});
|
||||
}
|
||||
|
||||
@OnJob('indexer.indexWorkspace')
|
||||
async indexWorkspace({ workspaceId }: Jobs['indexer.indexWorkspace']) {
|
||||
if (!this.config.indexer.enabled) {
|
||||
@@ -79,16 +119,13 @@ export class IndexerJob {
|
||||
return;
|
||||
}
|
||||
|
||||
const snapshot = await this.models.doc.getSnapshot(
|
||||
workspaceId,
|
||||
workspaceId
|
||||
);
|
||||
if (!snapshot) {
|
||||
const root = await this.doc.getDoc(workspaceId, workspaceId);
|
||||
if (!root) {
|
||||
this.logger.warn(`workspace snapshot ${workspaceId} not found`);
|
||||
return;
|
||||
}
|
||||
|
||||
const docIdsInWorkspace = readAllDocIdsFromWorkspaceSnapshot(snapshot.blob);
|
||||
const docIdsInWorkspace = readAllDocIdsFromWorkspaceSnapshot(root.bin);
|
||||
const docIdsInIndexer = await this.service.listDocIds(workspaceId);
|
||||
|
||||
const docIdsInWorkspaceSet = new Set(docIdsInWorkspace);
|
||||
|
||||
@@ -374,14 +374,6 @@ export class IndexerService {
|
||||
);
|
||||
|
||||
await this.deleteBlocksByDocId(workspaceId, docId, options);
|
||||
await this.queue.add('copilot.session.deleteDoc', {
|
||||
workspaceId,
|
||||
docId,
|
||||
});
|
||||
await this.queue.add('copilot.embedding.deleteDoc', {
|
||||
workspaceId,
|
||||
docId,
|
||||
});
|
||||
this.logger.log(`deleted doc ${workspaceId}/${docId}`);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user