feat(server): make server storage adapters (#7902)

This commit is contained in:
forehalo
2024-08-21 05:30:26 +00:00
parent 6f9f579e5d
commit e20bdbf925
29 changed files with 1987 additions and 2111 deletions
-394
View File
@@ -1,394 +0,0 @@
import { mock } from 'node:test';
import { TestingModule } from '@nestjs/testing';
import { PrismaClient } from '@prisma/client';
import test from 'ava';
import * as Sinon from 'sinon';
import { applyUpdate, Doc as YDoc, encodeStateAsUpdate } from 'yjs';
import { DocManager, DocModule } from '../src/core/doc';
import { QuotaModule } from '../src/core/quota';
import { StorageModule } from '../src/core/storage';
import { Config } from '../src/fundamentals/config';
import { createTestingModule } from './utils';
const createModule = () => {
return createTestingModule(
{
imports: [QuotaModule, StorageModule, DocModule],
},
false
);
};
let m: TestingModule;
let timer: Sinon.SinonFakeTimers;
// cleanup database before each test
test.beforeEach(async () => {
timer = Sinon.useFakeTimers({
toFake: ['setInterval'],
});
m = await createModule();
await m.init();
});
test.afterEach.always(async () => {
await m.close();
timer.restore();
});
test('should setup update poll interval', async t => {
const m = await createModule();
const manager = m.get(DocManager);
const fake = mock.method(manager, 'setup');
await m.init();
t.is(fake.mock.callCount(), 1);
// @ts-expect-error private member
t.truthy(manager.job);
m.close();
});
test('should be able to stop poll', async t => {
const manager = m.get(DocManager);
const fake = mock.method(manager, 'destroy');
await m.close();
t.is(fake.mock.callCount(), 1);
// @ts-expect-error private member
t.is(manager.job, null);
});
test('should poll when intervel due', async t => {
const manager = m.get(DocManager);
const interval = m.get(Config).doc.manager.updatePollInterval;
let resolve: any;
// @ts-expect-error private method
const fake = mock.method(manager, 'autoSquash', () => {
return new Promise(_resolve => {
resolve = _resolve;
});
});
timer.tick(interval);
t.is(fake.mock.callCount(), 1);
// busy
timer.tick(interval);
// @ts-expect-error private member
t.is(manager.busy, true);
t.is(fake.mock.callCount(), 1);
resolve();
await timer.tickAsync(1);
// @ts-expect-error private member
t.is(manager.busy, false);
timer.tick(interval);
t.is(fake.mock.callCount(), 2);
});
test('should merge update when intervel due', async t => {
const db = m.get(PrismaClient);
const manager = m.get(DocManager);
const doc = new YDoc();
const text = doc.getText('content');
text.insert(0, 'hello');
const update = encodeStateAsUpdate(doc);
const ws = await db.workspace.create({
data: {
id: '1',
public: false,
},
});
await db.update.createMany({
data: [
{
id: '1',
workspaceId: '1',
blob: Buffer.from([0, 0]),
seq: 1,
},
{
id: '1',
workspaceId: '1',
blob: Buffer.from(update),
seq: 2,
},
],
});
// @ts-expect-error private method
await manager.autoSquash();
t.deepEqual(
(await manager.getBinary(ws.id, '1'))?.binary.toString('hex'),
Buffer.from(update.buffer).toString('hex')
);
let appendUpdate = Buffer.from([]);
doc.on('update', update => {
appendUpdate = Buffer.from(update);
});
text.insert(5, 'world');
await db.update.create({
data: {
workspaceId: ws.id,
id: '1',
blob: appendUpdate,
seq: 3,
},
});
// @ts-expect-error private method
await manager.autoSquash();
t.deepEqual(
(await manager.getBinary(ws.id, '1'))?.binary.toString('hex'),
Buffer.from(encodeStateAsUpdate(doc)).toString('hex')
);
});
test('should have sequential update number', async t => {
const db = m.get(PrismaClient);
const manager = m.get(DocManager);
const doc = new YDoc();
const text = doc.getText('content');
const updates: Buffer[] = [];
doc.on('update', update => {
updates.push(Buffer.from(update));
});
text.insert(0, 'hello');
text.insert(5, 'world');
text.insert(5, ' ');
await Promise.all(updates.map(update => manager.push('2', '2', update)));
// [1,2,3]
let records = await manager.getUpdates('2', '2');
t.deepEqual(
records.map(({ seq }) => seq),
[1, 2, 3]
);
// @ts-expect-error private method
await manager.autoSquash();
await db.snapshot.update({
where: {
id_workspaceId: {
id: '2',
workspaceId: '2',
},
},
data: {
seq: 0x3ffffffe,
},
});
await Promise.all(updates.map(update => manager.push('2', '2', update)));
records = await manager.getUpdates('2', '2');
// push a new update with new seq num
await manager.push('2', '2', updates[0]);
// let the manager ignore update with the new seq num
const stub = Sinon.stub(manager, 'getUpdates').resolves(records);
// @ts-expect-error private method
await manager.autoSquash();
stub.restore();
records = await manager.getUpdates('2', '2');
// should not merge in one run
t.not(records.length, 0);
});
test('should have correct sequential update number with batching push', async t => {
const manager = m.get(DocManager);
const doc = new YDoc();
const text = doc.getText('content');
const updates: Buffer[] = [];
doc.on('update', update => {
updates.push(Buffer.from(update));
});
text.insert(0, 'hello');
text.insert(5, 'world');
text.insert(5, ' ');
await manager.batchPush('2', '2', updates);
// [1,2,3]
const records = await manager.getUpdates('2', '2');
t.deepEqual(
records.map(({ seq }) => seq),
[1, 2, 3]
);
});
test('should retry if seq num conflict', async t => {
const manager = m.get(DocManager);
// @ts-expect-error private method
const stub = Sinon.stub(manager, 'getUpdateSeq');
stub.onCall(0).resolves(1);
// seq num conflict
stub.onCall(1).resolves(1);
stub.onCall(2).resolves(2);
await t.notThrowsAsync(() => manager.push('1', '1', Buffer.from([0, 0])));
await t.notThrowsAsync(() => manager.push('1', '1', Buffer.from([0, 0])));
t.is(stub.callCount, 3);
});
test('should throw if meet max retry times', async t => {
const manager = m.get(DocManager);
// @ts-expect-error private method
const stub = Sinon.stub(manager, 'getUpdateSeq');
stub.resolves(1);
await t.notThrowsAsync(() => manager.push('1', '1', Buffer.from([0, 0])));
await t.throwsAsync(
() => manager.push('1', '1', Buffer.from([0, 0]), 3 /* retry 3 times */),
{ message: 'Failed to push update' }
);
t.is(stub.callCount, 5);
});
test('should be able to insert the snapshot if it is new created', async t => {
const manager = m.get(DocManager);
{
const doc = new YDoc();
const text = doc.getText('content');
text.insert(0, 'hello');
const update = encodeStateAsUpdate(doc);
await manager.push('1', '1', Buffer.from(update));
}
const updates = await manager.getUpdates('1', '1');
t.is(updates.length, 1);
// @ts-expect-error private
const { doc } = await manager.squash(null, updates);
t.truthy(doc);
t.is(doc.getText('content').toString(), 'hello');
const restUpdates = await manager.getUpdates('1', '1');
t.is(restUpdates.length, 0);
});
test('should be able to merge updates into snapshot', async t => {
const manager = m.get(DocManager);
const updates: Buffer[] = [];
{
const doc = new YDoc();
doc.on('update', data => {
updates.push(Buffer.from(data));
});
const text = doc.getText('content');
text.insert(0, 'hello');
text.insert(5, 'world');
text.insert(5, ' ');
text.insert(11, '!');
}
{
await manager.batchPush('1', '1', updates.slice(0, 2));
// do the merge
const { doc } = (await manager.get('1', '1'))!;
t.is(doc.getText('content').toString(), 'helloworld');
}
{
await manager.batchPush('1', '1', updates.slice(2));
const { doc } = (await manager.get('1', '1'))!;
t.is(doc.getText('content').toString(), 'hello world!');
}
const restUpdates = await manager.getUpdates('1', '1');
t.is(restUpdates.length, 0);
});
test('should not update snapshot if doc is outdated', async t => {
const manager = m.get(DocManager);
const db = m.get(PrismaClient);
const updates: Buffer[] = [];
{
const doc = new YDoc();
doc.on('update', data => {
updates.push(Buffer.from(data));
});
const text = doc.getText('content');
text.insert(0, 'hello');
text.insert(5, 'world');
text.insert(5, ' ');
text.insert(11, '!');
}
await manager.batchPush('2', '1', updates.slice(0, 2)); // 'helloworld'
// merge updates into snapshot
await manager.get('2', '1');
// fake the snapshot is a lot newer
await db.snapshot.update({
where: {
id_workspaceId: {
workspaceId: '2',
id: '1',
},
},
data: {
updatedAt: new Date(Date.now() + 10000),
},
});
{
const snapshot = await manager.getSnapshot('2', '1');
await manager.batchPush('2', '1', updates.slice(2)); // 'hello world!'
const updateRecords = await manager.getUpdates('2', '1');
// @ts-expect-error private
const { doc } = await manager.squash(snapshot, updateRecords);
// all updated will merged into doc not matter it's timestamp is outdated or not,
// but the snapshot record will not be updated
t.is(doc.getText('content').toString(), 'hello world!');
}
{
const doc = new YDoc();
applyUpdate(doc, (await manager.getSnapshot('2', '1'))!.blob);
// the snapshot will not get touched if the new doc's timestamp is outdated
t.is(doc.getText('content').toString(), 'helloworld');
// the updates are known as outdated, so they will be deleted
t.is((await manager.getUpdates('2', '1')).length, 0);
}
});
@@ -4,36 +4,42 @@ import { PrismaClient } from '@prisma/client';
import test from 'ava';
import * as Sinon from 'sinon';
import { DocHistoryManager } from '../src/core/doc';
import { PermissionModule } from '../src/core/permission';
import { QuotaModule } from '../src/core/quota';
import { StorageModule } from '../src/core/storage';
import type { EventPayload } from '../src/fundamentals/event';
import { createTestingModule } from './utils';
import {
DocStorageModule,
PgWorkspaceDocStorageAdapter,
} from '../../src/core/doc';
import { DocStorageOptions } from '../../src/core/doc/options';
import { DocRecord } from '../../src/core/doc/storage';
import { createTestingModule, initTestingDB } from '../utils';
let m: TestingModule;
let manager: DocHistoryManager;
let adapter: PgWorkspaceDocStorageAdapter;
let db: PrismaClient;
// cleanup database before each test
test.beforeEach(async () => {
test.before(async () => {
m = await createTestingModule({
imports: [StorageModule, QuotaModule, PermissionModule],
providers: [DocHistoryManager],
imports: [DocStorageModule],
});
manager = m.get(DocHistoryManager);
Sinon.stub(manager, 'getExpiredDateFromNow').resolves(
new Date(Date.now() + 1000)
);
adapter = m.get(PgWorkspaceDocStorageAdapter);
db = m.get(PrismaClient);
});
test.afterEach.always(async () => {
await m.close();
test.beforeEach(async () => {
await initTestingDB(db);
const options = m.get(DocStorageOptions);
Sinon.stub(options, 'historyMaxAge').resolves(1000);
});
test.afterEach(async () => {
Sinon.restore();
});
test.after.always(async () => {
await m.close();
});
const snapshot: Snapshot = {
workspaceId: '1',
id: 'doc1',
@@ -44,21 +50,22 @@ const snapshot: Snapshot = {
createdAt: new Date(),
};
function getEventData(
timestamp: Date = new Date()
): EventPayload<'snapshot.updated'> {
function getSnapshot(timestamp: number = Date.now()): DocRecord {
return {
workspaceId: snapshot.workspaceId,
id: snapshot.id,
previous: { ...snapshot, updatedAt: timestamp },
spaceId: snapshot.workspaceId,
docId: snapshot.id,
bin: snapshot.blob,
timestamp,
};
}
test('should create doc history if never created before', async t => {
Sinon.stub(manager, 'last').resolves(null);
// @ts-expect-error private method
Sinon.stub(adapter, 'lastDocHistory').resolves(null);
const timestamp = new Date();
await manager.onDocUpdated(getEventData(timestamp));
const timestamp = Date.now();
// @ts-expect-error private method
await adapter.createDocHistory(getSnapshot(timestamp));
const history = await db.snapshotHistory.findFirst({
where: {
@@ -68,33 +75,17 @@ test('should create doc history if never created before', async t => {
});
t.truthy(history);
t.is(history?.timestamp.getTime(), timestamp.getTime());
t.is(history?.timestamp.getTime(), timestamp);
});
test('should not create history if timestamp equals to last record', async t => {
const timestamp = new Date();
Sinon.stub(manager, 'last').resolves({ timestamp, state: null });
await manager.onDocUpdated(getEventData(timestamp));
// @ts-expect-error private method
Sinon.stub(adapter, 'lastDocHistory').resolves({ timestamp, state: null });
const history = await db.snapshotHistory.findFirst({
where: {
workspaceId: '1',
id: 'doc1',
},
});
t.falsy(history);
});
test('should not create history if state equals to last record', async t => {
const timestamp = new Date();
Sinon.stub(manager, 'last').resolves({
timestamp: new Date(timestamp.getTime() - 1),
state: snapshot.state,
});
await manager.onDocUpdated(getEventData(timestamp));
// @ts-expect-error private method
await adapter.createDocHistory(getSnapshot(timestamp));
const history = await db.snapshotHistory.findFirst({
where: {
@@ -108,12 +99,15 @@ test('should not create history if state equals to last record', async t => {
test('should not create history if time diff is less than interval config', async t => {
const timestamp = new Date();
Sinon.stub(manager, 'last').resolves({
// @ts-expect-error private method
Sinon.stub(adapter, 'lastDocHistory').resolves({
timestamp: new Date(timestamp.getTime() - 1000),
state: Buffer.from([0, 1]),
});
await manager.onDocUpdated(getEventData(timestamp));
// @ts-expect-error private method
await adapter.createDocHistory(getSnapshot(timestamp));
const history = await db.snapshotHistory.findFirst({
where: {
@@ -127,12 +121,15 @@ test('should not create history if time diff is less than interval config', asyn
test('should create history if time diff is larger than interval config and state diff', async t => {
const timestamp = new Date();
Sinon.stub(manager, 'last').resolves({
// @ts-expect-error private method
Sinon.stub(adapter, 'lastDocHistory').resolves({
timestamp: new Date(timestamp.getTime() - 1000 * 60 * 20),
state: Buffer.from([0, 1]),
});
await manager.onDocUpdated(getEventData(timestamp));
// @ts-expect-error private method
await adapter.createDocHistory(getSnapshot(timestamp));
const history = await db.snapshotHistory.findFirst({
where: {
@@ -146,12 +143,15 @@ test('should create history if time diff is larger than interval config and stat
test('should create history with force flag even if time diff in small', async t => {
const timestamp = new Date();
Sinon.stub(manager, 'last').resolves({
// @ts-expect-error private method
Sinon.stub(adapter, 'lastDocHistory').resolves({
timestamp: new Date(timestamp.getTime() - 1),
state: Buffer.from([0, 1]),
});
await manager.onDocUpdated(getEventData(timestamp), true);
// @ts-expect-error private method
await adapter.createDocHistory(getSnapshot(timestamp), true);
const history = await db.snapshotHistory.findFirst({
where: {
@@ -194,31 +194,31 @@ test('should correctly list all history records', async t => {
})),
});
const list = await manager.list(
const list = await adapter.listDocHistories(
snapshot.workspaceId,
snapshot.id,
new Date(timestamp + 20),
8
{ before: timestamp + 20, limit: 8 }
);
const count = await manager.count(snapshot.workspaceId, snapshot.id);
const count = await db.snapshotHistory.count();
t.is(list.length, 8);
t.is(count, 10);
t.is(count, 20);
});
test('should be able to get history data', async t => {
const timestamp = new Date();
const timestamp = Date.now();
await manager.onDocUpdated(getEventData(timestamp), true);
// @ts-expect-error private method
await adapter.createDocHistory(getSnapshot(timestamp), true);
const history = await manager.get(
const history = await adapter.getDocHistory(
snapshot.workspaceId,
snapshot.id,
timestamp
);
t.truthy(history);
t.deepEqual(history?.blob, snapshot.blob);
t.deepEqual(history?.bin, snapshot.blob);
});
test('should be able to get last history record', async t => {
@@ -238,7 +238,11 @@ test('should be able to get last history record', async t => {
})),
});
const history = await manager.last(snapshot.workspaceId, snapshot.id);
// @ts-expect-error private method
const history = await adapter.lastDocHistory(
snapshot.workspaceId,
snapshot.id
);
t.truthy(history);
t.is(history?.timestamp.getTime(), timestamp + 9);
@@ -253,12 +257,14 @@ test('should be able to recover from history', async t => {
},
});
const history1Timestamp = snapshot.updatedAt.getTime() - 10;
await manager.onDocUpdated(getEventData(new Date(history1Timestamp)));
await manager.recover(
// @ts-expect-error private method
await adapter.createDocHistory(getSnapshot(history1Timestamp));
await adapter.rollbackDoc(
snapshot.workspaceId,
snapshot.id,
new Date(history1Timestamp)
history1Timestamp
);
const [history1, history2] = await db.snapshotHistory.findMany({
@@ -273,49 +279,4 @@ test('should be able to recover from history', async t => {
// new history data force created with snapshot state before recovered
t.deepEqual(history2.blob, Buffer.from([1, 1]));
t.deepEqual(history2.state, Buffer.from([1, 1]));
});
test('should be able to cleanup expired history', async t => {
const timestamp = Date.now();
// insert expired data
await db.snapshotHistory.createMany({
data: Array.from({ length: 10 })
.fill(0)
.map((_, i) => ({
workspaceId: snapshot.workspaceId,
id: snapshot.id,
blob: snapshot.blob,
state: snapshot.state,
timestamp: new Date(timestamp - 10 - i),
expiredAt: new Date(timestamp - 1),
})),
});
// insert available data
await db.snapshotHistory.createMany({
data: Array.from({ length: 10 })
.fill(0)
.map((_, i) => ({
workspaceId: snapshot.workspaceId,
id: snapshot.id,
blob: snapshot.blob,
state: snapshot.state,
timestamp: new Date(timestamp + i),
expiredAt: new Date(timestamp + 1000),
})),
});
let count = await db.snapshotHistory.count();
t.is(count, 20);
await manager.cleanupExpiredHistory();
count = await db.snapshotHistory.count();
t.is(count, 10);
const example = await db.snapshotHistory.findFirst();
t.truthy(example);
t.true(example!.expiredAt > new Date());
});
@@ -116,7 +116,7 @@ test('should share a page', async t => {
const msg1 = await publishPage(app, u2.token.token, 'not_exists_ws', 'page2');
t.is(
msg1,
'You do not have permission to access workspace not_exists_ws.',
'You do not have permission to access Space not_exists_ws.',
'unauthorized user can share page'
);
const msg2 = await revokePublicPage(
@@ -127,7 +127,7 @@ test('should share a page', async t => {
);
t.is(
msg2,
'You do not have permission to access workspace not_exists_ws.',
'You do not have permission to access Space not_exists_ws.',
'unauthorized user can share page'
);
@@ -9,7 +9,7 @@ import request from 'supertest';
import { AppModule } from '../../src/app.module';
import { CurrentUser } from '../../src/core/auth';
import { AuthService } from '../../src/core/auth/service';
import { DocHistoryManager, DocManager } from '../../src/core/doc';
import { PgWorkspaceDocStorageAdapter } from '../../src/core/doc';
import { WorkspaceBlobStorage } from '../../src/core/storage';
import { createTestingApp, internalSignIn } from '../utils';
@@ -18,19 +18,17 @@ const test = ava as TestFn<{
db: PrismaClient;
app: INestApplication;
storage: Sinon.SinonStubbedInstance<WorkspaceBlobStorage>;
doc: Sinon.SinonStubbedInstance<DocManager>;
workspace: Sinon.SinonStubbedInstance<PgWorkspaceDocStorageAdapter>;
}>;
test.beforeEach(async t => {
test.before(async t => {
const { app } = await createTestingApp({
imports: [AppModule],
tapModule: m => {
m.overrideProvider(WorkspaceBlobStorage)
.useValue(Sinon.createStubInstance(WorkspaceBlobStorage))
.overrideProvider(DocManager)
.useValue(Sinon.createStubInstance(DocManager))
.overrideProvider(DocHistoryManager)
.useValue(Sinon.createStubInstance(DocHistoryManager));
.overrideProvider(PgWorkspaceDocStorageAdapter)
.useValue(Sinon.createStubInstance(PgWorkspaceDocStorageAdapter));
},
});
@@ -41,7 +39,7 @@ test.beforeEach(async t => {
t.context.db = db;
t.context.app = app;
t.context.storage = app.get(WorkspaceBlobStorage);
t.context.doc = app.get(DocManager);
t.context.workspace = app.get(PgWorkspaceDocStorageAdapter);
await db.workspacePage.create({
data: {
@@ -83,7 +81,7 @@ test.beforeEach(async t => {
});
});
test.afterEach.always(async t => {
test.after.always(async t => {
await t.context.app.close();
});
@@ -227,10 +225,12 @@ test('should not be able to get private workspace with private page', async t =>
});
test('should be able to get doc', async t => {
const { app, doc } = t.context;
const { app, workspace: doc } = t.context;
doc.getBinary.resolves({
binary: Buffer.from([0, 0]),
doc.getDoc.resolves({
spaceId: '',
docId: '',
bin: Buffer.from([0, 0]),
timestamp: Date.now(),
});
@@ -244,10 +244,12 @@ test('should be able to get doc', async t => {
});
test('should be able to change page publish mode', async t => {
const { app, doc, db } = t.context;
const { app, workspace: doc, db } = t.context;
doc.getBinary.resolves({
binary: Buffer.from([0, 0]),
doc.getDoc.resolves({
spaceId: '',
docId: '',
bin: Buffer.from([0, 0]),
timestamp: Date.now(),
});