perf(y-indexeddb): improve boost and loading time (#1879)

packages/y-indexeddb/benchmark/index.ts

@@ -0,0 +1,103 @@
+#!/usr/bin/env ts-node-esm
+import 'fake-indexeddb/auto';
+
+const map = new Map();
+const localStorage = {
+  getItem: (key: string) => map.get(key),
+  setItem: (key: string, value: string) => map.set(key, value),
+  clear: () => map.clear(),
+};
+
+// @ts-expect-error
+globalThis.localStorage = localStorage;
+
+import { Workspace } from '@blocksuite/store';
+import { IndexeddbPersistence } from 'y-indexeddb';
+
+const Y = Workspace.Y;
+
+// eslint-disable-next-line @typescript-eslint/no-restricted-imports
+import { createIndexedDBProvider } from '../src/index.js';
+
+async function yjs_create_persistence(n = 1e3) {
+  for (let i = 0; i < n; i++) {
+    const yDoc = new Y.Doc();
+    const persistence = new IndexeddbPersistence('test', yDoc);
+    await persistence.whenSynced;
+    persistence.destroy();
+  }
+}
+
+async function yjs_single_persistence(n = 1e5) {
+  const yDoc = new Y.Doc();
+  const map = yDoc.getMap();
+  for (let i = 0; i < n; i++) {
+    map.set(`${i}`, i);
+  }
+  {
+    const persistence = new IndexeddbPersistence('test', yDoc);
+    await persistence.whenSynced;
+    persistence.destroy();
+  }
+  {
+    const persistence = new IndexeddbPersistence('test', yDoc);
+    await persistence.whenSynced;
+    persistence.destroy();
+  }
+}
+
+async function toeverything_create_provider(n = 1e3) {
+  for (let i = 0; i < n; i++) {
+    const yDoc = new Y.Doc();
+    const provider = createIndexedDBProvider('test', yDoc);
+    provider.connect();
+    await provider.whenSynced;
+    provider.disconnect();
+  }
+}
+async function toeverything_single_persistence(n = 1e5) {
+  const yDoc = new Y.Doc();
+  const map = yDoc.getMap();
+  for (let i = 0; i < n; i++) {
+    map.set(`${i}`, i);
+  }
+  const provider = createIndexedDBProvider('test', yDoc, 'test');
+  provider.connect();
+  await provider.whenSynced;
+  provider.disconnect();
+  provider.connect();
+  await provider.whenSynced;
+  provider.disconnect();
+}
+
+async function main() {
+  console.log('create many persistence');
+  performance.mark('start');
+  await yjs_create_persistence();
+  performance.mark('end');
+  performance.measure('yjs', 'start', 'end');
+  indexedDB.deleteDatabase('test');
+  performance.mark('start');
+  await toeverything_create_provider();
+  performance.mark('end');
+  performance.measure('toeverything', 'start', 'end');
+  console.log(performance.getEntriesByType('measure'));
+  indexedDB.deleteDatabase('test');
+  performance.clearMarks();
+  performance.clearMeasures();
+  localStorage.clear();
+
+  console.log('single persistence with huge updates');
+  performance.mark('start');
+  await yjs_single_persistence();
+  performance.mark('end');
+  performance.measure('yjs', 'start', 'end');
+  indexedDB.deleteDatabase('test');
+  performance.mark('start');
+  await toeverything_single_persistence();
+  performance.mark('end');
+  performance.measure('toeverything', 'start', 'end');
+  console.log(performance.getEntriesByType('measure'));
+}
+
+main().then();

packages/y-indexeddb/package.json

@@ -28,7 +28,8 @@
     "@blocksuite/blocks": "0.0.0-20230411141436-ec6b051d-nightly",
     "@blocksuite/store": "0.0.0-20230411141436-ec6b051d-nightly",
     "vite": "^4.2.1",
-    "vite-plugin-dts": "^2.2.0"
+    "vite-plugin-dts": "^2.2.0",
+    "y-indexeddb": "^9.0.10"
   },
   "peerDependencies": {
     "yjs": "^13.5.51"

packages/y-indexeddb/src/__tests__/index.spec.ts

@@ -6,7 +6,7 @@ import 'fake-indexeddb/auto';
 import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
 import { assertExists, uuidv4, Workspace } from '@blocksuite/store';
 import { openDB } from 'idb';
-import { beforeEach, describe, expect, test, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
 import { applyUpdate, Doc, encodeStateAsUpdate } from 'yjs';
 
 import type { WorkspacePersist } from '../index';
@@ -20,7 +20,7 @@ import {
 } from '../index';
 
 async function getUpdates(id: string): Promise<ArrayBuffer[]> {
-  const db = await openDB('affine-local', dbVersion);
+  const db = await openDB(rootDBName, dbVersion);
   const store = await db
     .transaction('workspace', 'readonly')
     .objectStore('workspace');
@@ -32,6 +32,7 @@ async function getUpdates(id: string): Promise<ArrayBuffer[]> {
 
 let id: string;
 let workspace: Workspace;
+const rootDBName = 'affine-local';
 
 beforeEach(() => {
   id = uuidv4();
@@ -42,12 +43,16 @@ beforeEach(() => {
   workspace.register(AffineSchemas).register(__unstableSchemas);
 });
 
+afterEach(() => {
+  indexedDB.deleteDatabase('affine-local');
+});
+
 describe('indexeddb provider', () => {
   test('connect', async () => {
     const provider = createIndexedDBProvider(workspace.id, workspace.doc);
     provider.connect();
     await provider.whenSynced;
-    const db = await openDB('affine-local', dbVersion);
+    const db = await openDB(rootDBName, dbVersion);
     {
       const store = await db
         .transaction('workspace', 'readonly')
@@ -89,7 +94,8 @@ describe('indexeddb provider', () => {
       .register(__unstableSchemas);
     const provider2 = createIndexedDBProvider(
       secondWorkspace.id,
-      secondWorkspace.doc
+      secondWorkspace.doc,
+      rootDBName
     );
     provider2.connect();
     await provider2.whenSynced;
@@ -99,7 +105,11 @@ describe('indexeddb provider', () => {
   });
 
   test('disconnect suddenly', async () => {
-    const provider = createIndexedDBProvider(workspace.id, workspace.doc);
+    const provider = createIndexedDBProvider(
+      workspace.id,
+      workspace.doc,
+      rootDBName
+    );
     const fn = vi.fn();
     provider.connect();
     provider.disconnect();
@@ -109,10 +119,14 @@ describe('indexeddb provider', () => {
   });
 
   test('connect and disconnect', async () => {
-    const provider = createIndexedDBProvider(workspace.id, workspace.doc);
+    const provider = createIndexedDBProvider(
+      workspace.id,
+      workspace.doc,
+      rootDBName
+    );
     provider.connect();
     const p1 = provider.whenSynced;
-    await provider.whenSynced;
+    await p1;
     provider.disconnect();
     {
       const page = workspace.createPage('page0');
@@ -126,7 +140,7 @@ describe('indexeddb provider', () => {
     }
     provider.connect();
     const p2 = provider.whenSynced;
-    await provider.whenSynced;
+    await p2;
     {
       const updates = await getUpdates(workspace.id);
       expect(updates).not.toEqual([]);
@@ -137,7 +151,11 @@ describe('indexeddb provider', () => {
 
   test('merge', async () => {
     setMergeCount(5);
-    const provider = createIndexedDBProvider(workspace.id, workspace.doc);
+    const provider = createIndexedDBProvider(
+      workspace.id,
+      workspace.doc,
+      rootDBName
+    );
     provider.connect();
     {
       const page = workspace.createPage('page0');
@@ -153,6 +171,31 @@ describe('indexeddb provider', () => {
       expect(updates.length).lessThanOrEqual(5);
     }
   });
+
+  test("data won't be lost", async () => {
+    const id = uuidv4();
+    const doc = new Workspace.Y.Doc();
+    const map = doc.getMap('map');
+    for (let i = 0; i < 100; i++) {
+      map.set(`${i}`, i);
+    }
+    {
+      const provider = createIndexedDBProvider(id, doc, rootDBName);
+      provider.connect();
+      await provider.whenSynced;
+      provider.disconnect();
+    }
+    {
+      const newDoc = new Workspace.Y.Doc();
+      const provider = createIndexedDBProvider(id, newDoc, rootDBName);
+      provider.connect();
+      await provider.whenSynced;
+      provider.disconnect();
+      newDoc.getMap('map').forEach((value, key) => {
+        expect(value).toBe(parseInt(key));
+      });
+    }
+  });
 });
 
 describe('milestone', () => {

packages/y-indexeddb/src/index.ts

@@ -299,8 +299,16 @@ export const createIndexedDBProvider = (
         });
       } else {
         const updates = data.updates.map(({ update }) => update);
-        const update = mergeUpdates(updates);
-        const newUpdate = diffUpdate(encodeStateAsUpdate(doc), update);
+        const fakeDoc = new Doc();
+        fakeDoc.transact(() => {
+          updates.forEach(update => {
+            applyUpdate(fakeDoc, update);
+          });
+        }, indexeddbOrigin);
+        const newUpdate = diffUpdate(
+          encodeStateAsUpdate(doc),
+          encodeStateAsUpdate(fakeDoc)
+        );
         await store.put({
           ...data,
           updates: [