Merge branch 'canary' into stable

packages/frontend/workspace/package.json

@@ -22,7 +22,7 @@
     "@toeverything/hooks": "workspace:*",
     "@toeverything/y-indexeddb": "workspace:*",
     "async-call-rpc": "^6.3.1",
-    "idb": "^7.1.1",
+    "idb": "^8.0.0",
     "idb-keyval": "^6.2.1",
     "is-svg": "^5.0.0",
     "jotai": "^2.5.1",

packages/frontend/workspace/src/affine/crud.ts

@@ -40,13 +40,8 @@ export const CRUD: WorkspaceCRUD<WorkspaceFlavour.AFFINE_CLOUD> = {
     }
     const { createWorkspace } = await fetcher({
       query: createWorkspaceMutation,
-      variables: {
-        init: new File(
-          [Y.encodeStateAsUpdate(upstreamWorkspace.doc)],
-          'initBinary.yDoc'
-        ),
-      },
     });
+
     createdWorkspaces.push(upstreamWorkspace.id);
     const newBlockSuiteWorkspace = getOrCreateWorkspace(
       createWorkspace.id,

packages/frontend/workspace/src/atom.ts

@@ -1,7 +1,6 @@
 import { DebugLogger } from '@affine/debug';
 import type { WorkspaceAdapter } from '@affine/env/workspace';
 import { WorkspaceFlavour } from '@affine/env/workspace';
-import type { BlockHub } from '@blocksuite/blocks';
 import { assertEquals, assertExists } from '@blocksuite/global/utils';
 import {
   currentPageIdAtom,
@@ -259,7 +258,4 @@ export const refreshRootMetadataAtom = atom(null, (get, set) => {
   set(rootWorkspacesMetadataPrimitiveAtom, fetchMetadata(get));
 });
 
-// blocksuite atoms,
-// each app should have only one block-hub in the same time
-export const rootBlockHubAtom = atom<Readonly<BlockHub> | null>(null);
 //#endregion

packages/frontend/workspace/src/blob/storage/affine-cloud.ts

@@ -8,6 +8,7 @@ import {
 import { fetcher } from '@affine/workspace/affine/gql';
 
 import type { BlobStorage } from '../engine';
+import { bufferToBlob } from '../util';
 
 export const createAffineCloudBlobStorage = (
   workspaceId: string
@@ -26,7 +27,7 @@ export const createAffineCloudBlobStorage = (
             // status not in the range 200-299
             return undefined;
           }
-          return await res.blob();
+          return bufferToBlob(await res.arrayBuffer());
         }
       );
     },

packages/frontend/workspace/src/blob/storage/indexeddb.ts

@@ -1,6 +1,7 @@
 import { createStore, del, get, keys, set } from 'idb-keyval';
 
 import type { BlobStorage } from '../engine';
+import { bufferToBlob } from '../util';
 
 export const createIndexeddbBlobStorage = (
   workspaceId: string
@@ -13,7 +14,7 @@ export const createIndexeddbBlobStorage = (
     get: async (key: string) => {
       const res = await get<ArrayBuffer>(key, db);
       if (res) {
-        return new Blob([res], { type: await get(key, mimeTypeDb) });
+        return bufferToBlob(res);
       }
       return undefined;
     },

packages/frontend/workspace/src/manager/index.ts

@@ -1,4 +1,3 @@
-import { isBrowser } from '@affine/env/constant';
 import type { BlockSuiteFeatureFlags } from '@affine/env/global';
 import { WorkspaceFlavour } from '@affine/env/workspace';
 import { createAffinePublicProviders } from '@affine/workspace/providers';
@@ -25,10 +24,6 @@ function setEditorFlags(workspace: Workspace) {
       value
     );
   });
-  workspace.awarenessStore.setFlag(
-    'enable_bookmark_operation',
-    environment.isDesktop
-  );
 }
 
 type UpdateCallback = (
@@ -116,7 +111,6 @@ export function getOrCreateWorkspace(
 
   const workspace = new Workspace({
     id,
-    isSSR: !isBrowser,
     providerCreators: typeof window === 'undefined' ? [] : providerCreators,
     blobStorages: [
       () => ({

packages/frontend/workspace/src/providers/sync/__tests__/engine.spec.ts

@@ -25,7 +25,7 @@ describe('SyncEngine', () => {
     {
       const workspace = new Workspace({
         id: 'test',
-        isSSR: true,
+
         schema,
       });
 
@@ -57,7 +57,7 @@ describe('SyncEngine', () => {
     {
       const workspace = new Workspace({
         id: 'test',
-        isSSR: true,
+
         schema,
       });
       const syncEngine = new SyncEngine(
@@ -76,7 +76,7 @@ describe('SyncEngine', () => {
     {
       const workspace = new Workspace({
         id: 'test',
-        isSSR: true,
+
         schema,
       });
       const syncEngine = new SyncEngine(
@@ -95,7 +95,7 @@ describe('SyncEngine', () => {
     {
       const workspace = new Workspace({
         id: 'test',
-        isSSR: true,
+
         schema,
       });
       const syncEngine = new SyncEngine(

packages/frontend/workspace/src/providers/sync/__tests__/peer.spec.ts

@@ -21,7 +21,7 @@ describe('SyncPeer', () => {
     {
       const workspace = new Workspace({
         id: 'test',
-        isSSR: true,
+
         schema,
       });
 
@@ -49,7 +49,7 @@ describe('SyncPeer', () => {
     {
       const workspace = new Workspace({
         id: 'test',
-        isSSR: true,
+
         schema,
       });
       const syncPeer = new SyncPeer(
@@ -67,7 +67,7 @@ describe('SyncPeer', () => {
   test('status', async () => {
     const workspace = new Workspace({
       id: 'test - status',
-      isSSR: true,
+
       schema,
     });
 

packages/frontend/workspace/src/providers/sync/engine.ts

@@ -3,6 +3,7 @@ import { Slot } from '@blocksuite/global/utils';
 import type { Doc } from 'yjs';
 
 import type { Storage } from '../storage';
+import { SharedPriorityTarget } from '../utils/async-queue';
 import { MANUALLY_STOP, SyncEngineStep } from './consts';
 import { SyncPeer, type SyncPeerStatus, SyncPeerStep } from './peer';
 
@@ -56,6 +57,8 @@ export class SyncEngine {
     this.onStatusChange.emit(s);
   }
 
+  priorityTarget = new SharedPriorityTarget();
+
   get status() {
     return this._status;
   }
@@ -110,7 +113,11 @@ export class SyncEngine {
     const cleanUp: (() => void)[] = [];
     try {
       // Step 1: start local sync peer
-      state.localPeer = new SyncPeer(this.rootDoc, this.local);
+      state.localPeer = new SyncPeer(
+        this.rootDoc,
+        this.local,
+        this.priorityTarget
+      );
 
       cleanUp.push(
         state.localPeer.onStatusChange.on(() => {
@@ -126,7 +133,7 @@ export class SyncEngine {
 
       // Step 3: start remote sync peer
       state.remotePeers = this.remotes.map(remote => {
-        const peer = new SyncPeer(this.rootDoc, remote);
+        const peer = new SyncPeer(this.rootDoc, remote, this.priorityTarget);
         cleanUp.push(
           peer.onStatusChange.on(() => {
             if (!signal.aborted)
@@ -237,4 +244,8 @@ export class SyncEngine {
       ]);
     }
   }
+
+  setPriorityRule(target: ((id: string) => boolean) | null) {
+    this.priorityTarget.priorityRule = target;
+  }
 }

packages/frontend/workspace/src/providers/sync/peer.ts

@@ -5,7 +5,7 @@ import type { Doc } from 'yjs';
 import { applyUpdate, encodeStateAsUpdate, encodeStateVector } from 'yjs';
 
 import { mergeUpdates, type Storage } from '../storage';
-import { AsyncQueue } from '../utils/async-queue';
+import { PriorityAsyncQueue, SharedPriorityTarget } from '../utils/async-queue';
 import { throwIfAborted } from '../utils/throw-if-aborted';
 import { MANUALLY_STOP } from './consts';
 
@@ -70,7 +70,8 @@ export class SyncPeer {
 
   constructor(
     private readonly rootDoc: Doc,
-    private readonly storage: Storage
+    private readonly storage: Storage,
+    private readonly priorityTarget = new SharedPriorityTarget()
   ) {
     this.logger.debug('peer start');
 
@@ -152,24 +153,24 @@ export class SyncPeer {
 
   private readonly state: {
     connectedDocs: Map<string, Doc>;
-    pushUpdatesQueue: AsyncQueue<{
-      docId: string;
+    pushUpdatesQueue: PriorityAsyncQueue<{
+      id: string;
       data: Uint8Array[];
     }>;
     pushingUpdate: boolean;
-    pullUpdatesQueue: AsyncQueue<{
-      docId: string;
+    pullUpdatesQueue: PriorityAsyncQueue<{
+      id: string;
       data: Uint8Array;
     }>;
     subdocLoading: boolean;
-    subdocsLoadQueue: AsyncQueue<Doc>;
+    subdocsLoadQueue: PriorityAsyncQueue<{ id: string; doc: Doc }>;
   } = {
     connectedDocs: new Map(),
-    pushUpdatesQueue: new AsyncQueue(),
+    pushUpdatesQueue: new PriorityAsyncQueue([], this.priorityTarget),
     pushingUpdate: false,
-    pullUpdatesQueue: new AsyncQueue(),
+    pullUpdatesQueue: new PriorityAsyncQueue([], this.priorityTarget),
     subdocLoading: false,
-    subdocsLoadQueue: new AsyncQueue(),
+    subdocsLoadQueue: new PriorityAsyncQueue([], this.priorityTarget),
   };
 
   initState() {
@@ -211,7 +212,10 @@ export class SyncPeer {
 
       // Step 2: load subdocs
       this.state.subdocsLoadQueue.push(
-        ...Array.from(this.rootDoc.getSubdocs())
+        ...Array.from(this.rootDoc.getSubdocs()).map(doc => ({
+          id: doc.guid,
+          doc,
+        }))
       );
       this.reportSyncStatus();
 
@@ -227,7 +231,7 @@ export class SyncPeer {
             );
             this.state.subdocLoading = true;
             this.reportSyncStatus();
-            await this.connectDoc(subdoc, abortInner.signal);
+            await this.connectDoc(subdoc.doc, abortInner.signal);
             this.state.subdocLoading = false;
             this.reportSyncStatus();
           }
@@ -235,7 +239,7 @@ export class SyncPeer {
         // pull updates
         (async () => {
           while (throwIfAborted(abortInner.signal)) {
-            const { docId, data } = await this.state.pullUpdatesQueue.next(
+            const { id, data } = await this.state.pullUpdatesQueue.next(
               abortInner.signal
             );
             // don't apply empty data or Uint8Array([0, 0])
@@ -245,7 +249,7 @@ export class SyncPeer {
                 (data.byteLength === 2 && data[0] === 0 && data[1] === 0)
               )
             ) {
-              const subdoc = this.state.connectedDocs.get(docId);
+              const subdoc = this.state.connectedDocs.get(id);
               if (subdoc) {
                 applyUpdate(subdoc, data, this.name);
               }
@@ -256,7 +260,7 @@ export class SyncPeer {
         // push updates
         (async () => {
           while (throwIfAborted(abortInner.signal)) {
-            const { docId, data } = await this.state.pushUpdatesQueue.next(
+            const { id, data } = await this.state.pushUpdatesQueue.next(
               abortInner.signal
             );
             this.state.pushingUpdate = true;
@@ -271,7 +275,7 @@ export class SyncPeer {
                 (merged.byteLength === 2 && merged[0] === 0 && merged[1] === 0)
               )
             ) {
-              await this.storage.push(docId, merged);
+              await this.storage.push(id, merged);
             }
 
             this.state.pushingUpdate = false;
@@ -299,7 +303,7 @@ export class SyncPeer {
 
     // diff root doc and in-storage, save updates to pendingUpdates
     this.state.pushUpdatesQueue.push({
-      docId: doc.guid,
+      id: doc.guid,
       data: [encodeStateAsUpdate(doc, inStorageState)],
     });
 
@@ -327,14 +331,12 @@ export class SyncPeer {
       return;
     }
 
-    const exist = this.state.pushUpdatesQueue.find(
-      ({ docId }) => docId === doc.guid
-    );
+    const exist = this.state.pushUpdatesQueue.find(({ id }) => id === doc.guid);
     if (exist) {
       exist.data.push(update);
     } else {
       this.state.pushUpdatesQueue.push({
-        docId: doc.guid,
+        id: doc.guid,
         data: [update],
       });
     }
@@ -351,20 +353,20 @@ export class SyncPeer {
     removed: Set<Doc>;
   }) => {
     for (const subdoc of added) {
-      this.state.subdocsLoadQueue.push(subdoc);
+      this.state.subdocsLoadQueue.push({ id: subdoc.guid, doc: subdoc });
     }
 
     for (const subdoc of removed) {
       this.disconnectDoc(subdoc);
-      this.state.subdocsLoadQueue.remove(doc => doc === subdoc);
+      this.state.subdocsLoadQueue.remove(doc => doc.doc === subdoc);
     }
     this.reportSyncStatus();
   };
 
   // handle updates from storage
-  handleStorageUpdates = (docId: string, data: Uint8Array) => {
+  handleStorageUpdates = (id: string, data: Uint8Array) => {
     this.state.pullUpdatesQueue.push({
-      docId,
+      id,
       data,
     });
     this.reportSyncStatus();

packages/frontend/workspace/src/providers/utils/async-queue.ts

@@ -12,8 +12,11 @@ export class AsyncQueue<T> {
     return this._queue.length;
   }
 
-  async next(abort?: AbortSignal): Promise<T> {
-    const update = this._queue.shift();
+  async next(
+    abort?: AbortSignal,
+    dequeue: (arr: T[]) => T | undefined = a => a.shift()
+  ): Promise<T> {
+    const update = dequeue(this._queue);
     if (update) {
       return update;
     } else {
@@ -35,7 +38,7 @@ export class AsyncQueue<T> {
         }),
       ]);
 
-      return this.next(abort);
+      return this.next(abort, dequeue);
     }
   }
 
@@ -64,3 +67,35 @@ export class AsyncQueue<T> {
     this._queue = [];
   }
 }
+
+export class PriorityAsyncQueue<
+  T extends { id: string },
+> extends AsyncQueue<T> {
+  constructor(
+    init: T[] = [],
+    public readonly priorityTarget: SharedPriorityTarget = new SharedPriorityTarget()
+  ) {
+    super(init);
+  }
+
+  override next(abort?: AbortSignal | undefined): Promise<T> {
+    return super.next(abort, arr => {
+      if (this.priorityTarget.priorityRule !== null) {
+        const index = arr.findIndex(
+          update => this.priorityTarget.priorityRule?.(update.id)
+        );
+        if (index !== -1) {
+          return arr.splice(index, 1)[0];
+        }
+      }
+      return arr.shift();
+    });
+  }
+}
+
+/**
+ * Shared priority target can be shared by multiple queues.
+ */
+export class SharedPriorityTarget {
+  public priorityRule: ((id: string) => boolean) | null = null;
+}