feat(nbstore): new doc sync engine (#8918)

packages/common/nbstore/src/utils/__tests__/priority-queue.spec.ts

@@ -0,0 +1,41 @@
+import { describe, expect, test } from 'vitest';
+
+import { PriorityQueue } from '../priority-queue';
+
+describe('Priority Queue', () => {
+  test('priority', () => {
+    const queue = new PriorityQueue();
+
+    queue.push('foo', 1);
+    queue.push('bar', 2);
+    queue.push('baz', 0);
+
+    expect(queue.pop()).toBe('bar');
+    expect(queue.pop()).toBe('foo');
+    expect(queue.pop()).toBe('baz');
+    expect(queue.pop()).toBe(null);
+
+    queue.push('B', 1);
+    queue.push('A', 1);
+
+    // if priority same then follow id binary order
+    expect(queue.pop()).toBe('B');
+    expect(queue.pop()).toBe('A');
+    expect(queue.pop()).toBe(null);
+
+    queue.push('A', 1);
+    queue.push('B', 2);
+    queue.push('A', 3); // same id but different priority, update the priority
+
+    expect(queue.pop()).toBe('A');
+    expect(queue.pop()).toBe('B');
+    expect(queue.pop()).toBe(null);
+
+    queue.push('A', 1);
+    queue.push('B', 2);
+    queue.remove('B');
+
+    expect(queue.pop()).toBe('A');
+    expect(queue.pop()).toBe(null);
+  });
+});

packages/common/nbstore/src/utils/async-priority-queue.ts

@@ -0,0 +1,43 @@
+import { PriorityQueue } from './priority-queue';
+
+export class AsyncPriorityQueue extends PriorityQueue {
+  private _resolveUpdate: (() => void) | null = null;
+  private _waitForUpdate: Promise<void> | null = null;
+
+  async asyncPop(abort?: AbortSignal): Promise<string> {
+    const update = this.pop();
+    if (update) {
+      return update;
+    } else {
+      if (!this._waitForUpdate) {
+        this._waitForUpdate = new Promise(resolve => {
+          this._resolveUpdate = resolve;
+        });
+      }
+
+      await Promise.race([
+        this._waitForUpdate,
+        new Promise((_, reject) => {
+          if (abort?.aborted) {
+            reject(abort?.reason);
+          }
+          abort?.addEventListener('abort', () => {
+            reject(abort.reason);
+          });
+        }),
+      ]);
+
+      return this.asyncPop(abort);
+    }
+  }
+
+  override push(id: string, priority: number = 0) {
+    super.push(id, priority);
+    if (this._resolveUpdate) {
+      const resolve = this._resolveUpdate;
+      this._resolveUpdate = null;
+      this._waitForUpdate = null;
+      resolve();
+    }
+  }
+}

packages/common/nbstore/src/utils/clock.ts

@@ -0,0 +1,38 @@
+export class ClockMap {
+  max: Date = new Date(0);
+  constructor(private readonly map: Map<string, Date>) {
+    for (const value of map.values()) {
+      if (value.getTime() > this.max.getTime()) {
+        this.max = value;
+      }
+    }
+  }
+
+  get(id: string): Date {
+    return this.map.get(id) ?? new Date(0);
+  }
+
+  set(id: string, value: Date) {
+    this.map.set(id, value);
+    if (value.getTime() > this.max.getTime()) {
+      this.max = value;
+    }
+  }
+
+  setIfBigger(id: string, value: Date) {
+    if (value.getTime() > this.get(id).getTime()) {
+      this.set(id, value);
+      return true;
+    }
+    return false;
+  }
+
+  clear() {
+    this.map.clear();
+    this.max = new Date(0);
+  }
+
+  keys() {
+    return Array.from(this.map.keys());
+  }
+}

packages/common/nbstore/src/utils/is-empty-update.ts

@@ -0,0 +1,13 @@
+/**
+ * Tell a binary is empty yjs binary or not.
+ *
+ * NOTE:
+ *   `[0, 0]` is empty yjs update binary
+ *   `[0]` is empty yjs state vector binary
+ */
+export function isEmptyUpdate(binary: Uint8Array) {
+  return (
+    binary.byteLength === 0 ||
+    (binary.byteLength === 2 && binary[0] === 0 && binary[1] === 0)
+  );
+}

packages/common/nbstore/src/utils/priority-queue.ts

@@ -0,0 +1,69 @@
+import { BinarySearchTree } from '@datastructures-js/binary-search-tree';
+
+export class PriorityQueue {
+  tree = new BinarySearchTree<{ id: string; priority: number }>((a, b) => {
+    return a.priority === b.priority
+      ? a.id === b.id
+        ? 0
+        : a.id > b.id
+          ? 1
+          : -1
+      : a.priority - b.priority;
+  });
+  priorityMap = new Map<string, number>();
+
+  push(id: string, priority: number = 0) {
+    const oldPriority = this.priorityMap.get(id);
+    if (oldPriority === priority) {
+      return;
+    }
+    if (oldPriority !== undefined) {
+      this.remove(id);
+    }
+    this.tree.insert({ id, priority });
+    this.priorityMap.set(id, priority);
+  }
+
+  pop() {
+    const node = this.tree.max();
+
+    if (!node) {
+      return null;
+    }
+
+    this.tree.removeNode(node);
+
+    const { id } = node.getValue();
+    this.priorityMap.delete(id);
+
+    return id;
+  }
+
+  remove(id: string, priority?: number) {
+    priority ??= this.priorityMap.get(id);
+    if (priority === undefined) {
+      return false;
+    }
+    const removed = this.tree.remove({ id, priority });
+    if (removed) {
+      this.priorityMap.delete(id);
+    }
+
+    return removed;
+  }
+
+  clear() {
+    this.tree.clear();
+    this.priorityMap.clear();
+  }
+
+  updatePriority(id: string, priority: number) {
+    if (this.remove(id)) {
+      this.push(id, priority);
+    }
+  }
+
+  get length() {
+    return this.tree.count;
+  }
+}

packages/common/nbstore/src/utils/throw-if-aborted.ts

@@ -0,0 +1,9 @@
+// because AbortSignal.throwIfAborted is not available in abortcontroller-polyfill
+export function throwIfAborted(abort?: AbortSignal) {
+  if (abort?.aborted) {
+    throw new Error(abort.reason);
+  }
+  return true;
+}
+
+export const MANUALLY_STOP = 'manually-stop';