Merge branch 'master'

Conflicts:
	.vscode/settings.json
	package.json
	packages/data-center/package.json
	packages/data-center/src/datacenter.ts
	pnpm-lock.yaml

.changeset/config.json

@@ -5,7 +5,7 @@
   "fixed": [],
   "linked": [],
   "access": "restricted",
-  "baseBranch": "feat/cloud-sync",
+  "baseBranch": "feat/master",
   "updateInternalDependencies": "patch",
   "ignore": []
 }

.github/workflows/changlog.yml

@@ -1,8 +1,7 @@
 name: Pathfinder changelog
 
 on:
-  push:
+  workflow_dispatch:
-    branches: [feat/cloud-sync, master]
 
 # Cancels all previous workflow runs for pull requests that have not completed.
 # See https://docs.github.com/en/actions/using-jobs/using-concurrency

.github/workflows/publish.yml

@@ -1,10 +1,7 @@
-name: Pathfinder Check
+name: Build Pathfinder Self-hosted
 
 on:
-  push:
+  workflow_dispatch:
-    branches: [feat/cloud-sync]
-  pull_request:
-    branches: [feat/cloud-sync]
 
 # Cancels all previous workflow runs for pull requests that have not completed.
 # See https://docs.github.com/en/actions/using-jobs/using-concurrency
@@ -15,69 +12,12 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
+  build-self-hosted:
-    name: Build on Pull Request
+    name: Build Community
-    if: github.ref != 'refs/heads/master'
+    if: github.ref == 'refs/heads/master'
     runs-on: self-hosted
     environment: development
 
-    steps:
-      - uses: actions/checkout@v2
-      - uses: pnpm/action-setup@v2
-        with:
-          version: 'latest'
-
-      - name: Use Node.js
-        uses: actions/setup-node@v2
-        with:
-          node-version: 18.x
-          registry-url: https://npm.pkg.github.com
-          scope: '@toeverything'
-          cache: 'pnpm'
-
-      - run: node scripts/module-resolve/ci.js
-
-      - name: Restore cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            .next/cache
-          # Generate a new cache whenever packages or source files change.
-          key: ${{ runner.os }}-nextjs-${{ hashFiles('**/pnpm-lock.yaml') }}-${{ hashFiles('**.[jt]s', '**.[jt]sx') }}
-          # If source files changed but packages didn't, rebuild from a prior cache.
-          restore-keys: |
-            ${{ runner.os }}-nextjs-${{ hashFiles('**/pnpm-lock.yaml') }}-
-
-      - name: Install dependencies
-        run: pnpm install --no-frozen-lockfile
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_GITHUB_AUTH_TOKEN }}
-
-      - name: Build
-        run: pnpm build
-        env:
-          NEXT_PUBLIC_FIREBASE_API_KEY: ${{ secrets.NEXT_PUBLIC_FIREBASE_API_KEY }}
-          NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN: ${{ secrets.NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN }}
-          NEXT_PUBLIC_FIREBASE_PROJECT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_PROJECT_ID }}
-          NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET: ${{ secrets.NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET }}
-          NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID }}
-          NEXT_PUBLIC_FIREBASE_APP_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_APP_ID }}
-          NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID }}
-
-      - name: Export
-        run: pnpm export
-
-      - name: Upload artifact
-        uses: actions/upload-artifact@v3
-        with:
-          path: ./packages/app/.next
-
-  lint:
-    name: Lint and E2E Test
-    runs-on: ubuntu-latest
-    environment: development
-    needs: build
-
     steps:
       - uses: actions/checkout@v2
       - uses: pnpm/action-setup@v2
@@ -106,18 +46,8 @@ jobs:
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_GITHUB_AUTH_TOKEN }}
 
-      - name: Download artifact
+      - name: Build
-        uses: actions/download-artifact@v3
+        run: pnpm build
-        with:
-          name: artifact
-          path: packages/app/.next/
-
-      - name: Lint & E2E Test
-        run: |
-          pnpm lint --max-warnings=0
-          PLAYWRIGHT_BROWSERS_PATH=0 npx playwright install chromium
-          PLAYWRIGHT_BROWSERS_PATH=0 pnpm test
-          PLAYWRIGHT_BROWSERS_PATH=0 pnpm test:dc
         env:
           NEXT_PUBLIC_FIREBASE_API_KEY: ${{ secrets.NEXT_PUBLIC_FIREBASE_API_KEY }}
           NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN: ${{ secrets.NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN }}
@@ -126,3 +56,59 @@ jobs:
           NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID }}
           NEXT_PUBLIC_FIREBASE_APP_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_APP_ID }}
           NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID }}
+
+      - name: Export
+        run: pnpm export
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          path: ./packages/app/out
+
+  publish-self-hosted:
+    name: Push Community Image
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    needs: build-self-hosted
+
+    permissions:
+      contents: read
+      packages: write
+
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: 'toeverything/affine-static'
+      IMAGE_TAG_LATEST: abbey-wood
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: artifact
+          path: packages/app/out/
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: ${{ env.IMAGE_TAG_LATEST }}
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          file: ./.github/deployment/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.vscode/settings.json

@@ -11,6 +11,8 @@
     "jwst",
     "testid",
     "octobase",
+    "selfhosted",
+    "testid",
     "schemars"
   ],
   "explorer.fileNesting.enabled": true,

package.json

@@ -6,8 +6,8 @@
   "license": "MPL-2.0",
   "scripts": {
     "dev": "cross-env NODE_ENV=development pnpm --filter=!@affine/app build && pnpm --filter @affine/app dev",
-    "dev:ac": "pnpm --filter=!@affine/app build && pnpm --filter @affine/app dev:ac",
+    "dev:ac": "pnpm --filter=!@affine/app build && cross-env NODE_API_SERVER=ac pnpm --filter @affine/app dev",
-    "dev:client": "cross-env NODE_ENV=development concurrently \"pnpm --filter=@affine/client-app dev:app\" \"pnpm dev\"",
+    "dev:local": "pnpm --filter=!@affine/app build && cross-env NODE_API_SERVER=local pnpm --filter @affine/app dev",
     "build": " pnpm --filter=!@affine/app build && pnpm --filter!=@affine/datacenter -r build",
     "build:client": " pnpm --filter=@affine/client-app build:app",
     "export": "pnpm --filter @affine/app export",

packages/app/next.config.js

@@ -9,6 +9,27 @@ const EDITOR_VERSION = enableDebugLocal
   ? 'local-version'
   : dependencies['@blocksuite/editor'];
 
+const profileTarget = {
+  ac: '100.85.73.88:12001',
+  dev: '100.85.73.88:12001',
+  local: '127.0.0.1:3000',
+};
+
+const getRedirectConfig = profile => {
+  const target = profileTarget[profile || 'dev'] || profileTarget['dev'];
+
+  return [
+    [
+      { source: '/api/:path*', destination: `http://${target}/api/:path*` },
+      {
+        source: '/collaboration/:path*',
+        destination: `http://${target}/collaboration/:path*`,
+      },
+    ],
+    target,
+  ];
+};
+
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   productionBrowserSourceMaps: true,
@@ -36,30 +57,12 @@ const nextConfig = {
   images: {
     unoptimized: true,
   },
-  // XXX not test yet
   rewrites: async () => {
-    if (process.env.NODE_API_SERVER === 'ac') {
+    const [profile, desc] = getRedirectConfig(process.env.NODE_API_SERVER);
-      let destinationAC = 'http://100.85.73.88:12001/api/:path*';
+    printer.info(
-      printer.info('API request proxy to [AC Server] ' + destinationAC);
+      `API request proxy to [${process.env.NODE_API_SERVER} Server]: ` + desc
-      return [
+    );
-        {
+    return profile;
-          source: '/api/:path*',
-          destination: destinationAC,
-        },
-      ];
-    } else {
-      let destinationStandard = 'http://100.77.180.48:11001/api/:path*';
-      printer.info(
-        'API request proxy to [Standard Server] ' + destinationStandard
-      );
-
-      return [
-        {
-          source: '/api/:path*',
-          destination: destinationStandard,
-        },
-      ];
-    }
   },
   basePath: process.env.BASE_PATH,
 };

packages/app/package.json

@@ -3,7 +3,6 @@
   "version": "0.3.1",
   "scripts": {
     "dev": "next dev -p 8080",
-    "dev:ac": "NODE_API_SERVER=ac next dev -p 8080",
     "build": "next build",
     "export": "next export",
     "start": "next start",
@@ -11,10 +10,10 @@
   },
   "dependencies": {
     "@affine/datacenter": "workspace:*",
-    "@blocksuite/blocks": "0.3.1",
+    "@blocksuite/blocks": "=0.3.1-20230106060050-1aad55d",
-    "@blocksuite/editor": "0.3.1",
+    "@blocksuite/editor": "=0.3.1-20230106060050-1aad55d",
     "@blocksuite/icons": "^2.0.2",
-    "@blocksuite/store": "0.3.1",
+    "@blocksuite/store": "=0.3.1-20230106060050-1aad55d",
     "@emotion/css": "^11.10.0",
     "@emotion/react": "^11.10.4",
     "@emotion/server": "^11.10.0",

packages/app/src/components/header/editor-header.tsx

@@ -22,14 +22,14 @@ export const EditorHeader = () => {
 
   useEffect(() => {
     onPropsUpdated(editor => {
-      setTitle(editor.model?.title || 'Untitled');
+      setTitle(editor.pageBlockModel?.title || 'Untitled');
     });
   }, [onPropsUpdated]);
 
   useEffect(() => {
     setTimeout(() => {
       // If first time in, need to wait for editor to be inserted into DOM
-      setTitle(editor?.model?.title || 'Untitled');
+      setTitle(editor?.pageBlockModel?.title || 'Untitled');
     }, 300);
   }, [editor]);
 

packages/app/src/components/import/index.tsx

@@ -31,9 +31,10 @@ export const ImportModal = ({ open, onClose }: ImportModalProps) => {
         setTimeout(() => {
           const editor = document.querySelector('editor-container');
           if (editor) {
-            const groupId = page.addBlock({ flavour: 'affine:group' }, pageId);
+            page.addBlock({ flavour: 'affine:surface' }, null);
+            const frameId = page.addBlock({ flavour: 'affine:frame' }, pageId);
             // TODO blocksuite should offer a method to import markdown from store
-            editor.clipboard.importMarkdown(template.source, `${groupId}`);
+            editor.clipboard.importMarkdown(template.source, `${frameId}`);
             page.resetHistory();
             editor.requestUpdate();
           }

packages/app/src/components/workspace-slider-bar/index.tsx

@@ -4,14 +4,14 @@ import {
   StyledArrowButton,
   StyledLink,
   StyledListItem,
-  StyledListItemForWorkspace,
+  // StyledListItemForWorkspace,
   StyledNewPageButton,
   StyledSliderBar,
   StyledSliderBarWrapper,
   StyledSubListItem,
 } from './style';
 import { Arrow } from './icons';
-import { WorkspaceSelector } from './WorkspaceSelector';
+// import { WorkspaceSelector } from './WorkspaceSelector';
 import Collapse from '@mui/material/Collapse';
 import {
   ArrowDownIcon,
@@ -109,9 +109,9 @@ export const WorkSpaceSliderBar = () => {
         </Tooltip>
 
         <StyledSliderBarWrapper data-testid="sliderBar">
-          <StyledListItemForWorkspace>
+          {/* <StyledListItemForWorkspace>
             <WorkspaceSelector />
-          </StyledListItemForWorkspace>
+          </StyledListItemForWorkspace> */}
           <StyledListItem
             data-testid="sliderBar-quickSearchButton"
             style={{ cursor: 'pointer' }}

packages/app/src/hooks/use-ensure-workspace.ts

@@ -1,8 +1,8 @@
 import { useState, useEffect } from 'react';
 import { useAppState } from '@/providers/app-state-provider';
 import { useRouter } from 'next/router';
-const defaultOutLineWorkspaceId =
+const defaultOutLineWorkspaceId = 'affine';
-  'local-first-' + '85b4ca0b9081421d903bbc2501ea280f';
+// 'local-first-' + '85b4ca0b9081421d903bbc2501ea280f';
 // It is a fully effective hook
 // Cause it not just ensure workspace loaded, but also have router change.
 export const useEnsureWorkspace = () => {

packages/app/src/hooks/use-props-updated.ts

@@ -17,7 +17,7 @@ export const usePropsUpdated: UsePropsUpdated = () => {
       return;
     }
     setTimeout(() => {
-      editor.model?.propsUpdated.on(() => {
+      editor.pageBlockModel?.propsUpdated.on(() => {
         callbackQueue.current.forEach(callback => {
           callback(editor);
         });
@@ -26,7 +26,7 @@ export const usePropsUpdated: UsePropsUpdated = () => {
 
     return () => {
       callbackQueue.current = [];
-      editor?.model?.propsUpdated.dispose();
+      editor?.pageBlockModel?.propsUpdated?.dispose();
     };
   }, [editor]);
 

packages/app/src/pages/playground/templates.tsx

@@ -43,12 +43,16 @@ const All = () => {
     if (page) {
       currentWorkspace?.setPageMeta(page.id, { title });
       if (page && page.root === null) {
-        setTimeout(() => {
+        setTimeout(async () => {
           const editor = document.querySelector('editor-container');
           if (editor) {
-            const groupId = page.addBlock({ flavour: 'affine:group' }, pageId);
+            page.addBlock({ flavour: 'affine:surface' }, null);
+            const frameId = page.addBlock({ flavour: 'affine:frame' }, pageId);
             // TODO blocksuite should offer a method to import markdown from store
-            editor.clipboard.importMarkdown(template.source, `${groupId}`);
+            await editor.clipboard.importMarkdown(
+              template.source,
+              `${frameId}`
+            );
             page.resetHistory();
             editor.requestUpdate();
           }

packages/app/src/pages/workspace/[workspaceId]/[pageId].tsx

@@ -1,10 +1,10 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 import {
-  useRef,
-  useEffect,
-  useState,
-  ReactElement,
   PropsWithChildren,
+  ReactElement,
+  useEffect,
+  useRef,
+  useState,
 } from 'react';
 import { styled } from '@/styles';
 import { EditorHeader } from '@/components/header';
@@ -16,9 +16,11 @@ import type { NextPageWithLayout } from '../..//_app';
 import WorkspaceLayout from '@/components/workspace-layout';
 import { useRouter } from 'next/router';
 import { usePageHelper } from '@/hooks/use-page-helper';
+
 const StyledEditorContainer = styled('div')(() => {
   return {
     height: 'calc(100vh - 60px)',
+    padding: '0 32px',
   };
 });
 
@@ -54,15 +56,20 @@ const Page: NextPageWithLayout = () => {
           flavour: 'affine:page',
           title,
         });
-        const groupId = currentPage!.addBlock(
+        currentPage!.addBlock({ flavour: 'affine:surface' }, null);
-          { flavour: 'affine:group' },
+        const frameId = currentPage!.addBlock(
+          { flavour: 'affine:frame' },
           pageId
         );
-        currentPage!.addBlock({ flavour: 'affine:group' }, pageId);
+        currentPage!.addBlock({ flavour: 'affine:frame' }, pageId);
         // If this is a first page in workspace, init an introduction markdown
         if (isFirstPage) {
-          editor.clipboard.importMarkdown(exampleMarkdown, `${groupId}`);
+          editor.clipboard
-          currentWorkspace!.setPageMeta(currentPage!.id, { title });
+            .importMarkdown(exampleMarkdown, `${frameId}`)
+            .then(() => {
+              currentWorkspace!.setPageMeta(currentPage!.id, { title });
+              currentPage!.resetHistory();
+            });
         }
         currentPage!.resetHistory();
       }

packages/app/src/providers/app-state-provider/dynamic-blocksuite.tsx

@@ -18,7 +18,7 @@ const DynamicBlocksuite = ({
     const openWorkspace: LoadWorkspaceHandler = async (workspaceId: string) => {
       if (workspaceId) {
         const dc = await getDataCenter();
-        return dc.load(workspaceId, { providerId: 'affine' });
+        return dc.load(workspaceId, { providerId: 'selfhosted' });
       } else {
         return null;
       }

packages/app/src/templates/Welcome-to-AFFiNE-Alpha-v2.0.md

@@ -10,6 +10,11 @@ Let us know what you think of this latest version.
 2. More complete Markdown support and improved keyboard shortcuts;
 3. New features such as dark mode; Switch between view styles using the ☀ and 🌙.
 4. Clean and modern UI/UX design.
+5. You can self-host locally with Docker.
+
+```basic
+docker run -d -v [YOUR_PATH]:/app/data -p 3000:3000 ghcr.io/toeverything/affine-self-hosted:alpha-abbey-wood
+```
 
 **Looking for Markdown syntax or keyboard shortcuts?**
 
@@ -23,15 +28,21 @@ Let us know what you think of this latest version.
 - Copy and paste **images** into your pages, resize them and add captions
 - Add horizontal line dividers to your text with `---` and `***`
 - Changes are saved **locally**, but we still recommend you export your data to avoid data loss
-- Insert code blocks with syntax highlighting support using ```
+- Insert code blocks with syntax highlighting support using `````
 
 ### Playground:
 
 [] Try a horizontal line: `---`
 
-[] What about a code block? ```
+[] What about a code block? `````
 
-        console.log('Hello world');
+```javascript
+console.log('Hello world');
+```
+
+[] Can you resize this image?
+
+![](https://cdn.affine.pro/694fdbab78e0da3ed7922eba7d506dcf12f57308e1904dd694f53eb2.jpg)
 
 **How about page management?**
 

packages/data-center/package.json

@@ -26,9 +26,9 @@
     "typescript": "^4.8.4"
   },
   "dependencies": {
-    "@blocksuite/blocks": "^0.3.1",
-    "@blocksuite/store": "^0.3.1",
     "@tauri-apps/api": "^1.2.0",
+    "@blocksuite/blocks": "=0.3.1-20230106060050-1aad55d",
+    "@blocksuite/store": "=0.3.1-20230106060050-1aad55d",
     "debug": "^4.3.4",
     "encoding": "^0.1.13",
     "firebase": "^9.15.0",

packages/data-center/src/datacenter.ts

@@ -4,8 +4,13 @@ import { Workspace, Signal } from '@blocksuite/store';
 
 import { getLogger } from './index.js';
 import { getApis, Apis } from './apis/index.js';
-import { AffineProvider, BaseProvider } from './provider/index.js';
+import {
-import { LocalProvider } from './provider/index.js';
+  AffineProvider,
+  BaseProvider,
+  LocalProvider,
+  SelfHostedProvider,
+} from './provider/index.js';
+
 import { getKVConfigure } from './store.js';
 import { TauriIPCProvider } from './provider/tauri-ipc/index.js';
 
@@ -48,6 +53,7 @@ export class DataCenter {
     if (typeof window !== 'undefined' && window.CLIENT_APP) {
       dc.addProvider(TauriIPCProvider);
     }
+    dc.addProvider(SelfHostedProvider);
 
     return dc;
   }

packages/data-center/src/provider/index.ts

@@ -20,3 +20,4 @@ export type { Apis, ConfigStore, DataCenterSignals, Workspace };
 export type { BaseProvider } from './base.js';
 export { AffineProvider } from './affine/index.js';
 export { LocalProvider } from './local/index.js';
+export { SelfHostedProvider } from './selfhosted/index.js';

packages/data-center/src/provider/selfhosted/index.ts

@@ -0,0 +1,63 @@
+import assert from 'assert';
+
+import { LocalProvider } from '../local/index.js';
+import { WebsocketProvider } from './sync.js';
+
+export class SelfHostedProvider extends LocalProvider {
+  static id = 'selfhosted';
+  private _ws?: WebsocketProvider;
+
+  constructor() {
+    super();
+  }
+
+  async destroy() {
+    this._ws?.disconnect();
+  }
+
+  async initData() {
+    const databases = await indexedDB.databases();
+    await super.initData(
+      // set locally to true if exists a same name db
+      databases
+        .map(db => db.name)
+        .filter(v => v)
+        .includes(this._workspace.room)
+    );
+
+    const workspace = this._workspace;
+    const doc = workspace.doc;
+
+    if (workspace.room) {
+      try {
+        // Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later
+        this._ws = new WebsocketProvider(this.host, workspace.room, doc);
+        await new Promise<void>((resolve, reject) => {
+          // TODO: synced will also be triggered on reconnection after losing sync
+          // There needs to be an event mechanism to emit the synchronization state to the upper layer
+          assert(this._ws);
+          this._ws.once('synced', () => resolve());
+          this._ws.once('lost-connection', () => resolve());
+          this._ws.once('connection-error', () => reject());
+        });
+        this._signals.listAdd.emit({
+          workspace: workspace.room,
+          provider: this.id,
+          locally: true,
+        });
+      } catch (e) {
+        this._logger('Failed to init cloud workspace', e);
+      }
+    }
+
+    // if after update, the space:meta is empty
+    // then we need to get map with doc
+    // just a workaround for yjs
+    doc.getMap('space:meta');
+  }
+
+  private get host() {
+    const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:';
+    return `${protocol}//${location.host}/collaboration/`;
+  }
+}

packages/data-center/src/provider/selfhosted/sync.js

@@ -0,0 +1,508 @@
+/* eslint-disable no-undef */
+/**
+ * @module provider/websocket
+ */
+
+/* eslint-env browser */
+
+// import * as Y from 'yjs'; // eslint-disable-line
+import * as bc from 'lib0/broadcastchannel';
+import * as time from 'lib0/time';
+import * as encoding from 'lib0/encoding';
+import * as decoding from 'lib0/decoding';
+import * as syncProtocol from 'y-protocols/sync';
+import * as authProtocol from 'y-protocols/auth';
+import * as awarenessProtocol from 'y-protocols/awareness';
+import { Observable } from 'lib0/observable';
+import * as math from 'lib0/math';
+import * as url from 'lib0/url';
+
+export const messageSync = 0;
+export const messageQueryAwareness = 3;
+export const messageAwareness = 1;
+export const messageAuth = 2;
+
+/**
+ *                       encoder,          decoder,          provider,          emitSynced, messageType
+ * @type {Array<function(encoding.Encoder, decoding.Decoder, WebsocketProvider, boolean,    number):void>}
+ */
+const messageHandlers = [];
+
+messageHandlers[messageSync] = (
+  encoder,
+  decoder,
+  provider,
+  emitSynced,
+  _messageType
+) => {
+  encoding.writeVarUint(encoder, messageSync);
+  const syncMessageType = syncProtocol.readSyncMessage(
+    decoder,
+    encoder,
+    provider.doc,
+    provider
+  );
+  if (
+    emitSynced &&
+    syncMessageType === syncProtocol.messageYjsSyncStep2 &&
+    !provider.synced
+  ) {
+    provider.synced = true;
+  }
+};
+
+messageHandlers[messageQueryAwareness] = (
+  encoder,
+  _decoder,
+  provider,
+  _emitSynced,
+  _messageType
+) => {
+  encoding.writeVarUint(encoder, messageAwareness);
+  encoding.writeVarUint8Array(
+    encoder,
+    awarenessProtocol.encodeAwarenessUpdate(
+      provider.awareness,
+      Array.from(provider.awareness.getStates().keys())
+    )
+  );
+};
+
+messageHandlers[messageAwareness] = (
+  _encoder,
+  decoder,
+  provider,
+  _emitSynced,
+  _messageType
+) => {
+  awarenessProtocol.applyAwarenessUpdate(
+    provider.awareness,
+    decoding.readVarUint8Array(decoder),
+    provider
+  );
+};
+
+messageHandlers[messageAuth] = (
+  _encoder,
+  decoder,
+  provider,
+  _emitSynced,
+  _messageType
+) => {
+  authProtocol.readAuthMessage(decoder, provider.doc, (_ydoc, reason) =>
+    permissionDeniedHandler(provider, reason)
+  );
+};
+
+// @todo - this should depend on awareness.outdatedTime
+const messageReconnectTimeout = 30000;
+
+/**
+ * @param {WebsocketProvider} provider
+ * @param {string} reason
+ */
+const permissionDeniedHandler = (provider, reason) =>
+  console.warn(`Permission denied to access ${provider.url}.\n${reason}`);
+
+/**
+ * @param {WebsocketProvider} provider
+ * @param {Uint8Array} buf
+ * @param {boolean} emitSynced
+ * @return {encoding.Encoder}
+ */
+const readMessage = (provider, buf, emitSynced) => {
+  const decoder = decoding.createDecoder(buf);
+  const encoder = encoding.createEncoder();
+  const messageType = decoding.readVarUint(decoder);
+  const messageHandler = provider.messageHandlers[messageType];
+  if (/** @type {any} */ (messageHandler)) {
+    messageHandler(encoder, decoder, provider, emitSynced, messageType);
+  } else {
+    console.error('Unable to compute message');
+  }
+  return encoder;
+};
+
+/**
+ * @param {WebsocketProvider} provider
+ */
+const setupWS = provider => {
+  if (provider.shouldConnect && provider.ws === null) {
+    const websocket = new provider._WS(provider.url, 'AFFiNE');
+    websocket.binaryType = 'arraybuffer';
+    provider.ws = websocket;
+    provider.wsconnecting = true;
+    provider.wsconnected = false;
+    provider.synced = false;
+
+    websocket.onmessage = event => {
+      provider.wsLastMessageReceived = time.getUnixTime();
+      const encoder = readMessage(provider, new Uint8Array(event.data), true);
+      if (encoding.length(encoder) > 1) {
+        websocket.send(encoding.toUint8Array(encoder));
+      }
+    };
+    websocket.onerror = event => {
+      provider.emit('connection-error', [event, provider]);
+    };
+    websocket.onclose = event => {
+      provider.emit('connection-close', [event, provider]);
+      provider.ws = null;
+      provider.wsconnecting = false;
+      if (provider.wsconnected) {
+        provider.wsconnected = false;
+        provider.synced = false;
+        // update awareness (all users except local left)
+        awarenessProtocol.removeAwarenessStates(
+          provider.awareness,
+          Array.from(provider.awareness.getStates().keys()).filter(
+            client => client !== provider.doc.clientID
+          ),
+          provider
+        );
+        provider.emit('status', [
+          {
+            status: 'disconnected',
+          },
+        ]);
+      } else {
+        provider.wsUnsuccessfulReconnects++;
+      }
+      // Start with no reconnect timeout and increase timeout by
+      // using exponential backoff starting with 100ms
+      setTimeout(
+        setupWS,
+        math.min(
+          math.pow(2, provider.wsUnsuccessfulReconnects) * 100,
+          provider.maxBackoffTime
+        ),
+        provider
+      );
+    };
+    websocket.onopen = () => {
+      provider.wsLastMessageReceived = time.getUnixTime();
+      provider.wsconnecting = false;
+      provider.wsconnected = true;
+      provider.wsUnsuccessfulReconnects = 0;
+      provider.emit('status', [
+        {
+          status: 'connected',
+        },
+      ]);
+      // always send sync step 1 when connected
+      const encoder = encoding.createEncoder();
+      encoding.writeVarUint(encoder, messageSync);
+      syncProtocol.writeSyncStep1(encoder, provider.doc);
+      websocket.send(encoding.toUint8Array(encoder));
+      // broadcast local awareness state
+      if (provider.awareness.getLocalState() !== null) {
+        const encoderAwarenessState = encoding.createEncoder();
+        encoding.writeVarUint(encoderAwarenessState, messageAwareness);
+        encoding.writeVarUint8Array(
+          encoderAwarenessState,
+          awarenessProtocol.encodeAwarenessUpdate(provider.awareness, [
+            provider.doc.clientID,
+          ])
+        );
+        websocket.send(encoding.toUint8Array(encoderAwarenessState));
+      }
+    };
+
+    provider.emit('status', [
+      {
+        status: 'connecting',
+      },
+    ]);
+  }
+};
+
+/**
+ * @param {WebsocketProvider} provider
+ * @param {ArrayBuffer} buf
+ */
+const broadcastMessage = (provider, buf) => {
+  if (provider.wsconnected) {
+    /** @type {WebSocket} */ (provider.ws).send(buf);
+  }
+  if (provider.bcconnected) {
+    bc.publish(provider.bcChannel, buf, provider);
+  }
+};
+
+/**
+ * Websocket Provider for Yjs. Creates a websocket connection to sync the shared document.
+ * The document name is attached to the provided url. I.e. the following example
+ * creates a websocket connection to http://localhost:1234/my-document-name
+ *
+ * @example
+ *   import * as Y from 'yjs'
+ *   import { WebsocketProvider } from 'y-websocket'
+ *   const doc = new Y.Doc()
+ *   const provider = new WebsocketProvider('http://localhost:1234', 'my-document-name', doc)
+ *
+ * @extends {Observable<string>}
+ */
+export class WebsocketProvider extends Observable {
+  /**
+   * @param {string} serverUrl
+   * @param {string} roomname
+   * @param {Y.Doc} doc
+   * @param {object} [opts]
+   * @param {boolean} [opts.connect]
+   * @param {awarenessProtocol.Awareness} [opts.awareness]
+   * @param {Object<string,string>} [opts.params]
+   * @param {typeof WebSocket} [opts.WebSocketPolyfill] Optionall provide a WebSocket polyfill
+   * @param {number} [opts.resyncInterval] Request server state every `resyncInterval` milliseconds
+   * @param {number} [opts.maxBackoffTime] Maximum amount of time to wait before trying to reconnect (we try to reconnect using exponential backoff)
+   * @param {boolean} [opts.disableBc] Disable cross-tab BroadcastChannel communication
+   */
+  constructor(
+    serverUrl,
+    roomname,
+    doc,
+    {
+      connect = true,
+      awareness = new awarenessProtocol.Awareness(doc),
+      params = {},
+      WebSocketPolyfill = WebSocket,
+      resyncInterval = -1,
+      maxBackoffTime = 2500,
+      disableBc = false,
+    } = {}
+  ) {
+    super();
+    // ensure that url is always ends with /
+    while (serverUrl[serverUrl.length - 1] === '/') {
+      serverUrl = serverUrl.slice(0, serverUrl.length - 1);
+    }
+    const encodedParams = url.encodeQueryParams(params);
+    this.maxBackoffTime = maxBackoffTime;
+    this.bcChannel = serverUrl + '/' + roomname;
+    this.url =
+      serverUrl +
+      '/' +
+      roomname +
+      (encodedParams.length === 0 ? '' : '?' + encodedParams);
+    this.roomname = roomname;
+    this.doc = doc;
+    this._WS = WebSocketPolyfill;
+    this.awareness = awareness;
+    this.wsconnected = false;
+    this.wsconnecting = false;
+    this.bcconnected = false;
+    this.disableBc = disableBc;
+    this.wsUnsuccessfulReconnects = 0;
+    this.messageHandlers = messageHandlers.slice();
+    /**
+     * @type {boolean}
+     */
+    this._synced = false;
+    /**
+     * @type {WebSocket?}
+     */
+    this.ws = null;
+    this.wsLastMessageReceived = 0;
+    /**
+     * Whether to connect to other peers or not
+     * @type {boolean}
+     */
+    this.shouldConnect = connect;
+
+    /**
+     * @type {number}
+     */
+    this._resyncInterval = 0;
+    if (resyncInterval > 0) {
+      this._resyncInterval = /** @type {any} */ (
+        setInterval(() => {
+          if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            // resend sync step 1
+            const encoder = encoding.createEncoder();
+            encoding.writeVarUint(encoder, messageSync);
+            syncProtocol.writeSyncStep1(encoder, doc);
+            this.ws.send(encoding.toUint8Array(encoder));
+          }
+        }, resyncInterval)
+      );
+    }
+
+    /**
+     * @param {ArrayBuffer} data
+     * @param {any} origin
+     */
+    this._bcSubscriber = (data, origin) => {
+      if (origin !== this) {
+        const encoder = readMessage(this, new Uint8Array(data), false);
+        if (encoding.length(encoder) > 1) {
+          bc.publish(this.bcChannel, encoding.toUint8Array(encoder), this);
+        }
+      }
+    };
+    /**
+     * Listens to Yjs updates and sends them to remote peers (ws and broadcastchannel)
+     * @param {Uint8Array} update
+     * @param {any} origin
+     */
+    this._updateHandler = (update, origin) => {
+      if (origin !== this) {
+        const encoder = encoding.createEncoder();
+        encoding.writeVarUint(encoder, messageSync);
+        syncProtocol.writeUpdate(encoder, update);
+        broadcastMessage(this, encoding.toUint8Array(encoder));
+      }
+    };
+    this.doc.on('update', this._updateHandler);
+    /**
+     * @param {any} changed
+     * @param {any} _origin
+     */
+    this._awarenessUpdateHandler = ({ added, updated, removed }, _origin) => {
+      const changedClients = added.concat(updated).concat(removed);
+      const encoder = encoding.createEncoder();
+      encoding.writeVarUint(encoder, messageAwareness);
+      encoding.writeVarUint8Array(
+        encoder,
+        awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients)
+      );
+      broadcastMessage(this, encoding.toUint8Array(encoder));
+    };
+    this._unloadHandler = () => {
+      awarenessProtocol.removeAwarenessStates(
+        this.awareness,
+        [doc.clientID],
+        'window unload'
+      );
+    };
+    if (typeof window !== 'undefined') {
+      window.addEventListener('unload', this._unloadHandler);
+    } else if (typeof process !== 'undefined') {
+      process.on('exit', this._unloadHandler);
+    }
+    awareness.on('update', this._awarenessUpdateHandler);
+    this._checkInterval = /** @type {any} */ (
+      setInterval(() => {
+        if (
+          this.wsconnected &&
+          messageReconnectTimeout <
+            time.getUnixTime() - this.wsLastMessageReceived
+        ) {
+          // no message received in a long time - not even your own awareness
+          // updates (which are updated every 15 seconds)
+          /** @type {WebSocket} */ (this.ws).close();
+        }
+      }, messageReconnectTimeout / 10)
+    );
+    if (connect) {
+      this.connect();
+    }
+  }
+
+  /**
+   * @type {boolean}
+   */
+  get synced() {
+    return this._synced;
+  }
+
+  set synced(state) {
+    if (this._synced !== state) {
+      this._synced = state;
+      this.emit('synced', [state]);
+      this.emit('sync', [state]);
+    }
+  }
+
+  destroy() {
+    if (this._resyncInterval !== 0) {
+      clearInterval(this._resyncInterval);
+    }
+    clearInterval(this._checkInterval);
+    this.disconnect();
+    if (typeof window !== 'undefined') {
+      window.removeEventListener('unload', this._unloadHandler);
+    } else if (typeof process !== 'undefined') {
+      process.off('exit', this._unloadHandler);
+    }
+    this.awareness.off('update', this._awarenessUpdateHandler);
+    this.doc.off('update', this._updateHandler);
+    super.destroy();
+  }
+
+  connectBc() {
+    if (this.disableBc) {
+      return;
+    }
+    if (!this.bcconnected) {
+      bc.subscribe(this.bcChannel, this._bcSubscriber);
+      this.bcconnected = true;
+    }
+    // send sync step1 to bc
+    // write sync step 1
+    const encoderSync = encoding.createEncoder();
+    encoding.writeVarUint(encoderSync, messageSync);
+    syncProtocol.writeSyncStep1(encoderSync, this.doc);
+    bc.publish(this.bcChannel, encoding.toUint8Array(encoderSync), this);
+    // broadcast local state
+    const encoderState = encoding.createEncoder();
+    encoding.writeVarUint(encoderState, messageSync);
+    syncProtocol.writeSyncStep2(encoderState, this.doc);
+    bc.publish(this.bcChannel, encoding.toUint8Array(encoderState), this);
+    // write queryAwareness
+    const encoderAwarenessQuery = encoding.createEncoder();
+    encoding.writeVarUint(encoderAwarenessQuery, messageQueryAwareness);
+    bc.publish(
+      this.bcChannel,
+      encoding.toUint8Array(encoderAwarenessQuery),
+      this
+    );
+    // broadcast local awareness state
+    const encoderAwarenessState = encoding.createEncoder();
+    encoding.writeVarUint(encoderAwarenessState, messageAwareness);
+    encoding.writeVarUint8Array(
+      encoderAwarenessState,
+      awarenessProtocol.encodeAwarenessUpdate(this.awareness, [
+        this.doc.clientID,
+      ])
+    );
+    bc.publish(
+      this.bcChannel,
+      encoding.toUint8Array(encoderAwarenessState),
+      this
+    );
+  }
+
+  disconnectBc() {
+    // broadcast message with local awareness state set to null (indicating disconnect)
+    const encoder = encoding.createEncoder();
+    encoding.writeVarUint(encoder, messageAwareness);
+    encoding.writeVarUint8Array(
+      encoder,
+      awarenessProtocol.encodeAwarenessUpdate(
+        this.awareness,
+        [this.doc.clientID],
+        new Map()
+      )
+    );
+    broadcastMessage(this, encoding.toUint8Array(encoder));
+    if (this.bcconnected) {
+      bc.unsubscribe(this.bcChannel, this._bcSubscriber);
+      this.bcconnected = false;
+    }
+  }
+
+  disconnect() {
+    this.shouldConnect = false;
+    this.disconnectBc();
+    if (this.ws !== null) {
+      this.ws.close();
+    }
+  }
+
+  connect() {
+    this.shouldConnect = true;
+    if (!this.wsconnected && this.ws === null) {
+      setupWS(this);
+      this.connectBc();
+    }
+  }
+}

pnpm-lock.yaml

@@ -148,10 +148,10 @@ importers:
   packages/app:
     specifiers:
       '@affine/datacenter': workspace:*
-      '@blocksuite/blocks': 0.3.1
+      '@blocksuite/blocks': '=0.3.1-20230106060050-1aad55d'
-      '@blocksuite/editor': 0.3.1
+      '@blocksuite/editor': '=0.3.1-20230106060050-1aad55d'
       '@blocksuite/icons': ^2.0.2
-      '@blocksuite/store': 0.3.1
+      '@blocksuite/store': '=0.3.1-20230106060050-1aad55d'
       '@emotion/css': ^11.10.0
       '@emotion/react': ^11.10.4
       '@emotion/server': ^11.10.0
@@ -190,10 +190,10 @@ importers:
       yjs: ^13.5.44
     dependencies:
       '@affine/datacenter': link:../data-center
-      '@blocksuite/blocks': 0.3.1_yjs@13.5.44
+      '@blocksuite/blocks': 0.3.1-20230106060050-1aad55d_yjs@13.5.44
-      '@blocksuite/editor': 0.3.1_yjs@13.5.44
+      '@blocksuite/editor': 0.3.1-20230106060050-1aad55d_yjs@13.5.44
       '@blocksuite/icons': 2.0.4_w5j4k42lgipnm43s3brx6h3c34
-      '@blocksuite/store': 0.3.1_yjs@13.5.44
+      '@blocksuite/store': 0.3.1-20230106060050-1aad55d_yjs@13.5.44
       '@emotion/css': 11.10.0
       '@emotion/react': 11.10.4_w5j4k42lgipnm43s3brx6h3c34
       '@emotion/server': 11.10.0_@emotion+css@11.10.0
@@ -234,8 +234,8 @@ importers:
 
   packages/data-center:
     specifiers:
-      '@blocksuite/blocks': ^0.3.1
+      '@blocksuite/blocks': '=0.3.1-20230106060050-1aad55d'
-      '@blocksuite/store': ^0.3.1
+      '@blocksuite/store': '=0.3.1-20230106060050-1aad55d'
       '@playwright/test': ^1.29.1
       '@tauri-apps/api': ^1.2.0
       '@types/debug': ^4.1.7
@@ -252,9 +252,9 @@ importers:
       y-protocols: ^1.0.5
       yjs: ^13.5.44
     dependencies:
-      '@blocksuite/blocks': 0.3.1_yjs@13.5.44
-      '@blocksuite/store': 0.3.1_yjs@13.5.44
       '@tauri-apps/api': 1.2.0
+      '@blocksuite/blocks': 0.3.1-20230106060050-1aad55d_yjs@13.5.44
+      '@blocksuite/store': 0.3.1-20230106060050-1aad55d_yjs@13.5.44
       debug: 4.3.4
       encoding: 0.1.13
       firebase: 9.15.0_encoding@0.1.13
@@ -1610,10 +1610,11 @@ packages:
     resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==}
     dev: true
 
-  /@blocksuite/blocks/0.3.1_yjs@13.5.44:
+  /@blocksuite/blocks/0.3.1-20230106060050-1aad55d_yjs@13.5.44:
-    resolution: {integrity: sha512-b0dGz2MG4yIgngJPRumaMY58wAsd2FEVSZl3tpCXlagK9I0HD165Bq70PxcaRHVjBSV1Gf29ZVHUF6BVTYogPw==}
+    resolution: {integrity: sha512-qRNXmhjw+GAGsV1mI2XXPxYTlHfsFHv9ttTCNQ6IIcxvc5Hh6lWmdwVibxvlpYUkgEc1zv3/GxOEsR/ngpZXzQ==}
     dependencies:
-      '@blocksuite/store': 0.3.1_yjs@13.5.44
+      '@blocksuite/phasor': 0.3.1_yjs@13.5.44
+      '@blocksuite/store': 0.3.1-20230106060050-1aad55d_yjs@13.5.44
       '@tldraw/intersect': 1.8.0
       autosize: 5.0.2
       highlight.js: 11.7.0
@@ -1629,11 +1630,11 @@ packages:
       - yjs
     dev: false
 
-  /@blocksuite/editor/0.3.1_yjs@13.5.44:
+  /@blocksuite/editor/0.3.1-20230106060050-1aad55d_yjs@13.5.44:
-    resolution: {integrity: sha512-ycKcyvPW6R8R2GZOFneGH1xNi5gJBx5WtWjW9YwcQslFzXVWMCCBips1Bud2uL4kkbWQoodyua6k2vsXxGAKLw==}
+    resolution: {integrity: sha512-wSlAF9XVxIkHFJ1qCzn7oQ/gwXybFYMrzRl35UTJV509D+DuWZefRZWvpdIDCOUJ24uQscr1HxwsON11ltfWgA==}
     dependencies:
-      '@blocksuite/blocks': 0.3.1_yjs@13.5.44
+      '@blocksuite/blocks': 0.3.1-20230106060050-1aad55d_yjs@13.5.44
-      '@blocksuite/store': 0.3.1_yjs@13.5.44
+      '@blocksuite/store': 0.3.1-20230106060050-1aad55d_yjs@13.5.44
       lit: 2.5.0
       marked: 4.2.5
       turndown: 7.1.1
@@ -1654,8 +1655,16 @@ packages:
       react: 18.2.0
     dev: false
 
-  /@blocksuite/store/0.3.1_yjs@13.5.44:
+  /@blocksuite/phasor/0.3.1_yjs@13.5.44:
-    resolution: {integrity: sha512-kynVTDfNCSChz2JI2rtGHxRIV2YrLzvAgVajcbfDVCuXKG0siBoEjLasG1a0kvevbvW/FabrNAj+xaIplklioA==}
+    resolution: {integrity: sha512-aJmAQn2qoF6HxFZWgq7xa/pWVyzg3MmD6dynIHAKdfN7rBdKk3PNA+lRX919QkD2e270N/zgHEGFFQI1Nj5xrA==}
+    peerDependencies:
+      yjs: ^13
+    dependencies:
+      yjs: 13.5.44
+    dev: false
+
+  /@blocksuite/store/0.3.1-20230106060050-1aad55d_yjs@13.5.44:
+    resolution: {integrity: sha512-dRy+YzlWMwiYq0Im9NogK/NTkV+NKK+lgejYq56m6nH2m16/G9AMODqP0oQy/XeYFevUpL9i9RdV0rHsJ2gc0Q==}
     peerDependencies:
       yjs: ^13
     dependencies:

tests/quick-search.spec.ts

@@ -7,10 +7,13 @@ loadPage();
 const openQuickSearchByShortcut = async (page: Page) =>
   await withCtrlOrMeta(page, () => page.keyboard.press('k', { delay: 50 }));
 
-async function assertTitleTexts(page: Page, texts: string[]) {
+async function assertTitleTexts(page: Page, texts: string) {
-  const actual = await page
+  const actual = await page.evaluate(() => {
-    .locator('.affine-default-page-block-title')
+    const titleElement = <HTMLTextAreaElement>(
-    .allTextContents();
+      document.querySelector('.affine-default-page-block-title')
+    );
+    return titleElement.value;
+  });
   expect(actual).toEqual(texts);
 }
 async function assertResultList(page: Page, texts: string[]) {
@@ -55,7 +58,7 @@ test.describe('Add new page in quick search', () => {
     const addNewPage = page.locator('[data-testid=quickSearch-addNewPage]');
     await addNewPage.click();
     await page.waitForTimeout(200);
-    await assertTitleTexts(page, ['']);
+    await assertTitleTexts(page, '');
   });
 
   test('Create a new page with keyword', async ({ page }) => {
@@ -65,7 +68,7 @@ test.describe('Add new page in quick search', () => {
     const addNewPage = page.locator('[data-testid=quickSearch-addNewPage]');
     await addNewPage.click();
     await page.waitForTimeout(200);
-    await assertTitleTexts(page, ['test123456']);
+    await assertTitleTexts(page, 'test123456');
   });
 });
 
@@ -81,6 +84,6 @@ test.describe('Search and select', () => {
     await page.keyboard.insertText('test123456');
     await assertResultList(page, ['test123456']);
     await page.keyboard.press('Enter', { delay: 50 });
-    await assertTitleTexts(page, ['test123456']);
+    await assertTitleTexts(page, 'test123456');
   });
 });