diff --git a/.changeset/config.json b/.changeset/config.json index 1892e1a308..6ce3b1a2ce 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -5,7 +5,7 @@ "fixed": [], "linked": [], "access": "restricted", - "baseBranch": "feat/cloud-sync", + "baseBranch": "feat/master", "updateInternalDependencies": "patch", "ignore": [] } diff --git a/.github/workflows/changlog.yml b/.github/workflows/changlog.yml index 6a4ced3e20..d91559cf50 100644 --- a/.github/workflows/changlog.yml +++ b/.github/workflows/changlog.yml @@ -1,8 +1,7 @@ name: Pathfinder changelog on: - push: - branches: [feat/cloud-sync, master] + workflow_dispatch: # Cancels all previous workflow runs for pull requests that have not completed. # See https://docs.github.com/en/actions/using-jobs/using-concurrency diff --git a/.github/workflows/temp_test.yml b/.github/workflows/publish.yml similarity index 52% rename from .github/workflows/temp_test.yml rename to .github/workflows/publish.yml index a1fa93bdf0..09ec455e77 100644 --- a/.github/workflows/temp_test.yml +++ b/.github/workflows/publish.yml @@ -1,10 +1,7 @@ -name: Pathfinder Check +name: Build Pathfinder Self-hosted on: - push: - branches: [feat/cloud-sync] - pull_request: - branches: [feat/cloud-sync] + workflow_dispatch: # Cancels all previous workflow runs for pull requests that have not completed. # See https://docs.github.com/en/actions/using-jobs/using-concurrency @@ -15,69 +12,12 @@ concurrency: cancel-in-progress: true jobs: - build: - name: Build on Pull Request - if: github.ref != 'refs/heads/master' + build-self-hosted: + name: Build Community + if: github.ref == 'refs/heads/master' runs-on: self-hosted environment: development - steps: - - uses: actions/checkout@v2 - - uses: pnpm/action-setup@v2 - with: - version: 'latest' - - - name: Use Node.js - uses: actions/setup-node@v2 - with: - node-version: 18.x - registry-url: https://npm.pkg.github.com - scope: '@toeverything' - cache: 'pnpm' - - - run: node scripts/module-resolve/ci.js - - - name: Restore cache - uses: actions/cache@v3 - with: - path: | - .next/cache - # Generate a new cache whenever packages or source files change. - key: ${{ runner.os }}-nextjs-${{ hashFiles('**/pnpm-lock.yaml') }}-${{ hashFiles('**.[jt]s', '**.[jt]sx') }} - # If source files changed but packages didn't, rebuild from a prior cache. - restore-keys: | - ${{ runner.os }}-nextjs-${{ hashFiles('**/pnpm-lock.yaml') }}- - - - name: Install dependencies - run: pnpm install --no-frozen-lockfile - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_GITHUB_AUTH_TOKEN }} - - - name: Build - run: pnpm build - env: - NEXT_PUBLIC_FIREBASE_API_KEY: ${{ secrets.NEXT_PUBLIC_FIREBASE_API_KEY }} - NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN: ${{ secrets.NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN }} - NEXT_PUBLIC_FIREBASE_PROJECT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_PROJECT_ID }} - NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET: ${{ secrets.NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET }} - NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID }} - NEXT_PUBLIC_FIREBASE_APP_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_APP_ID }} - NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID }} - - - name: Export - run: pnpm export - - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - path: ./packages/app/.next - - lint: - name: Lint and E2E Test - runs-on: ubuntu-latest - environment: development - needs: build - steps: - uses: actions/checkout@v2 - uses: pnpm/action-setup@v2 @@ -106,18 +46,8 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_GITHUB_AUTH_TOKEN }} - - name: Download artifact - uses: actions/download-artifact@v3 - with: - name: artifact - path: packages/app/.next/ - - - name: Lint & E2E Test - run: | - pnpm lint --max-warnings=0 - PLAYWRIGHT_BROWSERS_PATH=0 npx playwright install chromium - PLAYWRIGHT_BROWSERS_PATH=0 pnpm test - PLAYWRIGHT_BROWSERS_PATH=0 pnpm test:dc + - name: Build + run: pnpm build env: NEXT_PUBLIC_FIREBASE_API_KEY: ${{ secrets.NEXT_PUBLIC_FIREBASE_API_KEY }} NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN: ${{ secrets.NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN }} @@ -126,3 +56,59 @@ jobs: NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID }} NEXT_PUBLIC_FIREBASE_APP_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_APP_ID }} NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID }} + + - name: Export + run: pnpm export + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + path: ./packages/app/out + + publish-self-hosted: + name: Push Community Image + if: github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + needs: build-self-hosted + + permissions: + contents: read + packages: write + + env: + REGISTRY: ghcr.io + IMAGE_NAME: 'toeverything/affine-static' + IMAGE_TAG_LATEST: abbey-wood + + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: artifact + path: packages/app/out/ + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: ${{ env.IMAGE_TAG_LATEST }} + + - name: Build Docker image + uses: docker/build-push-action@v3 + with: + context: . + push: true + file: ./.github/deployment/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.vscode/settings.json b/.vscode/settings.json index 2d75a23fb6..83567e87ef 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,6 +11,8 @@ "jwst", "testid", "octobase", + "selfhosted", + "testid", "schemars" ], "explorer.fileNesting.enabled": true, diff --git a/package.json b/package.json index 3db88747ff..1a2d878324 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,8 @@ "license": "MPL-2.0", "scripts": { "dev": "cross-env NODE_ENV=development pnpm --filter=!@affine/app build && pnpm --filter @affine/app dev", - "dev:ac": "pnpm --filter=!@affine/app build && pnpm --filter @affine/app dev:ac", - "dev:client": "cross-env NODE_ENV=development concurrently \"pnpm --filter=@affine/client-app dev:app\" \"pnpm dev\"", + "dev:ac": "pnpm --filter=!@affine/app build && cross-env NODE_API_SERVER=ac pnpm --filter @affine/app dev", + "dev:local": "pnpm --filter=!@affine/app build && cross-env NODE_API_SERVER=local pnpm --filter @affine/app dev", "build": " pnpm --filter=!@affine/app build && pnpm --filter!=@affine/datacenter -r build", "build:client": " pnpm --filter=@affine/client-app build:app", "export": "pnpm --filter @affine/app export", diff --git a/packages/app/next.config.js b/packages/app/next.config.js index 6d7225864c..df91a70108 100644 --- a/packages/app/next.config.js +++ b/packages/app/next.config.js @@ -9,6 +9,27 @@ const EDITOR_VERSION = enableDebugLocal ? 'local-version' : dependencies['@blocksuite/editor']; +const profileTarget = { + ac: '100.85.73.88:12001', + dev: '100.85.73.88:12001', + local: '127.0.0.1:3000', +}; + +const getRedirectConfig = profile => { + const target = profileTarget[profile || 'dev'] || profileTarget['dev']; + + return [ + [ + { source: '/api/:path*', destination: `http://${target}/api/:path*` }, + { + source: '/collaboration/:path*', + destination: `http://${target}/collaboration/:path*`, + }, + ], + target, + ]; +}; + /** @type {import('next').NextConfig} */ const nextConfig = { productionBrowserSourceMaps: true, @@ -36,30 +57,12 @@ const nextConfig = { images: { unoptimized: true, }, - // XXX not test yet rewrites: async () => { - if (process.env.NODE_API_SERVER === 'ac') { - let destinationAC = 'http://100.85.73.88:12001/api/:path*'; - printer.info('API request proxy to [AC Server] ' + destinationAC); - return [ - { - source: '/api/:path*', - destination: destinationAC, - }, - ]; - } else { - let destinationStandard = 'http://100.77.180.48:11001/api/:path*'; - printer.info( - 'API request proxy to [Standard Server] ' + destinationStandard - ); - - return [ - { - source: '/api/:path*', - destination: destinationStandard, - }, - ]; - } + const [profile, desc] = getRedirectConfig(process.env.NODE_API_SERVER); + printer.info( + `API request proxy to [${process.env.NODE_API_SERVER} Server]: ` + desc + ); + return profile; }, basePath: process.env.BASE_PATH, }; diff --git a/packages/app/package.json b/packages/app/package.json index ab9571a953..3ba74e2712 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -3,7 +3,6 @@ "version": "0.3.1", "scripts": { "dev": "next dev -p 8080", - "dev:ac": "NODE_API_SERVER=ac next dev -p 8080", "build": "next build", "export": "next export", "start": "next start", @@ -11,10 +10,10 @@ }, "dependencies": { "@affine/datacenter": "workspace:*", - "@blocksuite/blocks": "0.3.1", - "@blocksuite/editor": "0.3.1", + "@blocksuite/blocks": "=0.3.1-20230106060050-1aad55d", + "@blocksuite/editor": "=0.3.1-20230106060050-1aad55d", "@blocksuite/icons": "^2.0.2", - "@blocksuite/store": "0.3.1", + "@blocksuite/store": "=0.3.1-20230106060050-1aad55d", "@emotion/css": "^11.10.0", "@emotion/react": "^11.10.4", "@emotion/server": "^11.10.0", diff --git a/packages/app/src/components/header/editor-header.tsx b/packages/app/src/components/header/editor-header.tsx index c1b52ec464..7f433de467 100644 --- a/packages/app/src/components/header/editor-header.tsx +++ b/packages/app/src/components/header/editor-header.tsx @@ -22,14 +22,14 @@ export const EditorHeader = () => { useEffect(() => { onPropsUpdated(editor => { - setTitle(editor.model?.title || 'Untitled'); + setTitle(editor.pageBlockModel?.title || 'Untitled'); }); }, [onPropsUpdated]); useEffect(() => { setTimeout(() => { // If first time in, need to wait for editor to be inserted into DOM - setTitle(editor?.model?.title || 'Untitled'); + setTitle(editor?.pageBlockModel?.title || 'Untitled'); }, 300); }, [editor]); diff --git a/packages/app/src/components/import/index.tsx b/packages/app/src/components/import/index.tsx index 7387cb5de6..96a383dad2 100644 --- a/packages/app/src/components/import/index.tsx +++ b/packages/app/src/components/import/index.tsx @@ -31,9 +31,10 @@ export const ImportModal = ({ open, onClose }: ImportModalProps) => { setTimeout(() => { const editor = document.querySelector('editor-container'); if (editor) { - const groupId = page.addBlock({ flavour: 'affine:group' }, pageId); + page.addBlock({ flavour: 'affine:surface' }, null); + const frameId = page.addBlock({ flavour: 'affine:frame' }, pageId); // TODO blocksuite should offer a method to import markdown from store - editor.clipboard.importMarkdown(template.source, `${groupId}`); + editor.clipboard.importMarkdown(template.source, `${frameId}`); page.resetHistory(); editor.requestUpdate(); } diff --git a/packages/app/src/components/workspace-slider-bar/index.tsx b/packages/app/src/components/workspace-slider-bar/index.tsx index 4954b5cd31..61e81cb3ed 100644 --- a/packages/app/src/components/workspace-slider-bar/index.tsx +++ b/packages/app/src/components/workspace-slider-bar/index.tsx @@ -4,14 +4,14 @@ import { StyledArrowButton, StyledLink, StyledListItem, - StyledListItemForWorkspace, + // StyledListItemForWorkspace, StyledNewPageButton, StyledSliderBar, StyledSliderBarWrapper, StyledSubListItem, } from './style'; import { Arrow } from './icons'; -import { WorkspaceSelector } from './WorkspaceSelector'; +// import { WorkspaceSelector } from './WorkspaceSelector'; import Collapse from '@mui/material/Collapse'; import { ArrowDownIcon, @@ -109,9 +109,9 @@ export const WorkSpaceSliderBar = () => { - + {/* - + */} { diff --git a/packages/app/src/hooks/use-props-updated.ts b/packages/app/src/hooks/use-props-updated.ts index d76ada43d2..1bbd8d2cd4 100644 --- a/packages/app/src/hooks/use-props-updated.ts +++ b/packages/app/src/hooks/use-props-updated.ts @@ -17,7 +17,7 @@ export const usePropsUpdated: UsePropsUpdated = () => { return; } setTimeout(() => { - editor.model?.propsUpdated.on(() => { + editor.pageBlockModel?.propsUpdated.on(() => { callbackQueue.current.forEach(callback => { callback(editor); }); @@ -26,7 +26,7 @@ export const usePropsUpdated: UsePropsUpdated = () => { return () => { callbackQueue.current = []; - editor?.model?.propsUpdated.dispose(); + editor?.pageBlockModel?.propsUpdated?.dispose(); }; }, [editor]); diff --git a/packages/app/src/pages/playground/templates.tsx b/packages/app/src/pages/playground/templates.tsx index 0517498878..a83d25e516 100644 --- a/packages/app/src/pages/playground/templates.tsx +++ b/packages/app/src/pages/playground/templates.tsx @@ -43,12 +43,16 @@ const All = () => { if (page) { currentWorkspace?.setPageMeta(page.id, { title }); if (page && page.root === null) { - setTimeout(() => { + setTimeout(async () => { const editor = document.querySelector('editor-container'); if (editor) { - const groupId = page.addBlock({ flavour: 'affine:group' }, pageId); + page.addBlock({ flavour: 'affine:surface' }, null); + const frameId = page.addBlock({ flavour: 'affine:frame' }, pageId); // TODO blocksuite should offer a method to import markdown from store - editor.clipboard.importMarkdown(template.source, `${groupId}`); + await editor.clipboard.importMarkdown( + template.source, + `${frameId}` + ); page.resetHistory(); editor.requestUpdate(); } diff --git a/packages/app/src/pages/workspace/[workspaceId]/[pageId].tsx b/packages/app/src/pages/workspace/[workspaceId]/[pageId].tsx index 971e950f8b..05c2c17eba 100644 --- a/packages/app/src/pages/workspace/[workspaceId]/[pageId].tsx +++ b/packages/app/src/pages/workspace/[workspaceId]/[pageId].tsx @@ -1,10 +1,10 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import { - useRef, - useEffect, - useState, - ReactElement, PropsWithChildren, + ReactElement, + useEffect, + useRef, + useState, } from 'react'; import { styled } from '@/styles'; import { EditorHeader } from '@/components/header'; @@ -16,9 +16,11 @@ import type { NextPageWithLayout } from '../..//_app'; import WorkspaceLayout from '@/components/workspace-layout'; import { useRouter } from 'next/router'; import { usePageHelper } from '@/hooks/use-page-helper'; + const StyledEditorContainer = styled('div')(() => { return { height: 'calc(100vh - 60px)', + padding: '0 32px', }; }); @@ -54,15 +56,20 @@ const Page: NextPageWithLayout = () => { flavour: 'affine:page', title, }); - const groupId = currentPage!.addBlock( - { flavour: 'affine:group' }, + currentPage!.addBlock({ flavour: 'affine:surface' }, null); + const frameId = currentPage!.addBlock( + { flavour: 'affine:frame' }, pageId ); - currentPage!.addBlock({ flavour: 'affine:group' }, pageId); + currentPage!.addBlock({ flavour: 'affine:frame' }, pageId); // If this is a first page in workspace, init an introduction markdown if (isFirstPage) { - editor.clipboard.importMarkdown(exampleMarkdown, `${groupId}`); - currentWorkspace!.setPageMeta(currentPage!.id, { title }); + editor.clipboard + .importMarkdown(exampleMarkdown, `${frameId}`) + .then(() => { + currentWorkspace!.setPageMeta(currentPage!.id, { title }); + currentPage!.resetHistory(); + }); } currentPage!.resetHistory(); } diff --git a/packages/app/src/providers/app-state-provider/dynamic-blocksuite.tsx b/packages/app/src/providers/app-state-provider/dynamic-blocksuite.tsx index 3a8fb7671c..1875a35557 100644 --- a/packages/app/src/providers/app-state-provider/dynamic-blocksuite.tsx +++ b/packages/app/src/providers/app-state-provider/dynamic-blocksuite.tsx @@ -18,7 +18,7 @@ const DynamicBlocksuite = ({ const openWorkspace: LoadWorkspaceHandler = async (workspaceId: string) => { if (workspaceId) { const dc = await getDataCenter(); - return dc.load(workspaceId, { providerId: 'affine' }); + return dc.load(workspaceId, { providerId: 'selfhosted' }); } else { return null; } diff --git a/packages/app/src/templates/Welcome-to-AFFiNE-Alpha-v2.0.md b/packages/app/src/templates/Welcome-to-AFFiNE-Alpha-v2.0.md index e0fc661340..6495135903 100644 --- a/packages/app/src/templates/Welcome-to-AFFiNE-Alpha-v2.0.md +++ b/packages/app/src/templates/Welcome-to-AFFiNE-Alpha-v2.0.md @@ -10,6 +10,11 @@ Let us know what you think of this latest version. 2. More complete Markdown support and improved keyboard shortcuts; 3. New features such as dark mode; Switch between view styles using the ☀ and 🌙. 4. Clean and modern UI/UX design. +5. You can self-host locally with Docker. + +```basic +docker run -d -v [YOUR_PATH]:/app/data -p 3000:3000 ghcr.io/toeverything/affine-self-hosted:alpha-abbey-wood +``` **Looking for Markdown syntax or keyboard shortcuts?** @@ -23,15 +28,21 @@ Let us know what you think of this latest version. - Copy and paste **images** into your pages, resize them and add captions - Add horizontal line dividers to your text with `---` and `***` - Changes are saved **locally**, but we still recommend you export your data to avoid data loss -- Insert code blocks with syntax highlighting support using ``` +- Insert code blocks with syntax highlighting support using ````` ### Playground: [] Try a horizontal line: `---` -[] What about a code block? ``` +[] What about a code block? ````` -        console.log('Hello world'); +```javascript +console.log('Hello world'); +``` + +[] Can you resize this image? + +![](https://cdn.affine.pro/694fdbab78e0da3ed7922eba7d506dcf12f57308e1904dd694f53eb2.jpg) **How about page management?** diff --git a/packages/data-center/package.json b/packages/data-center/package.json index 932b160f68..b95ccaffed 100644 --- a/packages/data-center/package.json +++ b/packages/data-center/package.json @@ -26,9 +26,9 @@ "typescript": "^4.8.4" }, "dependencies": { - "@blocksuite/blocks": "^0.3.1", - "@blocksuite/store": "^0.3.1", "@tauri-apps/api": "^1.2.0", + "@blocksuite/blocks": "=0.3.1-20230106060050-1aad55d", + "@blocksuite/store": "=0.3.1-20230106060050-1aad55d", "debug": "^4.3.4", "encoding": "^0.1.13", "firebase": "^9.15.0", diff --git a/packages/data-center/src/datacenter.ts b/packages/data-center/src/datacenter.ts index 62fa9da8b8..b0ab02cb13 100644 --- a/packages/data-center/src/datacenter.ts +++ b/packages/data-center/src/datacenter.ts @@ -4,8 +4,13 @@ import { Workspace, Signal } from '@blocksuite/store'; import { getLogger } from './index.js'; import { getApis, Apis } from './apis/index.js'; -import { AffineProvider, BaseProvider } from './provider/index.js'; -import { LocalProvider } from './provider/index.js'; +import { + AffineProvider, + BaseProvider, + LocalProvider, + SelfHostedProvider, +} from './provider/index.js'; + import { getKVConfigure } from './store.js'; import { TauriIPCProvider } from './provider/tauri-ipc/index.js'; @@ -48,6 +53,7 @@ export class DataCenter { if (typeof window !== 'undefined' && window.CLIENT_APP) { dc.addProvider(TauriIPCProvider); } + dc.addProvider(SelfHostedProvider); return dc; } diff --git a/packages/data-center/src/provider/index.ts b/packages/data-center/src/provider/index.ts index 5718b7eef0..dc4667af7c 100644 --- a/packages/data-center/src/provider/index.ts +++ b/packages/data-center/src/provider/index.ts @@ -20,3 +20,4 @@ export type { Apis, ConfigStore, DataCenterSignals, Workspace }; export type { BaseProvider } from './base.js'; export { AffineProvider } from './affine/index.js'; export { LocalProvider } from './local/index.js'; +export { SelfHostedProvider } from './selfhosted/index.js'; diff --git a/packages/data-center/src/provider/selfhosted/index.ts b/packages/data-center/src/provider/selfhosted/index.ts new file mode 100644 index 0000000000..48632480ef --- /dev/null +++ b/packages/data-center/src/provider/selfhosted/index.ts @@ -0,0 +1,63 @@ +import assert from 'assert'; + +import { LocalProvider } from '../local/index.js'; +import { WebsocketProvider } from './sync.js'; + +export class SelfHostedProvider extends LocalProvider { + static id = 'selfhosted'; + private _ws?: WebsocketProvider; + + constructor() { + super(); + } + + async destroy() { + this._ws?.disconnect(); + } + + async initData() { + const databases = await indexedDB.databases(); + await super.initData( + // set locally to true if exists a same name db + databases + .map(db => db.name) + .filter(v => v) + .includes(this._workspace.room) + ); + + const workspace = this._workspace; + const doc = workspace.doc; + + if (workspace.room) { + try { + // Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later + this._ws = new WebsocketProvider(this.host, workspace.room, doc); + await new Promise((resolve, reject) => { + // TODO: synced will also be triggered on reconnection after losing sync + // There needs to be an event mechanism to emit the synchronization state to the upper layer + assert(this._ws); + this._ws.once('synced', () => resolve()); + this._ws.once('lost-connection', () => resolve()); + this._ws.once('connection-error', () => reject()); + }); + this._signals.listAdd.emit({ + workspace: workspace.room, + provider: this.id, + locally: true, + }); + } catch (e) { + this._logger('Failed to init cloud workspace', e); + } + } + + // if after update, the space:meta is empty + // then we need to get map with doc + // just a workaround for yjs + doc.getMap('space:meta'); + } + + private get host() { + const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:'; + return `${protocol}//${location.host}/collaboration/`; + } +} diff --git a/packages/data-center/src/provider/selfhosted/sync.js b/packages/data-center/src/provider/selfhosted/sync.js new file mode 100644 index 0000000000..54c8dec628 --- /dev/null +++ b/packages/data-center/src/provider/selfhosted/sync.js @@ -0,0 +1,508 @@ +/* eslint-disable no-undef */ +/** + * @module provider/websocket + */ + +/* eslint-env browser */ + +// import * as Y from 'yjs'; // eslint-disable-line +import * as bc from 'lib0/broadcastchannel'; +import * as time from 'lib0/time'; +import * as encoding from 'lib0/encoding'; +import * as decoding from 'lib0/decoding'; +import * as syncProtocol from 'y-protocols/sync'; +import * as authProtocol from 'y-protocols/auth'; +import * as awarenessProtocol from 'y-protocols/awareness'; +import { Observable } from 'lib0/observable'; +import * as math from 'lib0/math'; +import * as url from 'lib0/url'; + +export const messageSync = 0; +export const messageQueryAwareness = 3; +export const messageAwareness = 1; +export const messageAuth = 2; + +/** + * encoder, decoder, provider, emitSynced, messageType + * @type {Array} + */ +const messageHandlers = []; + +messageHandlers[messageSync] = ( + encoder, + decoder, + provider, + emitSynced, + _messageType +) => { + encoding.writeVarUint(encoder, messageSync); + const syncMessageType = syncProtocol.readSyncMessage( + decoder, + encoder, + provider.doc, + provider + ); + if ( + emitSynced && + syncMessageType === syncProtocol.messageYjsSyncStep2 && + !provider.synced + ) { + provider.synced = true; + } +}; + +messageHandlers[messageQueryAwareness] = ( + encoder, + _decoder, + provider, + _emitSynced, + _messageType +) => { + encoding.writeVarUint(encoder, messageAwareness); + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate( + provider.awareness, + Array.from(provider.awareness.getStates().keys()) + ) + ); +}; + +messageHandlers[messageAwareness] = ( + _encoder, + decoder, + provider, + _emitSynced, + _messageType +) => { + awarenessProtocol.applyAwarenessUpdate( + provider.awareness, + decoding.readVarUint8Array(decoder), + provider + ); +}; + +messageHandlers[messageAuth] = ( + _encoder, + decoder, + provider, + _emitSynced, + _messageType +) => { + authProtocol.readAuthMessage(decoder, provider.doc, (_ydoc, reason) => + permissionDeniedHandler(provider, reason) + ); +}; + +// @todo - this should depend on awareness.outdatedTime +const messageReconnectTimeout = 30000; + +/** + * @param {WebsocketProvider} provider + * @param {string} reason + */ +const permissionDeniedHandler = (provider, reason) => + console.warn(`Permission denied to access ${provider.url}.\n${reason}`); + +/** + * @param {WebsocketProvider} provider + * @param {Uint8Array} buf + * @param {boolean} emitSynced + * @return {encoding.Encoder} + */ +const readMessage = (provider, buf, emitSynced) => { + const decoder = decoding.createDecoder(buf); + const encoder = encoding.createEncoder(); + const messageType = decoding.readVarUint(decoder); + const messageHandler = provider.messageHandlers[messageType]; + if (/** @type {any} */ (messageHandler)) { + messageHandler(encoder, decoder, provider, emitSynced, messageType); + } else { + console.error('Unable to compute message'); + } + return encoder; +}; + +/** + * @param {WebsocketProvider} provider + */ +const setupWS = provider => { + if (provider.shouldConnect && provider.ws === null) { + const websocket = new provider._WS(provider.url, 'AFFiNE'); + websocket.binaryType = 'arraybuffer'; + provider.ws = websocket; + provider.wsconnecting = true; + provider.wsconnected = false; + provider.synced = false; + + websocket.onmessage = event => { + provider.wsLastMessageReceived = time.getUnixTime(); + const encoder = readMessage(provider, new Uint8Array(event.data), true); + if (encoding.length(encoder) > 1) { + websocket.send(encoding.toUint8Array(encoder)); + } + }; + websocket.onerror = event => { + provider.emit('connection-error', [event, provider]); + }; + websocket.onclose = event => { + provider.emit('connection-close', [event, provider]); + provider.ws = null; + provider.wsconnecting = false; + if (provider.wsconnected) { + provider.wsconnected = false; + provider.synced = false; + // update awareness (all users except local left) + awarenessProtocol.removeAwarenessStates( + provider.awareness, + Array.from(provider.awareness.getStates().keys()).filter( + client => client !== provider.doc.clientID + ), + provider + ); + provider.emit('status', [ + { + status: 'disconnected', + }, + ]); + } else { + provider.wsUnsuccessfulReconnects++; + } + // Start with no reconnect timeout and increase timeout by + // using exponential backoff starting with 100ms + setTimeout( + setupWS, + math.min( + math.pow(2, provider.wsUnsuccessfulReconnects) * 100, + provider.maxBackoffTime + ), + provider + ); + }; + websocket.onopen = () => { + provider.wsLastMessageReceived = time.getUnixTime(); + provider.wsconnecting = false; + provider.wsconnected = true; + provider.wsUnsuccessfulReconnects = 0; + provider.emit('status', [ + { + status: 'connected', + }, + ]); + // always send sync step 1 when connected + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, messageSync); + syncProtocol.writeSyncStep1(encoder, provider.doc); + websocket.send(encoding.toUint8Array(encoder)); + // broadcast local awareness state + if (provider.awareness.getLocalState() !== null) { + const encoderAwarenessState = encoding.createEncoder(); + encoding.writeVarUint(encoderAwarenessState, messageAwareness); + encoding.writeVarUint8Array( + encoderAwarenessState, + awarenessProtocol.encodeAwarenessUpdate(provider.awareness, [ + provider.doc.clientID, + ]) + ); + websocket.send(encoding.toUint8Array(encoderAwarenessState)); + } + }; + + provider.emit('status', [ + { + status: 'connecting', + }, + ]); + } +}; + +/** + * @param {WebsocketProvider} provider + * @param {ArrayBuffer} buf + */ +const broadcastMessage = (provider, buf) => { + if (provider.wsconnected) { + /** @type {WebSocket} */ (provider.ws).send(buf); + } + if (provider.bcconnected) { + bc.publish(provider.bcChannel, buf, provider); + } +}; + +/** + * Websocket Provider for Yjs. Creates a websocket connection to sync the shared document. + * The document name is attached to the provided url. I.e. the following example + * creates a websocket connection to http://localhost:1234/my-document-name + * + * @example + * import * as Y from 'yjs' + * import { WebsocketProvider } from 'y-websocket' + * const doc = new Y.Doc() + * const provider = new WebsocketProvider('http://localhost:1234', 'my-document-name', doc) + * + * @extends {Observable} + */ +export class WebsocketProvider extends Observable { + /** + * @param {string} serverUrl + * @param {string} roomname + * @param {Y.Doc} doc + * @param {object} [opts] + * @param {boolean} [opts.connect] + * @param {awarenessProtocol.Awareness} [opts.awareness] + * @param {Object} [opts.params] + * @param {typeof WebSocket} [opts.WebSocketPolyfill] Optionall provide a WebSocket polyfill + * @param {number} [opts.resyncInterval] Request server state every `resyncInterval` milliseconds + * @param {number} [opts.maxBackoffTime] Maximum amount of time to wait before trying to reconnect (we try to reconnect using exponential backoff) + * @param {boolean} [opts.disableBc] Disable cross-tab BroadcastChannel communication + */ + constructor( + serverUrl, + roomname, + doc, + { + connect = true, + awareness = new awarenessProtocol.Awareness(doc), + params = {}, + WebSocketPolyfill = WebSocket, + resyncInterval = -1, + maxBackoffTime = 2500, + disableBc = false, + } = {} + ) { + super(); + // ensure that url is always ends with / + while (serverUrl[serverUrl.length - 1] === '/') { + serverUrl = serverUrl.slice(0, serverUrl.length - 1); + } + const encodedParams = url.encodeQueryParams(params); + this.maxBackoffTime = maxBackoffTime; + this.bcChannel = serverUrl + '/' + roomname; + this.url = + serverUrl + + '/' + + roomname + + (encodedParams.length === 0 ? '' : '?' + encodedParams); + this.roomname = roomname; + this.doc = doc; + this._WS = WebSocketPolyfill; + this.awareness = awareness; + this.wsconnected = false; + this.wsconnecting = false; + this.bcconnected = false; + this.disableBc = disableBc; + this.wsUnsuccessfulReconnects = 0; + this.messageHandlers = messageHandlers.slice(); + /** + * @type {boolean} + */ + this._synced = false; + /** + * @type {WebSocket?} + */ + this.ws = null; + this.wsLastMessageReceived = 0; + /** + * Whether to connect to other peers or not + * @type {boolean} + */ + this.shouldConnect = connect; + + /** + * @type {number} + */ + this._resyncInterval = 0; + if (resyncInterval > 0) { + this._resyncInterval = /** @type {any} */ ( + setInterval(() => { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + // resend sync step 1 + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, messageSync); + syncProtocol.writeSyncStep1(encoder, doc); + this.ws.send(encoding.toUint8Array(encoder)); + } + }, resyncInterval) + ); + } + + /** + * @param {ArrayBuffer} data + * @param {any} origin + */ + this._bcSubscriber = (data, origin) => { + if (origin !== this) { + const encoder = readMessage(this, new Uint8Array(data), false); + if (encoding.length(encoder) > 1) { + bc.publish(this.bcChannel, encoding.toUint8Array(encoder), this); + } + } + }; + /** + * Listens to Yjs updates and sends them to remote peers (ws and broadcastchannel) + * @param {Uint8Array} update + * @param {any} origin + */ + this._updateHandler = (update, origin) => { + if (origin !== this) { + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, messageSync); + syncProtocol.writeUpdate(encoder, update); + broadcastMessage(this, encoding.toUint8Array(encoder)); + } + }; + this.doc.on('update', this._updateHandler); + /** + * @param {any} changed + * @param {any} _origin + */ + this._awarenessUpdateHandler = ({ added, updated, removed }, _origin) => { + const changedClients = added.concat(updated).concat(removed); + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, messageAwareness); + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients) + ); + broadcastMessage(this, encoding.toUint8Array(encoder)); + }; + this._unloadHandler = () => { + awarenessProtocol.removeAwarenessStates( + this.awareness, + [doc.clientID], + 'window unload' + ); + }; + if (typeof window !== 'undefined') { + window.addEventListener('unload', this._unloadHandler); + } else if (typeof process !== 'undefined') { + process.on('exit', this._unloadHandler); + } + awareness.on('update', this._awarenessUpdateHandler); + this._checkInterval = /** @type {any} */ ( + setInterval(() => { + if ( + this.wsconnected && + messageReconnectTimeout < + time.getUnixTime() - this.wsLastMessageReceived + ) { + // no message received in a long time - not even your own awareness + // updates (which are updated every 15 seconds) + /** @type {WebSocket} */ (this.ws).close(); + } + }, messageReconnectTimeout / 10) + ); + if (connect) { + this.connect(); + } + } + + /** + * @type {boolean} + */ + get synced() { + return this._synced; + } + + set synced(state) { + if (this._synced !== state) { + this._synced = state; + this.emit('synced', [state]); + this.emit('sync', [state]); + } + } + + destroy() { + if (this._resyncInterval !== 0) { + clearInterval(this._resyncInterval); + } + clearInterval(this._checkInterval); + this.disconnect(); + if (typeof window !== 'undefined') { + window.removeEventListener('unload', this._unloadHandler); + } else if (typeof process !== 'undefined') { + process.off('exit', this._unloadHandler); + } + this.awareness.off('update', this._awarenessUpdateHandler); + this.doc.off('update', this._updateHandler); + super.destroy(); + } + + connectBc() { + if (this.disableBc) { + return; + } + if (!this.bcconnected) { + bc.subscribe(this.bcChannel, this._bcSubscriber); + this.bcconnected = true; + } + // send sync step1 to bc + // write sync step 1 + const encoderSync = encoding.createEncoder(); + encoding.writeVarUint(encoderSync, messageSync); + syncProtocol.writeSyncStep1(encoderSync, this.doc); + bc.publish(this.bcChannel, encoding.toUint8Array(encoderSync), this); + // broadcast local state + const encoderState = encoding.createEncoder(); + encoding.writeVarUint(encoderState, messageSync); + syncProtocol.writeSyncStep2(encoderState, this.doc); + bc.publish(this.bcChannel, encoding.toUint8Array(encoderState), this); + // write queryAwareness + const encoderAwarenessQuery = encoding.createEncoder(); + encoding.writeVarUint(encoderAwarenessQuery, messageQueryAwareness); + bc.publish( + this.bcChannel, + encoding.toUint8Array(encoderAwarenessQuery), + this + ); + // broadcast local awareness state + const encoderAwarenessState = encoding.createEncoder(); + encoding.writeVarUint(encoderAwarenessState, messageAwareness); + encoding.writeVarUint8Array( + encoderAwarenessState, + awarenessProtocol.encodeAwarenessUpdate(this.awareness, [ + this.doc.clientID, + ]) + ); + bc.publish( + this.bcChannel, + encoding.toUint8Array(encoderAwarenessState), + this + ); + } + + disconnectBc() { + // broadcast message with local awareness state set to null (indicating disconnect) + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, messageAwareness); + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate( + this.awareness, + [this.doc.clientID], + new Map() + ) + ); + broadcastMessage(this, encoding.toUint8Array(encoder)); + if (this.bcconnected) { + bc.unsubscribe(this.bcChannel, this._bcSubscriber); + this.bcconnected = false; + } + } + + disconnect() { + this.shouldConnect = false; + this.disconnectBc(); + if (this.ws !== null) { + this.ws.close(); + } + } + + connect() { + this.shouldConnect = true; + if (!this.wsconnected && this.ws === null) { + setupWS(this); + this.connectBc(); + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0174671219..47f90158c6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -148,10 +148,10 @@ importers: packages/app: specifiers: '@affine/datacenter': workspace:* - '@blocksuite/blocks': 0.3.1 - '@blocksuite/editor': 0.3.1 + '@blocksuite/blocks': '=0.3.1-20230106060050-1aad55d' + '@blocksuite/editor': '=0.3.1-20230106060050-1aad55d' '@blocksuite/icons': ^2.0.2 - '@blocksuite/store': 0.3.1 + '@blocksuite/store': '=0.3.1-20230106060050-1aad55d' '@emotion/css': ^11.10.0 '@emotion/react': ^11.10.4 '@emotion/server': ^11.10.0 @@ -190,10 +190,10 @@ importers: yjs: ^13.5.44 dependencies: '@affine/datacenter': link:../data-center - '@blocksuite/blocks': 0.3.1_yjs@13.5.44 - '@blocksuite/editor': 0.3.1_yjs@13.5.44 + '@blocksuite/blocks': 0.3.1-20230106060050-1aad55d_yjs@13.5.44 + '@blocksuite/editor': 0.3.1-20230106060050-1aad55d_yjs@13.5.44 '@blocksuite/icons': 2.0.4_w5j4k42lgipnm43s3brx6h3c34 - '@blocksuite/store': 0.3.1_yjs@13.5.44 + '@blocksuite/store': 0.3.1-20230106060050-1aad55d_yjs@13.5.44 '@emotion/css': 11.10.0 '@emotion/react': 11.10.4_w5j4k42lgipnm43s3brx6h3c34 '@emotion/server': 11.10.0_@emotion+css@11.10.0 @@ -234,8 +234,8 @@ importers: packages/data-center: specifiers: - '@blocksuite/blocks': ^0.3.1 - '@blocksuite/store': ^0.3.1 + '@blocksuite/blocks': '=0.3.1-20230106060050-1aad55d' + '@blocksuite/store': '=0.3.1-20230106060050-1aad55d' '@playwright/test': ^1.29.1 '@tauri-apps/api': ^1.2.0 '@types/debug': ^4.1.7 @@ -252,9 +252,9 @@ importers: y-protocols: ^1.0.5 yjs: ^13.5.44 dependencies: - '@blocksuite/blocks': 0.3.1_yjs@13.5.44 - '@blocksuite/store': 0.3.1_yjs@13.5.44 '@tauri-apps/api': 1.2.0 + '@blocksuite/blocks': 0.3.1-20230106060050-1aad55d_yjs@13.5.44 + '@blocksuite/store': 0.3.1-20230106060050-1aad55d_yjs@13.5.44 debug: 4.3.4 encoding: 0.1.13 firebase: 9.15.0_encoding@0.1.13 @@ -1610,10 +1610,11 @@ packages: resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} dev: true - /@blocksuite/blocks/0.3.1_yjs@13.5.44: - resolution: {integrity: sha512-b0dGz2MG4yIgngJPRumaMY58wAsd2FEVSZl3tpCXlagK9I0HD165Bq70PxcaRHVjBSV1Gf29ZVHUF6BVTYogPw==} + /@blocksuite/blocks/0.3.1-20230106060050-1aad55d_yjs@13.5.44: + resolution: {integrity: sha512-qRNXmhjw+GAGsV1mI2XXPxYTlHfsFHv9ttTCNQ6IIcxvc5Hh6lWmdwVibxvlpYUkgEc1zv3/GxOEsR/ngpZXzQ==} dependencies: - '@blocksuite/store': 0.3.1_yjs@13.5.44 + '@blocksuite/phasor': 0.3.1_yjs@13.5.44 + '@blocksuite/store': 0.3.1-20230106060050-1aad55d_yjs@13.5.44 '@tldraw/intersect': 1.8.0 autosize: 5.0.2 highlight.js: 11.7.0 @@ -1629,11 +1630,11 @@ packages: - yjs dev: false - /@blocksuite/editor/0.3.1_yjs@13.5.44: - resolution: {integrity: sha512-ycKcyvPW6R8R2GZOFneGH1xNi5gJBx5WtWjW9YwcQslFzXVWMCCBips1Bud2uL4kkbWQoodyua6k2vsXxGAKLw==} + /@blocksuite/editor/0.3.1-20230106060050-1aad55d_yjs@13.5.44: + resolution: {integrity: sha512-wSlAF9XVxIkHFJ1qCzn7oQ/gwXybFYMrzRl35UTJV509D+DuWZefRZWvpdIDCOUJ24uQscr1HxwsON11ltfWgA==} dependencies: - '@blocksuite/blocks': 0.3.1_yjs@13.5.44 - '@blocksuite/store': 0.3.1_yjs@13.5.44 + '@blocksuite/blocks': 0.3.1-20230106060050-1aad55d_yjs@13.5.44 + '@blocksuite/store': 0.3.1-20230106060050-1aad55d_yjs@13.5.44 lit: 2.5.0 marked: 4.2.5 turndown: 7.1.1 @@ -1654,8 +1655,16 @@ packages: react: 18.2.0 dev: false - /@blocksuite/store/0.3.1_yjs@13.5.44: - resolution: {integrity: sha512-kynVTDfNCSChz2JI2rtGHxRIV2YrLzvAgVajcbfDVCuXKG0siBoEjLasG1a0kvevbvW/FabrNAj+xaIplklioA==} + /@blocksuite/phasor/0.3.1_yjs@13.5.44: + resolution: {integrity: sha512-aJmAQn2qoF6HxFZWgq7xa/pWVyzg3MmD6dynIHAKdfN7rBdKk3PNA+lRX919QkD2e270N/zgHEGFFQI1Nj5xrA==} + peerDependencies: + yjs: ^13 + dependencies: + yjs: 13.5.44 + dev: false + + /@blocksuite/store/0.3.1-20230106060050-1aad55d_yjs@13.5.44: + resolution: {integrity: sha512-dRy+YzlWMwiYq0Im9NogK/NTkV+NKK+lgejYq56m6nH2m16/G9AMODqP0oQy/XeYFevUpL9i9RdV0rHsJ2gc0Q==} peerDependencies: yjs: ^13 dependencies: diff --git a/tests/quick-search.spec.ts b/tests/quick-search.spec.ts index 242e09bc51..1fd1b1efb9 100644 --- a/tests/quick-search.spec.ts +++ b/tests/quick-search.spec.ts @@ -7,10 +7,13 @@ loadPage(); const openQuickSearchByShortcut = async (page: Page) => await withCtrlOrMeta(page, () => page.keyboard.press('k', { delay: 50 })); -async function assertTitleTexts(page: Page, texts: string[]) { - const actual = await page - .locator('.affine-default-page-block-title') - .allTextContents(); +async function assertTitleTexts(page: Page, texts: string) { + const actual = await page.evaluate(() => { + const titleElement = ( + document.querySelector('.affine-default-page-block-title') + ); + return titleElement.value; + }); expect(actual).toEqual(texts); } async function assertResultList(page: Page, texts: string[]) { @@ -55,7 +58,7 @@ test.describe('Add new page in quick search', () => { const addNewPage = page.locator('[data-testid=quickSearch-addNewPage]'); await addNewPage.click(); await page.waitForTimeout(200); - await assertTitleTexts(page, ['']); + await assertTitleTexts(page, ''); }); test('Create a new page with keyword', async ({ page }) => { @@ -65,7 +68,7 @@ test.describe('Add new page in quick search', () => { const addNewPage = page.locator('[data-testid=quickSearch-addNewPage]'); await addNewPage.click(); await page.waitForTimeout(200); - await assertTitleTexts(page, ['test123456']); + await assertTitleTexts(page, 'test123456'); }); }); @@ -81,6 +84,6 @@ test.describe('Search and select', () => { await page.keyboard.insertText('test123456'); await assertResultList(page, ['test123456']); await page.keyboard.press('Enter', { delay: 50 }); - await assertTitleTexts(page, ['test123456']); + await assertTitleTexts(page, 'test123456'); }); });