chore: v0.10.4-beta.2

close #5261

https://github.com/toeverything/AFFiNE/assets/102217452/8a7c4347-96df-45a3-973a-806876e5703d

.devcontainer/Dockerfile

@@ -0,0 +1,9 @@
+FROM mcr.microsoft.com/devcontainers/base:bookworm
+
+# Install Homebrew For Linux
+RUN /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" && \
+    eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" && \
+    echo "eval \"\$($(brew --prefix)/bin/brew shellenv)\"" >> /home/vscode/.zshrc && \
+    echo "eval \"\$($(brew --prefix)/bin/brew shellenv)\"" >> /home/vscode/.bashrc && \
+    # Install Graphite
+    brew install withgraphite/tap/graphite && gt --version

.devcontainer/build.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# This is a script used by the devcontainer to build the project
+
+#Enable yarn
+corepack enable
+corepack prepare yarn@stable --activate
+
+# install dependencies
+yarn install
+
+# Create database
+yarn workspace @affine/server prisma db push

.devcontainer/devcontainer.json

@@ -0,0 +1,25 @@
+// For format details, see https://aka.ms/devcontainer.json.
+{
+  "name": "Debian",
+  "dockerComposeFile": "docker-compose.yml",
+  "service": "app",
+  "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+  "features": {
+    "ghcr.io/devcontainers/features/node:1": {
+      "version": "18"
+    },
+    "ghcr.io/devcontainers/features/rust:1": {}
+  },
+  // Configure tool-specific properties.
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-playwright.playwright",
+        "esbenp.prettier-vscode",
+        "streetsidesoftware.code-spell-checker"
+      ]
+    }
+  },
+  "updateContentCommand": "bash ./.devcontainer/build.sh",
+  "postCreateCommand": "bash ./.devcontainer/setup-user.sh"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3.8'
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ../..:/workspaces:cached
+    command: sleep infinity
+    network_mode: service:db
+    environment:
+      DATABASE_URL: postgresql://affine:affine@db:5432/affine
+
+  db:
+    image: postgres:latest
+    restart: unless-stopped
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: affine
+      POSTGRES_USER: affine
+      POSTGRES_DB: affine
+
+volumes:
+  postgres-data:

.devcontainer/setup-user.sh

@@ -0,0 +1,7 @@
+if [ -v GRAPHITE_TOKEN ];then
+    gt auth --token $GRAPHITE_TOKEN
+fi
+
+git fetch
+git branch canary -t origin/canary
+gt init --trunk canary

.eslintrc.js

@@ -58,7 +58,7 @@ const createPattern = packageName => [
 const allPackages = [
   'packages/backend/server',
   'packages/frontend/component',
-  'packages/frontend/web',
+  'packages/frontend/core',
   'packages/frontend/electron',
   'packages/frontend/graphql',
   'packages/frontend/hooks',
@@ -126,6 +126,8 @@ const config = {
     'no-cond-assign': 'off',
     'no-constant-binary-expression': 'error',
     'no-constructor-return': 'error',
+    'no-self-compare': 'error',
+    eqeqeq: ['error', 'always', { null: 'ignore' }],
     'react/prop-types': 'off',
     'react/jsx-no-useless-fragment': 'error',
     '@typescript-eslint/consistent-type-imports': 'error',
@@ -133,6 +135,9 @@ const config = {
     '@typescript-eslint/no-explicit-any': 'off',
     '@typescript-eslint/no-empty-function': 'off',
     '@typescript-eslint/await-thenable': 'error',
+    '@typescript-eslint/require-array-sort-compare': 'error',
+    '@typescript-eslint/unified-signatures': 'error',
+    '@typescript-eslint/prefer-for-of': 'error',
     '@typescript-eslint/no-unused-vars': [
       'error',
       {
@@ -204,6 +209,17 @@ const config = {
       },
     ],
     'unicorn/no-unnecessary-await': 'error',
+    'unicorn/no-useless-fallback-in-spread': 'error',
+    'unicorn/prefer-dom-node-dataset': 'error',
+    'unicorn/prefer-dom-node-append': 'error',
+    'unicorn/prefer-dom-node-remove': 'error',
+    'unicorn/prefer-array-some': 'error',
+    'unicorn/prefer-date-now': 'error',
+    'unicorn/prefer-blob-reading-methods': 'error',
+    'unicorn/no-typeof-undefined': 'error',
+    'unicorn/no-useless-promise-resolve-reject': 'error',
+    'unicorn/no-new-array': 'error',
+    'unicorn/new-for-builtins': 'error',
     'sonarjs/no-all-duplicated-branches': 'error',
     'sonarjs/no-element-overwrite': 'error',
     'sonarjs/no-empty-collection': 'error',
@@ -254,7 +270,14 @@ const config = {
           },
         ],
         '@typescript-eslint/no-misused-promises': ['error'],
+        '@typescript-eslint/prefer-readonly': 'error',
         'i/no-extraneous-dependencies': ['error'],
+        'react-hooks/exhaustive-deps': [
+          'warn',
+          {
+            additionalHooks: 'useAsyncCallback',
+          },
+        ],
       },
     })),
     {

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -58,6 +58,6 @@ body:
       label: Are you willing to submit a PR?
       description: >
         (Optional) We encourage you to submit a [Pull Request](https://github.com/toeverything/affine/pulls) (PR) to help improve AFFiNE for everyone, especially if you have a good understanding of how to implement a fix or feature.
-        See the AFFiNE [Contributing Guide](https://github.com/toeverything/affine/blob/master/CONTRIBUTING.md) to get started.
+        See the AFFiNE [Contributing Guide](https://github.com/toeverything/affine/blob/canary/CONTRIBUTING.md) to get started.
       options:
         - label: Yes I'd like to help by submitting a PR!

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -31,6 +31,6 @@ body:
       label: Are you willing to submit a PR?
       description: >
         (Optional) We encourage you to submit a [Pull Request](https://github.com/toeverything/affine/pulls) (PR) to help improve AFFiNE for everyone, especially if you have a good understanding of how to implement a fix or feature.
-        See the AFFiNE [Contributing Guide](https://github.com/toeverything/affine/blob/master/CONTRIBUTING.md) to get started.
+        See the AFFiNE [Contributing Guide](https://github.com/toeverything/affine/blob/canary/CONTRIBUTING.md) to get started.
       options:
         - label: Yes I'd like to help by submitting a PR!

.github/actions/build-rust/action.yml

@@ -14,14 +14,28 @@ inputs:
 runs:
   using: 'composite'
   steps:
+    - name: Print rustup toolchain version
+      shell: bash
+      id: rustup-version
+      run: |
+        export RUST_TOOLCHAIN_VERSION="$(grep 'channel' rust-toolchain.toml | head -1 | awk -F '"' '{print $2}')"
+        echo "Rust toolchain version: $RUST_TOOLCHAIN_VERSION"
+        echo "RUST_TOOLCHAIN_VERSION=$RUST_TOOLCHAIN_VERSION" >> "$GITHUB_OUTPUT"
     - name: Setup Rust
       uses: dtolnay/rust-toolchain@stable
       with:
-        toolchain: stable
+        toolchain: '${{ steps.rustup-version.outputs.RUST_TOOLCHAIN_VERSION }}'
         targets: ${{ inputs.target }}
       env:
         CARGO_INCREMENTAL: '1'
 
+    - name: Set CC
+      if: ${{ contains(inputs.target, 'linux') && inputs.package != '@affine/native' }}
+      shell: bash
+      run: |
+        echo "CC=clang" >> "$GITHUB_ENV"
+        echo "TARGET_CC=clang" >> "$GITHUB_ENV"
+
     - name: Cache cargo
       uses: actions/cache@v3
       with:
@@ -29,47 +43,13 @@ runs:
           ~/.cargo/registry/index/
           ~/.cargo/registry/cache/
           ~/.cargo/git/db/
-          .cargo-cache
+          ~/.napi-rs
           target/${{ inputs.target }}
         key: stable-${{ inputs.target }}-cargo-cache
     - name: Build
-      if: ${{ inputs.target != 'x86_64-unknown-linux-gnu' && inputs.target != 'aarch64-unknown-linux-gnu' }}
       shell: bash
       run: |
-        yarn workspace ${{ inputs.package }} nx build ${{ inputs.package }} --target ${{ inputs.target }}
+        yarn workspace ${{ inputs.package }} nx build ${{ inputs.package }} --target ${{ inputs.target }} --use-napi-cross
       env:
         NX_CLOUD_ACCESS_TOKEN: ${{ inputs.nx_token }}
-
-    - name: Build
-      if: ${{ inputs.target == 'x86_64-unknown-linux-gnu' }}
-      uses: addnab/docker-run-action@v3
-      with:
-        image: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
-        options: --user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build -e NX_CLOUD_ACCESS_TOKEN=${{ inputs.nx_token }}
-        run: |
-          export CC=x86_64-unknown-linux-gnu-gcc
-          export CC_x86_64_unknown_linux_gnu=x86_64-unknown-linux-gnu-gcc
-          export RUSTFLAGS="-C debuginfo=1"
-          yarn workspace ${{ inputs.package }} nx build ${{ inputs.package }} --target ${{ inputs.target }}
-          if [ -d "node_modules/.cache" ]; then
-            chmod -R 777 node_modules/.cache
-          fi
-          if [ -d "target" ]; then
-            chmod -R 777 target;
-          fi
-
-    - name: Build
-      if: ${{ inputs.target == 'aarch64-unknown-linux-gnu' }}
-      uses: addnab/docker-run-action@v3
-      with:
-        image: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
-        options: --user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build -e NX_CLOUD_ACCESS_TOKEN=${{ inputs.nx_token }}
-        run: |
-          export RUSTFLAGS="-C debuginfo=1"
-          yarn workspace ${{ inputs.package }} nx build ${{ inputs.package }} --target ${{ inputs.target }}
-          if [ -d "node_modules/.cache" ]; then
-            chmod -R 777 node_modules/.cache
-          fi
-          if [ -d "target" ]; then
-            chmod -R 777 target;
-          fi
+        DEBUG: 'napi:*'

.github/actions/deploy/action.yml

@@ -26,7 +26,7 @@ runs:
         echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
     - uses: azure/setup-helm@v3
     - id: auth
-      uses: google-github-actions/auth@v1
+      uses: google-github-actions/auth@v2
       with:
         workload_identity_provider: 'projects/${{ inputs.gcp-project-number }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions-helm-deploy'
         service_account: '${{ inputs.service-account }}'
@@ -34,7 +34,7 @@ runs:
         project_id: '${{ inputs.gcp-project-id }}'
 
     - name: 'Setup gcloud cli'
-      uses: 'google-github-actions/setup-gcloud@v1'
+      uses: 'google-github-actions/setup-gcloud@v2'
       with:
         install_components: 'gke-gcloud-auth-plugin'
 

.github/actions/deploy/deploy.mjs

@@ -1,6 +1,7 @@
 import { execSync } from 'node:child_process';
 
 const {
+  APP_VERSION,
   BUILD_TYPE,
   DEPLOY_HOST,
   CANARY_DEPLOY_HOST,
@@ -41,8 +42,8 @@ const createHelmCommand = ({ isDryRun }) => {
   const staticIpName = isProduction
     ? 'affine-cluster-production'
     : isBeta
-    ? 'affine-cluster-beta'
-    : 'affine-cluster-dev';
+      ? 'affine-cluster-beta'
+      : 'affine-cluster-dev';
   const redisAndPostgres =
     isProduction || isBeta
       ? [
@@ -68,8 +69,8 @@ const createHelmCommand = ({ isDryRun }) => {
         ]
       : [];
   const webReplicaCount = isProduction ? 3 : isBeta ? 2 : 2;
-  const graphqlReplicaCount = isProduction ? 10 : isBeta ? 10 : 2;
-  const syncReplicaCount = isProduction ? 10 : isBeta ? 10 : 2;
+  const graphqlReplicaCount = isProduction ? 10 : isBeta ? 5 : 2;
+  const syncReplicaCount = isProduction ? 10 : isBeta ? 5 : 2;
   const namespace = isProduction ? 'production' : isBeta ? 'beta' : 'dev';
   // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
   const host = DEPLOY_HOST || CANARY_DEPLOY_HOST;
@@ -79,6 +80,7 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set        global.ingress.enabled=true`,
     `--set-json   global.ingress.annotations=\"{ \\"kubernetes.io/ingress.class\\": \\"gce\\", \\"kubernetes.io/ingress.allow-http\\": \\"true\\", \\"kubernetes.io/ingress.global-static-ip-name\\": \\"${staticIpName}\\" }\"`,
     `--set-string global.ingress.host="${host}"`,
+    `--set-string global.version="${APP_VERSION}"`,
     ...redisAndPostgres,
     `--set        web.replicaCount=${webReplicaCount}`,
     `--set-string web.image.tag="${imageTag}"`,
@@ -105,7 +107,7 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set        sync.replicaCount=${syncReplicaCount}`,
     `--set-string sync.image.tag="${imageTag}"`,
     ...serviceAnnotations,
-    `--version "0.0.0-${buildType}.${GIT_SHORT_HASH}" --timeout 10m`,
+    `--timeout 10m`,
     flag,
   ].join(' ');
   return deployCommand;

.github/actions/download-core/action.yml

@@ -0,0 +1,22 @@
+name: 'Download core artifacts'
+description: 'Download core artifacts and extract to dist'
+inputs:
+  path:
+    description: 'Path to extract'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Download tar.gz
+      uses: actions/download-artifact@v3
+      with:
+        name: core
+        path: .
+
+    - name: Extract core artifacts
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.path }}
+        tar -xvf dist.tar.gz --directory ${{ inputs.path }}
+        rm dist.tar.gz

.github/actions/setup-node/action.yml

@@ -36,22 +36,24 @@ inputs:
     description: 'Set enableScripts in .yarnrc.yml'
     required: false
     default: 'true'
+  full-cache:
+    description: 'Full installation cache'
+    required: false
 
 runs:
   using: 'composite'
   steps:
     - name: Setup Node.js
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version-file: '.nvmrc'
         registry-url: https://npm.pkg.github.com
         scope: '@toeverything'
-        cache: 'yarn'
 
     - name: Set nmMode
-      if: ${{ inputs.hard-link-nm == 'true' }}
+      if: ${{ inputs.hard-link-nm == 'false' }}
       shell: bash
-      run: yarn config set nmMode hardlinks-local
+      run: yarn config set nmMode classic
 
     - name: Set nmHoistingLimits
       if: ${{ inputs.nmHoistingLimits }}
@@ -63,6 +65,48 @@ runs:
       shell: bash
       run: yarn config set enableScripts false
 
+    - name: Set yarn global cache path
+      shell: bash
+      id: yarn-cache
+      run: node -e "const p = $(yarn config cacheFolder --json).effective; console.log('yarn_global_cache=' + p)" >> $GITHUB_OUTPUT
+
+    - name: Cache non-full yarn cache on Linux
+      uses: actions/cache@v3
+      if: ${{ inputs.full-cache != 'true' && runner.os == 'Linux' }}
+      with:
+        path: |
+          node_modules
+          ${{ steps.yarn-cache.outputs.yarn_global_cache }}
+        key: node_modules-cache-${{ github.job }}-${{ runner.os }}
+
+    # The network performance on macOS is very poor
+    # and the decompression performance on Windows is very terrible
+    # so we reduce the number of cached files on non-Linux systems by remove node_modules from cache path.
+    - name: Cache non-full yarn cache on non-Linux
+      uses: actions/cache@v3
+      if: ${{ inputs.full-cache != 'true' && runner.os != 'Linux' }}
+      with:
+        path: |
+          ${{ steps.yarn-cache.outputs.yarn_global_cache }}
+        key: node_modules-cache-${{ github.job }}-${{ runner.os }}
+
+    - name: Cache full yarn cache on Linux
+      uses: actions/cache@v3
+      if: ${{ inputs.full-cache == 'true' && runner.os == 'Linux' }}
+      with:
+        path: |
+          node_modules
+          ${{ steps.yarn-cache.outputs.yarn_global_cache }}
+        key: node_modules-cache-full-${{ runner.os }}
+
+    - name: Cache full yarn cache on non-Linux
+      uses: actions/cache@v3
+      if: ${{ inputs.full-cache == 'true' && runner.os != 'Linux' }}
+      with:
+        path: |
+          ${{ steps.yarn-cache.outputs.yarn_global_cache }}
+        key: node_modules-cache-full-${{ runner.os }}
+
     - name: yarn install
       if: ${{ inputs.package-install == 'true' }}
       continue-on-error: true
@@ -102,8 +146,8 @@ runs:
       id: playwright-cache
       if: ${{ inputs.playwright-install == 'true' }}
       with:
-        path: '~/.cache/ms-playwright'
-        key: '${{ runner.os }}-${{ runner.arch }}-playwright-${{ steps.playwright-version.outputs.version }}'
+        path: ${{ github.workspace }}/node_modules/.cache/ms-playwright
+        key: '${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}'
         # As a fallback, if the Playwright version has changed, try use the
         # most recently cached version. There's a good chance that at least one
         # of the browser binary versions haven't been updated, so Playwright can
@@ -113,7 +157,7 @@ runs:
         # date cache, but still let Playwright decide if it needs to download
         # new binaries or not.
         restore-keys: |
-          ${{ runner.os }}-${{ runner.arch }}-playwright-
+          ${{ runner.os }}-playwright-
 
     # If the Playwright browser binaries weren't able to be restored, we tell
     # playwright to install everything for us.
@@ -121,6 +165,8 @@ runs:
       shell: bash
       if: inputs.playwright-install == 'true'
       run: yarn playwright install --with-deps chromium
+      env:
+        PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/node_modules/.cache/ms-playwright
 
     - name: Get installed Electron version
       id: electron-version
@@ -134,16 +180,16 @@ runs:
       if: ${{ inputs.electron-install == 'true' }}
       with:
         path: 'node_modules/.cache/electron'
-        key: '${{ runner.os }}-{{ runner.arch }}-electron-${{ steps.electron-version.outputs.version }}'
+        key: '${{ runner.os }}-electron-${{ steps.electron-version.outputs.version }}'
         restore-keys: |
-          ${{ runner.os }}-{{ runner.arch }}-electron-
+          ${{ runner.os }}-electron-
 
     - name: Install Electron binary
       shell: bash
       if: inputs.electron-install == 'true'
       run: node ./node_modules/electron/install.js
       env:
-        ELECTRON_OVERRIDE_DIST_PATH: ./node_modules/.cache/electron
+        electron_config_cache: ./node_modules/.cache/electron
 
     - name: Build Infra
       shell: bash

.github/dependabot.yml

@@ -1,31 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: 'npm'
-    directory: '/'
-    groups:
-      all-npm-dependencies:
-        patterns:
-          - '*'
-    schedule:
-      interval: 'weekly'
-    versioning-strategy: increase
-    commit-message:
-      prefix: 'chore'
-  - package-ecosystem: 'cargo'
-    directory: '/'
-    schedule:
-      interval: 'weekly'
-    versioning-strategy: auto
-    commit-message:
-      prefix: 'chore'
-    groups:
-      all-cargo-dependencies:
-        patterns:
-          - '*'
-
-  - package-ecosystem: 'github-actions'
-    directory: '/'
-    schedule:
-      interval: 'daily'
-    commit-message:
-      prefix: 'ci'

.github/helm/affine-cloud/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
-  version: 12.5.8
-digest: sha256:c91c0dc1370e879538dc9d6e435e731a726ef99d6a3b081372318483792b48a7
-generated: "2023-06-27T18:34:12.683806+08:00"
+  version: 13.2.23
+digest: sha256:5b64538509bd067bb0f67bf082847a2c5d66dc37d0b9d7948a40405d9c446400
+generated: "2023-12-05T03:04:57.997927753Z"

.github/helm/affine-cloud/Chart.yaml

@@ -8,5 +8,5 @@ appVersion: '0.6.1'
 
 dependencies:
   - name: postgresql
-    version: 12.5.8
+    version: 13.2.23
     repository: https://charts.bitnami.com/bitnami

.github/helm/affine/Chart.yaml

@@ -3,4 +3,4 @@ name: affine
 description: AFFiNE cloud chart
 type: application
 version: 0.0.0
-appVersion: '0.7.0-canary.18'
+appVersion: "0.10.4-beta.2"

.github/helm/affine/charts/graphql/Chart.yaml

@@ -3,4 +3,4 @@ name: graphql
 description: AFFiNE GraphQL server
 type: application
 version: 0.0.0
-appVersion: '0.7.0-canary.18'
+appVersion: "0.10.4-beta.2"

.github/helm/affine/charts/graphql/templates/deployment.yaml

@@ -35,6 +35,8 @@ spec:
                 key: key
           - name: NODE_ENV
             value: "{{ .Values.env }}"
+          - name: NODE_OPTIONS
+            value: "--max-old-space-size=4096"
           - name: NO_COLOR
             value: "1"
           - name: SERVER_FLAVOR

.github/helm/affine/charts/graphql/templates/monitoring.yaml

@@ -1,13 +0,0 @@
-{{- if .Values.global.gke.enabled -}}
-apiVersion: monitoring.googleapis.com/v1
-kind: PodMonitoring
-metadata:
-  name: "{{ .Chart.Name }}-monitoring"
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: "{{ include "graphql.name" . }}"
-  endpoints:
-    - port: {{ .Values.service.port }}
-      interval: 30s
-{{- end }}

.github/helm/affine/charts/graphql/values.yaml

@@ -72,11 +72,8 @@ podSecurityContext:
   fsGroup: 2000
 
 resources:
-  limits:
-    cpu: '4'
-    memory: 8Gi
   requests:
-    cpu: '2'
+    cpu: '4'
     memory: 4Gi
 
 probe:

.github/helm/affine/charts/sync/Chart.yaml

@@ -3,4 +3,4 @@ name: sync
 description: A Helm chart for Kubernetes
 type: application
 version: 0.0.0
-appVersion: "0.7.0-canary.18"
+appVersion: "0.10.4-beta.2"

.github/helm/affine/charts/sync/templates/monitoring.yaml

@@ -1,13 +0,0 @@
-{{- if .Values.global.gke.enabled -}}
-apiVersion: monitoring.googleapis.com/v1
-kind: PodMonitoring
-metadata:
-  name: "{{ .Chart.Name }}-monitoring"
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: "{{ include "sync.name" . }}"
-  endpoints:
-    - port: {{ .Values.service.port }}
-      interval: 30s
-{{- end }}

.github/labeler.yml

@@ -1,62 +1,115 @@
 docs:
-  - 'docs/**/*'
-  - '**/README.md'
-  - 'packages/frontend/templates/**/*'
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'docs/**/*'
+          - '**/README.md'
+          - 'packages/frontend/templates/**/*'
 
 test:
-  - 'tests/**/*'
-  - '**/tests/**/*'
-  - '**/__tests__/**/*'
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'tests/**/*'
+          - '**/tests/**/*'
+          - '**/__tests__/**/*'
 
 mod:dev:
-  - 'scripts/**/*'
-  - 'tools/cli/**/*'
-  - 'packages/common/debug/**/*'
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'scripts/**/*'
+          - 'tools/cli/**/*'
+          - 'packages/common/debug/**/*'
 
 mod:plugin:
-  - 'packages/plugins/**/*'
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/plugins/**/*'
 
 plugin:copilot:
-  - 'packages/plugins/copilot/**/*'
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/plugins/copilot/**/*'
 
 mod:infra:
-  - 'packages/common/infra/**/*'
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/common/infra/**/*'
 
 mod:sdk:
-  - 'packages/common/sdk/**/*'
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/common/sdk/**/*'
 
 mod:plugin-cli:
-  - 'tools/plugin-cli/**/*'
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'tools/plugin-cli/**/*'
 
-mod:workspace: 'packages/frontend/workspace/**/*'
+mod:workspace:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/frontend/workspace/**/*'
 
-mod:i18n: 'packages/frontend/i18n/**/*'
+mod:i18n:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/frontend/i18n/**/*'
 
-mod:env: 'packages/common/env/**/*'
+mod:env:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/common/env/**/*'
 
-mod:hooks: 'packages/frontend/hooks/**/*'
+mod:hooks:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/frontend/hooks/**/*'
 
-mod:component: 'packages/frontend/component/**/*'
+mod:component:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/frontend/component/**/*'
 
-mod:storage: 'packages/backend/storage/**/*'
+mod:storage:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/backend/storage/**/*'
 
-mod:native: 'packages/frontend/native/**/*'
+mod:native:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/frontend/native/**/*'
 
 mod:store:
-  - '**/atoms/**/*'
+  - changed-files:
+      - any-glob-to-any-file:
+          - '**/atoms/**/*'
 
 rust:
-  - '**/*.rs'
-  - '**/Cargo.toml'
-  - '**/Cargo.lock'
-  - '**/rust-toolchain'
-  - '**/rust-toolchain.toml'
-  - '**/rustfmt.toml'
+  - changed-files:
+      - any-glob-to-any-file:
+          - '**/*.rs'
+          - '**/Cargo.toml'
+          - '**/Cargo.lock'
+          - '**/rust-toolchain'
+          - '**/rust-toolchain.toml'
+          - '**/rustfmt.toml'
 
-package:y-indexeddb: 'packages/common/y-indexeddb/**/*'
+package:y-indexeddb:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/common/y-indexeddb/**/*'
 
-app:core: 'packages/frontend/core/**/*'
+app:core:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/frontend/core/**/*'
 
-app:electron: 'packages/frontend/electron/**/*'
+app:electron:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/frontend/electron/**/*'
 
-app:server: 'packages/backend/server/**/*'
+app:server:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'packages/backend/server/**/*'

.github/renovate.json

@@ -0,0 +1,53 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base",
+    "group:allNonMajor",
+    ":preserveSemverRanges",
+    ":disablePeerDependencies"
+  ],
+  "labels": ["dependencies"],
+  "packageRules": [
+    {
+      "matchPackageNames": ["napi", "napi-build", "napi-derive"],
+      "groupName": "napi-rs"
+    },
+    {
+      "matchPackagePatterns": ["^eslint", "^@typescript-eslint"],
+      "groupName": "linter"
+    },
+    {
+      "matchPackagePatterns": ["^@nestjs"],
+      "groupName": "nestjs"
+    },
+    {
+      "matchPackagePatterns": ["^@opentelemetry"],
+      "groupName": "opentelemetry"
+    },
+    {
+      "matchPackageNames": [
+        "@prisma/client",
+        "@prisma/instrumentation",
+        "prisma"
+      ],
+      "groupName": "prisma"
+    },
+    {
+      "matchPackagePatterns": ["^@electron-forge"],
+      "groupName": "electron-forge"
+    },
+    {
+      "matchPackagePatterns": ["^@blocksuite"],
+      "excludePackageNames": ["@blocksuite/icons"],
+      "followTag": "nightly"
+    }
+  ],
+  "commitMessagePrefix": "chore: ",
+  "commitMessageAction": "bump up",
+  "commitMessageTopic": "{{depName}} version",
+  "ignoreDeps": [],
+  "lockFileMaintenance": {
+    "enabled": true,
+    "extends": ["schedule:weekly"]
+  }
+}

.github/workflows/auto-labeler.yml

@@ -9,4 +9,5 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/labeler@v4
+      - uses: actions/checkout@v4
+      - uses: actions/labeler@v5

.github/workflows/build-desktop.yml

@@ -1,189 +0,0 @@
-name: Build(Desktop) & Test
-
-on:
-  push:
-    branches:
-      - master
-      - v[0-9]+.[0-9]+.x-staging
-      - v[0-9]+.[0-9]+.x
-    paths-ignore:
-      - README.md
-      - .github/**
-      - '!.github/workflows/build-desktop.yml'
-      - '!.github/actions/build-rust/action.yml'
-      - '!.github/actions/setup-node/action.yml'
-  pull_request:
-  merge_group:
-    branches:
-      - master
-      - v[0-9]+.[0-9]+.x-staging
-      - v[0-9]+.[0-9]+.x
-    paths-ignore:
-      - README.md
-      - .github/**
-      - '!.github/workflows/build-desktop.yml'
-      - '!.github/actions/build-rust/action.yml'
-      - '!.github/actions/setup-node/action.yml'
-
-env:
-  DEBUG: napi:*
-  BUILD_TYPE: canary
-  APP_NAME: affine
-  COVERAGE: true
-  DISTRIBUTION: desktop
-  MACOSX_DEPLOYMENT_TARGET: '10.13'
-  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-jobs:
-  build-core:
-    name: Build @affine/core
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          electron-install: false
-      - name: Build Core
-        run: yarn nx build @affine/core
-      - name: Upload core artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: core
-          path: ./packages/frontend/core/dist
-          if-no-files-found: error
-
-  build-native:
-    name: Build Native
-    runs-on: ubuntu-latest
-    needs: build-core
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-      - name: Build AFFiNE native
-        uses: ./.github/actions/build-rust
-        with:
-          target: x86_64-unknown-linux-gnu
-          package: '@affine/native'
-          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-      - name: Run tests
-        run: yarn test
-        working-directory: ./packages/frontend/native
-
-  desktop-test:
-    name: Desktop Test
-    runs-on: ${{ matrix.spec.os }}
-    strategy:
-      fail-fast: false
-      # all combinations: macos-latest x64, macos-latest arm64, windows-latest x64, ubuntu-latest x64
-      matrix:
-        spec:
-          - {
-              os: macos-latest,
-              platform: macos,
-              arch: x64,
-              target: x86_64-apple-darwin,
-              test: true,
-            }
-          - {
-              os: macos-latest,
-              platform: macos,
-              arch: arm64,
-              target: aarch64-apple-darwin,
-              test: false,
-            }
-          - {
-              os: ubuntu-latest,
-              platform: linux,
-              arch: x64,
-              target: x86_64-unknown-linux-gnu,
-              test: true,
-            }
-          - {
-              os: windows-latest,
-              platform: windows,
-              arch: x64,
-              target: x86_64-pc-windows-msvc,
-              test: true,
-            }
-    needs: build-core
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        timeout-minutes: 10
-        with:
-          extra-flags: workspaces focus @affine/electron @affine/monorepo @affine-test/affine-desktop
-          playwright-install: true
-          hard-link-nm: false
-          enableScripts: false
-
-      - name: Build AFFiNE native
-        uses: ./.github/actions/build-rust
-        with:
-          target: ${{ matrix.spec.target }}
-          package: '@affine/native'
-          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-      - name: Run unit tests
-        if: ${{ matrix.spec.test }}
-        shell: bash
-        run: yarn vitest
-        working-directory: packages/frontend/electron
-
-      - name: Download core artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: core
-          path: packages/frontend/electron/resources/web-static
-
-      - name: Build Desktop Layers
-        run: yarn workspace @affine/electron build
-
-      - name: Run desktop tests
-        if: ${{ matrix.spec.test && matrix.spec.os == 'ubuntu-latest' }}
-        run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine-test/affine-desktop e2e
-        env:
-          COVERAGE: true
-
-      - name: Run desktop tests
-        if: ${{ matrix.spec.test && matrix.spec.os != 'ubuntu-latest' }}
-        run: yarn workspace @affine-test/affine-desktop e2e
-        env:
-          COVERAGE: true
-
-      - name: Make bundle
-        if: ${{ matrix.spec.os == 'macos-latest' && matrix.spec.arch == 'arm64' }}
-        env:
-          SKIP_BUNDLE: true
-          SKIP_WEB_BUILD: true
-        run: yarn workspace @affine/electron make --platform=darwin --arch=arm64
-
-      - name: Output check
-        if: ${{ matrix.spec.os == 'macos-latest' && matrix.spec.arch == 'arm64' }}
-        run: |
-          yarn workspace @affine/electron ts-node ./scripts/macos-arm64-output-check.ts
-
-      - name: Collect code coverage report
-        if: ${{ matrix.spec.test }}
-        run: yarn exec nyc report -t .nyc_output --report-dir .coverage --reporter=lcov
-
-      - name: Upload e2e test coverage results
-        if: ${{ matrix.spec.test }}
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./.coverage/lcov.info
-          flags: e2etest-${{ matrix.spec.os }}-${{ matrix.spec.arch }}
-          name: affine
-          fail_ci_if_error: false
-
-      - name: Upload test results
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results-e2e-${{ matrix.spec.os }}-${{ matrix.spec.arch }}
-          path: ./test-results
-          if-no-files-found: ignore

.github/workflows/build-server.yml

@@ -1,311 +0,0 @@
-name: Build(Server) & Test
-
-on:
-  push:
-    branches:
-      - master
-      - v[0-9]+.[0-9]+.x-staging
-      - v[0-9]+.[0-9]+.x
-    paths-ignore:
-      - README.md
-      - .github/**
-      - '!.github/workflows/build-server.yml'
-      - '!.github/actions/build-rust/action.yml'
-      - '!.github/actions/setup-node/action.yml'
-  pull_request:
-  merge_group:
-    branches:
-      - master
-      - v[0-9]+.[0-9]+.x-staging
-      - v[0-9]+.[0-9]+.x
-    paths-ignore:
-      - README.md
-      - .github/**
-      - '!.github/workflows/build-server.yml'
-      - '!.github/actions/build-rust/action.yml'
-      - '!.github/actions/setup-node/action.yml'
-
-env:
-  DEBUG: napi:*
-  BUILD_TYPE: canary
-  APP_NAME: affine
-  COVERAGE: true
-  DISTRIBUTION: browser
-  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-jobs:
-  build-storage:
-    name: Build Storage
-    runs-on: ubuntu-latest
-    env:
-      RUSTFLAGS: '-C debuginfo=1'
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          extra-flags: workspaces focus @affine/storage
-          electron-install: false
-          build-infra: false
-          build-plugins: false
-      - name: Build Rust
-        uses: ./.github/actions/build-rust
-        with:
-          target: 'x86_64-unknown-linux-gnu'
-          package: '@affine/storage'
-          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-      - name: Upload storage.node
-        uses: actions/upload-artifact@v3
-        with:
-          name: storage.node
-          path: ./packages/backend/storage/storage.node
-          if-no-files-found: error
-
-  server-test:
-    name: Server Test
-    runs-on: ubuntu-latest
-    needs: build-storage
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: affine
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-      mailer:
-        image: mailhog/mailhog
-        ports:
-          - 1025:1025
-          - 8025:8025
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          electron-install: false
-
-      - name: Initialize database
-        run: |
-          psql -h localhost -U postgres -c "CREATE DATABASE affine;"
-          psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
-          psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
-        env:
-          PGPASSWORD: affine
-
-      - name: Generate prisma client
-        run: |
-          yarn workspace @affine/server exec prisma generate
-          yarn workspace @affine/server exec prisma db push
-        env:
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
-
-      - name: Run init-db script
-        run: yarn workspace @affine/server exec ts-node ./scripts/init-db.ts
-        env:
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
-
-      - name: Download storage.node
-        uses: actions/download-artifact@v3
-        with:
-          name: storage.node
-          path: ./packages/backend/server
-
-      - name: Run server tests
-        run: yarn workspace @affine/server test:coverage
-        env:
-          CARGO_TARGET_DIR: '${{ github.workspace }}/target'
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
-
-      - name: Upload server test coverage results
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./packages/backend/server/.coverage/lcov.info
-          flags: server-test
-          name: affine
-          fail_ci_if_error: false
-
-  server-e2e-test:
-    name: Server E2E Test
-    runs-on: ubuntu-latest
-    needs: build-storage
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: affine
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-      mailer:
-        image: mailhog/mailhog
-        ports:
-          - 1025:1025
-          - 8025:8025
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          playwright-install: true
-
-      - name: Initialize database
-        run: |
-          psql -h localhost -U postgres -c "CREATE DATABASE affine;"
-          psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
-          psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
-        env:
-          PGPASSWORD: affine
-
-      - name: Generate prisma client
-        run: |
-          yarn workspace @affine/server exec prisma generate
-          yarn workspace @affine/server exec prisma db push
-        env:
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
-
-      - name: Run init-db script
-        run: yarn workspace @affine/server exec ts-node ./scripts/init-db.ts
-        env:
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
-
-      - name: Download storage.node
-        uses: actions/download-artifact@v3
-        with:
-          name: storage.node
-          path: ./packages/backend/server
-
-      - name: Run playwright tests
-        run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine-test/affine-cloud e2e --forbid-only
-        env:
-          COVERAGE: true
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
-
-      - name: Collect code coverage report
-        run: yarn exec nyc report -t .nyc_output --report-dir .coverage --reporter=lcov
-
-      - name: Upload e2e test coverage results
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./.coverage/lcov.info
-          flags: server-e2etest
-          name: affine
-          fail_ci_if_error: false
-
-      - name: Upload test results
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results-e2e-server
-          path: ./tests/affine-cloud/test-results
-          if-no-files-found: ignore
-
-  server-desktop-e2e-test:
-    name: Server Desktop E2E Test
-    runs-on: ubuntu-latest
-    needs: build-storage
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: affine
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-      mailer:
-        image: mailhog/mailhog
-        ports:
-          - 1025:1025
-          - 8025:8025
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          playwright-install: true
-          hard-link-nm: false
-
-      - name: Build AFFiNE native
-        uses: ./.github/actions/build-rust
-        with:
-          target: x86_64-unknown-linux-gnu
-          package: '@affine/native'
-          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-      - name: Initialize database
-        run: |
-          psql -h localhost -U postgres -c "CREATE DATABASE affine;"
-          psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
-          psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
-        env:
-          PGPASSWORD: affine
-
-      - name: Generate prisma client
-        run: |
-          yarn workspace @affine/server exec prisma generate
-          yarn workspace @affine/server prisma db push
-        env:
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
-
-      - name: Run init-db script
-        run: yarn workspace @affine/server exec ts-node ./scripts/init-db.ts
-        env:
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
-
-      - name: Download storage.node
-        uses: actions/download-artifact@v3
-        with:
-          name: storage.node
-          path: ./packages/backend/server
-
-      - name: Build Plugins
-        run: yarn run build:plugins
-
-      - name: Build Desktop Layers
-        run: yarn workspace @affine/electron build:dev
-
-      - name: Run playwright tests
-        run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" yarn workspace @affine-test/affine-desktop-cloud e2e
-        env:
-          COVERAGE: true
-          DEV_SERVER_URL: http://localhost:8080
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
-          ENABLE_LOCAL_EMAIL: true
-
-      - name: Collect code coverage report
-        run: yarn exec nyc report -t .nyc_output --report-dir .coverage --reporter=lcov
-
-      - name: Upload e2e test coverage results
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./.coverage/lcov.info
-          flags: server-e2etest
-          name: affine
-          fail_ci_if_error: false
-
-      - name: Upload test results
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results-e2e-server
-          path: ./tests/affine-cloud/test-results
-          if-no-files-found: ignore

.github/workflows/build-test.yml

@@ -0,0 +1,599 @@
+name: Build & Test
+
+on:
+  push:
+    branches:
+      - canary
+      - v[0-9]+.[0-9]+.x-staging
+      - v[0-9]+.[0-9]+.x
+    paths-ignore:
+      - README.md
+  pull_request:
+
+env:
+  DEBUG: napi:*
+  BUILD_TYPE: canary
+  APP_NAME: affine
+  AFFINE_ENV: dev
+  COVERAGE: true
+  MACOSX_DEPLOYMENT_TARGET: '10.13'
+  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+  PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/node_modules/.cache/ms-playwright
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ['javascript', 'typescript']
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+
+          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+      #   If the Autobuild fails above, remove it and uncomment the following three lines.
+      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+      # - run: |
+      #   echo "Run, Build Application using script"
+      #   ./location_of_script_within_repo/buildscript.sh
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run oxlint
+        # oxlint is fast, so wrong code will fail quickly
+        run: yarn dlx $(node -e "console.log(require('./package.json').scripts['lint:ox'])")
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          electron-install: false
+          full-cache: true
+      - name: Run i18n codegen
+        run: yarn i18n-codegen gen
+      - name: Run ESLint
+        run: yarn lint:eslint --max-warnings=0
+      - name: Run Prettier
+        # Set nmMode in `actions/setup-node` will modify the .yarnrc.yml
+        run: |
+          git checkout .yarnrc.yml
+          yarn lint:prettier
+      - name: Run Type Check
+        run: yarn typecheck
+
+  check-yarn-binary:
+    name: Check yarn binary
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run check
+        run: |
+          yarn set version $(node -e "console.log(require('./package.json').packageManager.split('@')[1])")
+          git diff --exit-code
+
+  e2e-plugin-test:
+    name: E2E Plugin Test
+    runs-on: ubuntu-latest
+    env:
+      DISTRIBUTION: browser
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          playwright-install: true
+          electron-install: false
+          full-cache: true
+      - name: Run playwright tests
+        run: yarn e2e --forbid-only
+        working-directory: tests/affine-plugin
+        env:
+          COVERAGE: true
+      - name: Collect code coverage report
+        run: yarn exec nyc report -t .nyc_output --report-dir .coverage --reporter=lcov
+
+      - name: Upload e2e test coverage results
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./.coverage/lcov.info
+          flags: e2e-plugin-test
+          name: affine
+          fail_ci_if_error: false
+
+      - name: Upload test results
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results-e2e-plugin
+          path: ./test-results
+          if-no-files-found: ignore
+
+  e2e-test:
+    name: E2E Test
+    runs-on: ubuntu-latest
+    env:
+      DISTRIBUTION: browser
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4, 5]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          playwright-install: true
+          electron-install: false
+          full-cache: true
+
+      - name: Run playwright tests
+        run: yarn workspace @affine-test/affine-local e2e --forbid-only --shard=${{ matrix.shard }}/${{ strategy.job-total }}
+
+      - name: Upload test results
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results-e2e-${{ matrix.shard }}
+          path: ./test-results
+          if-no-files-found: ignore
+
+  e2e-migration-test:
+    name: E2E Migration Test
+    runs-on: ubuntu-latest
+    env:
+      DISTRIBUTION: browser
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          playwright-install: true
+          electron-install: false
+          full-cache: true
+
+      - name: Run playwright tests
+        run: yarn workspace @affine-test/affine-migration e2e --forbid-only
+
+      - name: Upload test results
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results-e2e-migration
+          path: ./tests/affine-migration/test-results
+          if-no-files-found: ignore
+
+  unit-test:
+    name: Unit Test
+    runs-on: ubuntu-latest
+    needs:
+      - build-native
+    env:
+      DISTRIBUTION: browser
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          electron-install: false
+          full-cache: true
+
+      - name: Download affine.linux-x64-gnu.node
+        uses: actions/download-artifact@v3
+        with:
+          name: affine.linux-x64-gnu.node
+          path: ./packages/frontend/native
+
+      - name: Unit Test
+        run: yarn nx test:coverage @affine/monorepo
+
+      - name: Upload unit test coverage results
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./.coverage/store/lcov.info
+          flags: unittest
+          name: affine
+          fail_ci_if_error: false
+
+  build-native:
+    name: Build AFFiNE native (${{ matrix.spec.target }})
+    runs-on: ${{ matrix.spec.os }}
+    env:
+      CARGO_PROFILE_RELEASE_DEBUG: '1'
+    strategy:
+      fail-fast: false
+      matrix:
+        spec:
+          - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu }
+          - { os: windows-latest, target: x86_64-pc-windows-msvc }
+          - { os: macos-latest, target: x86_64-apple-darwin }
+          - { os: macos-latest, target: aarch64-apple-darwin }
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          extra-flags: workspaces focus @affine/native
+          electron-install: false
+          build-infra: false
+          build-plugins: false
+      - name: Setup filename
+        id: filename
+        shell: bash
+        run: |
+          export PLATFORM_ARCH_ABI=$(node -e "console.log(require('@napi-rs/cli').parseTriple('${{ matrix.spec.target }}').platformArchABI)")
+          echo "filename=affine.$PLATFORM_ARCH_ABI.node" >> "$GITHUB_OUTPUT"
+      - name: Build AFFiNE native
+        uses: ./.github/actions/build-rust
+        with:
+          target: ${{ matrix.spec.target }}
+          package: '@affine/native'
+          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+      - name: Upload ${{ steps.filename.outputs.filename }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ steps.filename.outputs.filename }}
+          path: ./packages/frontend/native/${{ steps.filename.outputs.filename }}
+          if-no-files-found: error
+
+  build-storage:
+    name: Build Storage
+    runs-on: ubuntu-latest
+    env:
+      CARGO_PROFILE_RELEASE_DEBUG: '1'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          extra-flags: workspaces focus @affine/storage
+          electron-install: false
+          build-infra: false
+          build-plugins: false
+      - name: Build Rust
+        uses: ./.github/actions/build-rust
+        with:
+          target: 'x86_64-unknown-linux-gnu'
+          package: '@affine/storage'
+          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+      - name: Upload storage.node
+        uses: actions/upload-artifact@v3
+        with:
+          name: storage.node
+          path: ./packages/backend/storage/storage.node
+          if-no-files-found: error
+
+  build-core:
+    name: Build @affine/core
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          electron-install: false
+          build-plugins: false
+          full-cache: true
+      - name: Build Core
+        # always skip cache because its fast, and cache configuration is always changing
+        run: yarn nx build @affine/core --skip-nx-cache
+      - name: zip core
+        run: tar -czf dist.tar.gz --directory=packages/frontend/core/dist .
+      - name: Upload core artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: core
+          path: dist.tar.gz
+          if-no-files-found: error
+
+  server-test:
+    name: Server Test
+    runs-on: ubuntu-latest
+    needs: build-storage
+    env:
+      DISTRIBUTION: browser
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: affine
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+      mailer:
+        image: mailhog/mailhog
+        ports:
+          - 1025:1025
+          - 8025:8025
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          electron-install: false
+          full-cache: true
+
+      - name: Initialize database
+        run: |
+          psql -h localhost -U postgres -c "CREATE DATABASE affine;"
+          psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
+          psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
+        env:
+          PGPASSWORD: affine
+
+      - name: Generate prisma client
+        run: |
+          yarn workspace @affine/server exec prisma generate
+          yarn workspace @affine/server exec prisma db push
+        env:
+          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+
+      - name: Run init-db script
+        run: yarn workspace @affine/server exec node --loader ts-node/esm/transpile-only ./scripts/init-db.ts
+        env:
+          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+
+      - name: Download storage.node
+        uses: actions/download-artifact@v3
+        with:
+          name: storage.node
+          path: ./packages/backend/server
+
+      - name: Run server tests
+        run: yarn workspace @affine/server test:coverage
+        env:
+          CARGO_TARGET_DIR: '${{ github.workspace }}/target'
+          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+
+      - name: Upload server test coverage results
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./packages/backend/server/.coverage/lcov.info
+          flags: server-test
+          name: affine
+          fail_ci_if_error: false
+
+  server-e2e-test:
+    name: ${{ matrix.tests.name }}
+    runs-on: ubuntu-latest
+    env:
+      DISTRIBUTION: browser
+      DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+    strategy:
+      fail-fast: false
+      matrix:
+        tests:
+          - name: 'Server E2E Test 1/3'
+            script: yarn workspace @affine-test/affine-cloud e2e --forbid-only --shard=1/3
+          - name: 'Server E2E Test 2/3'
+            script: yarn workspace @affine-test/affine-cloud e2e --forbid-only --shard=2/3
+          - name: 'Server E2E Test 3/3'
+            script: yarn workspace @affine-test/affine-cloud e2e --forbid-only --shard=3/3
+          - name: 'Server Desktop E2E Test'
+            script: |
+              yarn workspace @affine/electron build:dev
+              xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine-test/affine-desktop-cloud e2e
+    needs:
+      - build-storage
+      - build-native
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: affine
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+      mailer:
+        image: mailhog/mailhog
+        ports:
+          - 1025:1025
+          - 8025:8025
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          playwright-install: true
+          hard-link-nm: false
+
+      - name: Initialize database
+        run: |
+          psql -h localhost -U postgres -c "CREATE DATABASE affine;"
+          psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
+          psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
+        env:
+          PGPASSWORD: affine
+
+      - name: Generate prisma client
+        run: |
+          yarn workspace @affine/server exec prisma generate
+          yarn workspace @affine/server exec prisma db push
+        env:
+          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+
+      - name: Run init-db script
+        run: yarn workspace @affine/server exec node --loader ts-node/esm/transpile-only ./scripts/init-db.ts
+      - name: Download storage.node
+        uses: actions/download-artifact@v3
+        with:
+          name: storage.node
+          path: ./packages/backend/server
+
+      - name: Download affine.linux-x64-gnu.node
+        uses: actions/download-artifact@v3
+        with:
+          name: affine.linux-x64-gnu.node
+          path: ./packages/frontend/native
+
+      - name: ${{ matrix.tests.name }}
+        run: |
+          ${{ matrix.tests.script }}
+        env:
+          DEV_SERVER_URL: http://localhost:8080
+          ENABLE_LOCAL_EMAIL: true
+
+      - name: Upload test results
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results-e2e-server
+          path: ./tests/affine-cloud/test-results
+          if-no-files-found: ignore
+
+  desktop-test:
+    name: Desktop Test (${{ matrix.spec.os }}, ${{ matrix.spec.platform }}, ${{ matrix.spec.arch }}, ${{ matrix.spec.target }}, ${{ matrix.spec.test }})
+    runs-on: ${{ matrix.spec.os }}
+    strategy:
+      fail-fast: false
+      # all combinations: macos-latest x64, macos-latest arm64, windows-latest x64, ubuntu-latest x64
+      matrix:
+        spec:
+          - {
+              os: macos-latest,
+              platform: macos,
+              arch: x64,
+              target: x86_64-apple-darwin,
+              test: true,
+            }
+          - {
+              os: macos-latest,
+              platform: macos,
+              arch: arm64,
+              target: aarch64-apple-darwin,
+              test: false,
+            }
+          - {
+              os: ubuntu-latest,
+              platform: linux,
+              arch: x64,
+              target: x86_64-unknown-linux-gnu,
+              test: true,
+            }
+          - {
+              os: windows-latest,
+              platform: windows,
+              arch: x64,
+              target: x86_64-pc-windows-msvc,
+              test: true,
+            }
+    needs:
+      - build-core
+      - build-native
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        timeout-minutes: 10
+        with:
+          extra-flags: workspaces focus @affine/electron @affine/monorepo @affine-test/affine-desktop
+          playwright-install: true
+          hard-link-nm: false
+          enableScripts: false
+
+      - name: Setup filename
+        id: filename
+        shell: bash
+        run: |
+          export PLATFORM_ARCH_ABI=$(node -e "console.log(require('@napi-rs/cli').parseTriple('${{ matrix.spec.target }}').platformArchABI)")
+          echo "filename=affine.$PLATFORM_ARCH_ABI.node" >> "$GITHUB_OUTPUT"
+
+      - name: Download ${{ steps.filename.outputs.filename }}
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ steps.filename.outputs.filename }}
+          path: ./packages/frontend/native
+
+      - name: Run unit tests
+        if: ${{ matrix.spec.test }}
+        shell: bash
+        run: yarn vitest
+        working-directory: packages/frontend/electron
+
+      - name: Download core artifact
+        uses: ./.github/actions/download-core
+        with:
+          path: packages/frontend/electron/resources/web-static
+
+      - name: Build Desktop Layers
+        run: yarn workspace @affine/electron build
+
+      - name: Run desktop tests
+        if: ${{ matrix.spec.test && matrix.spec.os == 'ubuntu-latest' }}
+        run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine-test/affine-desktop e2e
+
+      - name: Run desktop tests
+        if: ${{ matrix.spec.test && matrix.spec.os != 'ubuntu-latest' }}
+        run: yarn workspace @affine-test/affine-desktop e2e
+
+      - name: Make bundle
+        if: ${{ matrix.spec.os == 'macos-latest' && matrix.spec.arch == 'arm64' }}
+        env:
+          SKIP_BUNDLE: true
+          SKIP_WEB_BUILD: true
+          HOIST_NODE_MODULES: 1
+        run: yarn workspace @affine/electron package --platform=darwin --arch=arm64
+
+      - name: Output check
+        if: ${{ matrix.spec.os == 'macos-latest' && matrix.spec.arch == 'arm64' }}
+        run: |
+          yarn workspace @affine/electron exec node --loader ts-node/esm/transpile-only ./scripts/macos-arm64-output-check.ts
+
+      - name: Upload test results
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results-e2e-${{ matrix.spec.os }}-${{ matrix.spec.arch }}
+          path: ./test-results
+          if-no-files-found: ignore

.github/workflows/build.yml

@@ -1,201 +0,0 @@
-name: Build & Test
-
-on:
-  push:
-    branches:
-      - master
-      - v[0-9]+.[0-9]+.x-staging
-      - v[0-9]+.[0-9]+.x
-    paths-ignore:
-      - README.md
-      - .github/**
-      - '!.github/workflows/build.yml'
-      - '!.github/actions/build-rust/action.yml'
-      - '!.github/actions/setup-node/action.yml'
-  pull_request:
-  merge_group:
-    branches:
-      - master
-      - v[0-9]+.[0-9]+.x-staging
-      - v[0-9]+.[0-9]+.x
-    paths-ignore:
-      - README.md
-      - .github/**
-      - '!.github/workflows/build.yml'
-      - '!.github/actions/build-rust/action.yml'
-      - '!.github/actions/setup-node/action.yml'
-
-env:
-  DEBUG: napi:*
-  BUILD_TYPE: canary
-  APP_NAME: affine
-  AFFINE_ENV: dev
-  COVERAGE: true
-  DISTRIBUTION: browser
-  MACOSX_DEPLOYMENT_TARGET: '10.13'
-  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run oxlint
-        # oxlint is fast, so wrong code will fail quickly
-        run: yarn dlx oxlint@latest .
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          electron-install: false
-      - name: Run i18n codegen
-        run: yarn i18n-codegen gen
-      - name: Run ESLint
-        run: yarn lint:eslint --max-warnings=0
-      - name: Run Prettier
-        # Set nmMode in `actions/setup-node` will modify the .yarnrc.yml
-        run: |
-          git checkout .yarnrc.yml
-          yarn lint:prettier
-      - name: Run circular
-        run: yarn circular
-      - name: Run Type Check
-        run: yarn typecheck
-
-  check-yarn-binary:
-    name: Check yarn binary
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run check
-        run: |
-          yarn set version $(node -e "console.log(require('./package.json').packageManager.split('@')[1])")
-          git diff --exit-code
-
-  e2e-plugin-test:
-    name: E2E Plugin Test
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          playwright-install: true
-          electron-install: false
-      - name: Run playwright tests
-        run: yarn e2e --forbid-only
-        working-directory: tests/affine-plugin
-        env:
-          COVERAGE: true
-      - name: Collect code coverage report
-        run: yarn exec nyc report -t .nyc_output --report-dir .coverage --reporter=lcov
-
-      - name: Upload e2e test coverage results
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./.coverage/lcov.info
-          flags: e2e-plugin-test
-          name: affine
-          fail_ci_if_error: false
-
-      - name: Upload test results
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results-e2e-plugin
-          path: ./test-results
-          if-no-files-found: ignore
-
-  e2e-test:
-    name: E2E Test
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        shard: [1, 2, 3, 4, 5]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          playwright-install: true
-          electron-install: false
-
-      - name: Run playwright tests
-        run: yarn e2e --forbid-only --shard=${{ matrix.shard }}/${{ strategy.job-total }}
-        working-directory: tests/affine-local
-        env:
-          COVERAGE: true
-
-      - name: Collect code coverage report
-        run: yarn exec nyc report -t .nyc_output --report-dir .coverage --reporter=lcov
-
-      - name: Upload e2e test coverage results
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./.coverage/lcov.info
-          flags: e2etest
-          name: affine
-          fail_ci_if_error: false
-
-      - name: Upload test results
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results-e2e-${{ matrix.shard }}
-          path: ./test-results
-          if-no-files-found: ignore
-
-  e2e-migration-test:
-    name: E2E Migration Test
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          playwright-install: true
-          electron-install: false
-
-      - name: Run playwright tests
-        run: yarn workspace @affine-test/affine-migration e2e --forbid-only
-
-      - name: Upload test results
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results-e2e-migration
-          path: ./tests/affine-migration/test-results
-          if-no-files-found: ignore
-
-  unit-test:
-    name: Unit Test
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          electron-install: false
-
-      - name: Build AFFiNE native
-        uses: ./.github/actions/build-rust
-        with:
-          target: x86_64-unknown-linux-gnu
-          package: '@affine/native'
-          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-      - name: Unit Test
-        run: yarn nx test:coverage @affine/monorepo
-
-      - name: Upload unit test coverage results
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./.coverage/store/lcov.info
-          flags: unittest
-          name: affine
-          fail_ci_if_error: false

.github/workflows/cache-cleanup.yml

@@ -1,36 +0,0 @@
-# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
-name: Cleanup caches for closed branches
-
-on:
-  pull_request:
-    types:
-      - closed
-  workflow_dispatch:
-
-jobs:
-  cleanup:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v4
-
-      - name: Cleanup
-        run: |
-          gh extension install actions/gh-actions-cache
-
-          REPO=${{ github.repository }}
-          BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
-
-          echo "Fetching list of cache key"
-          cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 )
-
-          ## Setting this to not fail the workflow while deleting cache keys.
-          set +e
-          echo "Deleting caches..."
-          for cacheKey in $cacheKeysForPR
-          do
-              gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
-          done
-          echo "Done"
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/cancel.yml

@@ -1,18 +0,0 @@
-name: Cancel
-on:
-  pull_request_target:
-    types:
-      - edited
-      - synchronize
-
-jobs:
-  cancel:
-    name: 'Cancel Previous Runs'
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    steps:
-      - uses: styfle/cancel-workflow-action@0.12.0
-        with:
-          # See https://api.github.com/repos/toeverything/AFFiNE/actions/workflows
-          workflow_id: 44038251, 61883931, 65188160, 66789140
-          access_token: ${{ github.token }}

.github/workflows/codeql.yml

@@ -1,70 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: 'CodeQL'
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-  merge_group:
-    # The branches below must be a subset of the branches above
-    branches: [master]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ['javascript']
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-
-          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-          # queries: security-extended,security-and-quality
-
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
-
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-      #   If the Autobuild fails above, remove it and uncomment the following three lines.
-      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-      # - run: |
-      #   echo "Run, Build Application using script"
-      #   ./location_of_script_within_repo/buildscript.sh
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2

.github/workflows/deploy.yml

@@ -9,7 +9,6 @@ on:
         default: canary
 
 env:
-  BUILD_TYPE: canary
   APP_NAME: affine
   NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
 
@@ -17,7 +16,6 @@ jobs:
   build-server:
     name: Build Server
     runs-on: ubuntu-latest
-    environment: ${{ github.event.inputs.flavor }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Node.js
@@ -35,7 +33,7 @@ jobs:
   build-core:
     name: Build @affine/core
     runs-on: ubuntu-latest
-
+    environment: ${{ github.event.inputs.flavor }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Node.js
@@ -43,15 +41,19 @@ jobs:
       - name: Build Plugins
         run: yarn run build:plugins
       - name: Build Core
-        run: yarn nx build @affine/core
+        run: yarn nx build @affine/core --skip-nx-cache
         env:
           R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-          BUILD_TYPE_OVERRIDE: ${{ github.event.inputs.flavor }}
+          BUILD_TYPE: ${{ github.event.inputs.flavor }}
           SHOULD_REPORT_TRACE: true
           TRACE_REPORT_ENDPOINT: ${{ secrets.TRACE_REPORT_ENDPOINT }}
           CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
+          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+          SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
       - name: Upload core artifact
         uses: actions/upload-artifact@v3
         with:
@@ -67,7 +69,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
-      - name: Setup Rust
+      - name: Build Rust
         uses: ./.github/actions/build-rust
         with:
           target: 'x86_64-unknown-linux-gnu'
@@ -88,7 +90,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
-      - name: Setup Rust
+      - name: Build Rust
         uses: ./.github/actions/build-rust
         with:
           target: 'aarch64-unknown-linux-gnu'
@@ -205,7 +207,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Deploy to dev
+      - name: setup deploy version
+        id: version
+        run: |
+          export APP_VERSION=`node -e "console.log(require('./package.json').version)"`
+          echo $APP_VERSION
+          echo "APP_VERSION=$APP_VERSION" >> "$GITHUB_OUTPUT"
+      - name: Deploy to ${{ github.event.inputs.flavor }}
         uses: ./.github/actions/deploy
         with:
           build-type: ${{ github.event.inputs.flavor }}
@@ -215,6 +223,7 @@ jobs:
           cluster-name: ${{ secrets.GCP_CLUSTER_NAME }}
           cluster-location: ${{ secrets.GCP_CLUSTER_LOCATION }}
         env:
+          APP_VERSION: ${{ steps.version.outputs.APP_VERSION }}
           DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
           CANARY_DEPLOY_HOST: ${{ secrets.CANARY_DEPLOY_HOST }}
           R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}

.github/workflows/helm-releaser.yml

@@ -2,7 +2,7 @@ name: Release Charts
 
 on:
   push:
-    branches: [master]
+    branches: [canary]
     paths:
       - '.github/helm/**/Chart.yml'
 

.github/workflows/label-checker.yml

@@ -6,7 +6,7 @@ on:
       - labeled
       - unlabeled
     branches:
-      - master
+      - canary
 
 jobs:
   check_labels:

.github/workflows/languages-sync.yml

@@ -2,13 +2,13 @@ name: Languages Sync
 
 on:
   push:
-    branches: ['master']
+    branches: ['canary']
     paths:
       - 'packages/frontend/i18n/**'
       - '.github/workflows/languages-sync.yml'
       - '!.github/actions/setup-node/action.yml'
   pull_request_target:
-    branches: ['master']
+    branches: ['canary']
     paths:
       - 'packages/frontend/i18n/**'
       - '.github/workflows/languages-sync.yml'
@@ -23,13 +23,13 @@ jobs:
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
       - name: Check Language Key
-        if: github.ref != 'refs/heads/master'
+        if: github.ref != 'refs/heads/canary'
         run: yarn workspace @affine/i18n run sync-languages:check
         env:
           TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}
 
       - name: Sync Languages
-        if: github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/canary'
         run: yarn workspace @affine/i18n run sync-languages
         env:
           TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}

.github/workflows/nightly-build.yml

@@ -65,14 +65,15 @@ jobs:
       - name: Replace Version
         run: ./scripts/set-version.sh ${{ needs.set-build-version.outputs.version }}
       - name: generate-assets
-        working-directory: packages/frontend/electron
-        run: yarn generate-assets
+        run: yarn workspace @affine/electron generate-assets
         env:
           SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
           SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
-          NEXT_PUBLIC_SENTRY_DSN: ${{ secrets.NEXT_PUBLIC_SENTRY_DSN }}
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
           RELEASE_VERSION: ${{ needs.set-build-version.outputs.version }}
+          SKIP_PLUGIN_BUILD: 'true'
+          SKIP_NX_CACHE: 'true'
 
       - name: Upload core artifact
         uses: actions/upload-artifact@v3
@@ -230,6 +231,11 @@ jobs:
           node ./packages/frontend/electron/scripts/generate-yml.js
         env:
           RELEASE_VERSION: ${{ needs.set-build-version.outputs.version }}
+      - name: Generate SHA512 checksums
+        run: |
+          sha512sum *-linux-* > SHA512SUMS.txt
+          sha512sum *-macos-* >> SHA512SUMS.txt
+          sha512sum *-windows-* >> SHA512SUMS.txt
       - name: Create Release Draft
         uses: softprops/action-gh-release@v1
         env:
@@ -240,6 +246,7 @@ jobs:
           tag_name: ${{ needs.set-build-version.outputs.version }}
           prerelease: true
           files: |
+            ./SHA512SUMS.txt
             ./VERSION
             ./*.zip
             ./*.dmg

.github/workflows/pr-title-lint.yml

@@ -7,7 +7,7 @@ on:
       - edited
       - synchronize
     branches:
-      - master
+      - canary
 
 permissions:
   contents: read
@@ -19,7 +19,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Setup Node.js
-        uses: ./.github/actions/setup-node
+        uses: actions/setup-node@v4
         with:
-          electron-install: false
-      - run: echo "${{ github.event.pull_request.title }}" | yarn dlx commitlint -g ./.commitlintrc.json
+          cache: 'yarn'
+          node-version-file: '.nvmrc'
+      - name: Install dependencies
+        run: yarn workspaces focus @affine/commitlint-config
+      - run: echo "${{ github.event.pull_request.title }}" | yarn workspace @affine/commitlint-config commitlint -g ./.commitlintrc.json

.github/workflows/publish-storybook.yml

@@ -7,10 +7,10 @@ on:
   workflow_dispatch:
   push:
     branches:
-      - master
+      - canary
   pull_request:
     branches:
-      - master
+      - canary
     paths-ignore:
       - README.md
       - .github/**

.github/workflows/release-desktop-app.yml

@@ -40,6 +40,7 @@ env:
 jobs:
   before-make:
     runs-on: ubuntu-latest
+    environment: ${{ github.event.inputs.build-type || (github.ref_type == 'tag' && contains(github.ref, 'canary') && 'canary') }}
     outputs:
       RELEASE_VERSION: ${{ steps.get-canary-version.outputs.RELEASE_VERSION }}
     steps:
@@ -65,8 +66,10 @@ jobs:
           SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
           SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
           RELEASE_VERSION: ${{ github.event.inputs.version || steps.get-canary-version.outputs.RELEASE_VERSION }}
           SKIP_PLUGIN_BUILD: 'true'
+          SKIP_NX_CACHE: 'true'
 
       - name: Upload core artifact
         uses: actions/upload-artifact@v3

.github/workflows/release.yml

@@ -1,165 +0,0 @@
-name: Release
-
-on:
-  push:
-    branches:
-      - master
-
-env:
-  BUILD_TYPE: stable
-  APP_NAME: affine
-  COVERAGE: false
-  DISTRIBUTION: browser
-  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-jobs:
-  release:
-    name: Try publishing npm@latest release
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-      - name: Try publishing to NPM
-        run: ./scripts/publish.sh
-        env:
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-
-  build-core:
-    name: Build @affine/core
-    runs-on: ubuntu-latest
-    environment: development
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-      - name: Build Plugins
-        run: yarn run build:plugins
-      - name: Build Core
-        run: yarn nx build @affine/core
-      - name: Upload core artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: core
-          path: ./packages/frontend/core/dist
-          if-no-files-found: error
-
-  build-server:
-    name: Build Server
-    runs-on: ubuntu-latest
-    environment: development
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          electron-install: false
-      - name: Build Server
-        run: yarn nx build @affine/server
-      - name: Upload server dist
-        uses: actions/upload-artifact@v3
-        with:
-          name: server-dist
-          path: ./packages/backend/server/dist
-          if-no-files-found: error
-
-  build-storage:
-    name: Build Storage
-    runs-on: ubuntu-latest
-    env:
-      RUSTFLAGS: '-C debuginfo=1'
-    environment: development
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-      - name: Setup Rust
-        uses: ./.github/actions/build-rust
-        with:
-          target: 'x86_64-unknown-linux-gnu'
-          package: '@affine/storage'
-          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-      - name: Upload storage.node
-        uses: actions/upload-artifact@v3
-        with:
-          name: storage.node
-          path: ./packages/backend/storage/storage.node
-          if-no-files-found: error
-
-  build-docker:
-    if: github.ref == 'refs/heads/master'
-    name: Build Docker
-    runs-on: ubuntu-latest
-    needs:
-      - build-server
-      - build-core
-      - build-storage
-    steps:
-      - uses: actions/checkout@v4
-      - name: Download core artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: core
-          path: ./packages/frontend/core/dist
-      - name: Download server dist
-        uses: actions/download-artifact@v3
-        with:
-          name: server-dist
-          path: ./packages/backend/server/dist
-      - name: Download storage.node
-        uses: actions/download-artifact@v3
-        with:
-          name: storage.node
-          path: ./packages/backend/server
-      - name: Setup Git short hash
-        run: |
-          echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          logout: false
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Build front Dockerfile
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          pull: true
-          platforms: linux/amd64,linux/arm64
-          provenance: true
-          file: .github/deployment/front/Dockerfile
-          tags: ghcr.io/toeverything/affine-front:${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-front:latest
-
-      # setup node without cache configuration
-      # Prisma cache is not compatible with docker build cache
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: '.nvmrc'
-          registry-url: https://npm.pkg.github.com
-          scope: '@toeverything'
-
-      - name: Install Node.js dependencies
-        run: yarn workspaces focus @affine/server --production
-
-      - name: Generate Prisma client
-        run: yarn workspace @affine/server prisma generate
-
-      - name: Build graphql Dockerfile
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          pull: true
-          platforms: linux/amd64,linux/arm64
-          provenance: true
-          file: .github/deployment/node/Dockerfile
-          tags: ghcr.io/toeverything/affine-graphql:${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-graphql:latest

.github/workflows/workers.yml

@@ -3,7 +3,7 @@ name: Deploy Cloudflare Worker
 on:
   push:
     branches:
-      - master
+      - canary
     paths:
       - tools/workers/**
 

.gitignore

@@ -78,3 +78,4 @@ tsconfig.node.tsbuildinfo
 lib
 affine.db
 apps/web/next-routes.conf
+.nx

.husky/pre-commit

@@ -1,23 +1,4 @@
 #!/usr/bin/env sh
 . "$(dirname -- "$0")/_/husky.sh"
 
-# check lockfile is up to date
-yarn install --mode=skip-build --inline-builds --immutable
-
-# build infra code
-yarn -T run build:infra
-
-# generate prisma client type
-yarn workspace @affine/server prisma generate
-
-# generate i18n
-yarn i18n-codegen gen
-
-# lint staged files
-yarn exec lint-staged
-
-# type check
-yarn typecheck
-
-# circular dependency check
-yarn circular
+yarn lint-staged && yarn lint:ox

.prettierignore

@@ -15,6 +15,7 @@ packages/backend/server/src/schema.gql
 packages/frontend/i18n/src/i18n-generated.ts
 packages/frontend/graphql/src/graphql/index.ts
 tests/affine-legacy/**/static
+.yarnrc.yml
 
 # auto-generated by NAPI-RS
 # fixme(@joooye34): need script to check and generate ignore list here

.yarn/patches/@electron-forge-core-npm-6.4.2-ab60c87e75.patch

@@ -1,13 +0,0 @@
-diff --git a/dist/util/forge-config.js b/dist/util/forge-config.js
-index 3466ac1a340c8dfe5ea8997178961e8328457d68..ceb33770db48df80e4355e6bac12e8c99162d7bc 100644
---- a/dist/util/forge-config.js
-+++ b/dist/util/forge-config.js
-@@ -130,7 +130,7 @@ exports.default = async (dir) => {
-         try {
-             // The loaded "config" could potentially be a static forge config, ESM module or async function
-             // eslint-disable-next-line @typescript-eslint/no-var-requires
--            const loaded = require(path_1.default.resolve(dir, forgeConfig));
-+            const loaded = await import(require('node:url').pathToFileURL(path_1.default.join(dir, forgeConfig)))
-             const maybeForgeConfig = 'default' in loaded ? loaded.default : loaded;
-             forgeConfig = typeof maybeForgeConfig === 'function' ? await maybeForgeConfig() : maybeForgeConfig;
-         }

.yarn/patches/next-auth-npm-4.24.5-8428e11927.patch

@@ -1,15 +1,15 @@
 diff --git a/package.json b/package.json
-index 26dcf8217f3e221e4c53722f14d29bb788332772..57a66dcb0943b9dd5cdaac2eaffccd9225a6b735 100644
+index ca30bca63196b923fa5a27eb85ce2ee890222d36..39e9d08dea40f25568a39bfbc0154458d32c8a66 100644
 --- a/package.json
 +++ b/package.json
-@@ -34,6 +34,10 @@
-     "./adapters": {
-       "types": "./adapters.d.ts"
+@@ -31,6 +31,10 @@
+       "types": "./index.d.ts",
+       "default": "./index.js"
      },
 +    "./core": {
 +      "types": "./core/index.d.ts",
 +      "default": "./core/index.js"
 +    },
-     "./jwt": {
-       "types": "./jwt/index.d.ts",
-       "default": "./jwt/index.js"
+     "./adapters": {
+       "types": "./adapters.d.ts"
+     },

.yarnrc.yml

@@ -6,10 +6,10 @@ nmMode: hardlinks-local
 
 nodeLinker: node-modules
 
-npmAuthToken: '${NPM_TOKEN:-NONE}'
+npmAuthToken: "${NPM_TOKEN:-NONE}"
 
 npmPublishAccess: public
 
-npmPublishRegistry: 'https://registry.npmjs.org'
+npmPublishRegistry: "https://registry.npmjs.org"
 
-yarnPath: .yarn/releases/yarn-4.0.1.cjs
+yarnPath: .yarn/releases/yarn-4.0.2.cjs

Cargo.lock

@@ -83,14 +83,15 @@ dependencies = [
 
 [[package]]
 name = "ahash"
-version = "0.8.3"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
+checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
 dependencies = [
  "cfg-if",
  "getrandom",
  "once_cell",
  "version_check",
+ "zerocopy",
 ]
 
 [[package]]
@@ -240,6 +241,16 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "atomic-write-file"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c232177ba50b16fe7a4588495bd474a62a9e45a8e4ca6fd7d0b7ac29d164631e"
+dependencies = [
+ "nix",
+ "rand",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -941,7 +952,7 @@ version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
 dependencies = [
- "ahash 0.8.3",
+ "ahash 0.8.6",
 ]
 
 [[package]]
@@ -950,7 +961,7 @@ version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156"
 dependencies = [
- "ahash 0.8.3",
+ "ahash 0.8.6",
  "allocator-api2",
 ]
 
@@ -1112,7 +1123,7 @@ dependencies = [
 [[package]]
 name = "jwst-codec"
 version = "0.1.0"
-source = "git+https://github.com/toeverything/OctoBase.git?rev=aad9e5b#aad9e5b7e9d6f479e6cf7555f5845bbbaaadbc66"
+source = "git+https://github.com/toeverything/OctoBase.git?rev=49a6b7a#49a6b7af25ce1fe54e8383e10980e9536821d286"
 dependencies = [
  "arbitrary",
  "bitvec",
@@ -1133,7 +1144,7 @@ dependencies = [
 [[package]]
 name = "jwst-core"
 version = "0.1.0"
-source = "git+https://github.com/toeverything/OctoBase.git?rev=aad9e5b#aad9e5b7e9d6f479e6cf7555f5845bbbaaadbc66"
+source = "git+https://github.com/toeverything/OctoBase.git?rev=49a6b7a#49a6b7af25ce1fe54e8383e10980e9536821d286"
 dependencies = [
  "async-trait",
  "base64",
@@ -1151,7 +1162,7 @@ dependencies = [
 [[package]]
 name = "jwst-logger"
 version = "0.1.0"
-source = "git+https://github.com/toeverything/OctoBase.git?rev=aad9e5b#aad9e5b7e9d6f479e6cf7555f5845bbbaaadbc66"
+source = "git+https://github.com/toeverything/OctoBase.git?rev=49a6b7a#49a6b7af25ce1fe54e8383e10980e9536821d286"
 dependencies = [
  "chrono",
  "nu-ansi-term 0.49.0",
@@ -1164,7 +1175,7 @@ dependencies = [
 [[package]]
 name = "jwst-storage"
 version = "0.1.0"
-source = "git+https://github.com/toeverything/OctoBase.git?rev=aad9e5b#aad9e5b7e9d6f479e6cf7555f5845bbbaaadbc66"
+source = "git+https://github.com/toeverything/OctoBase.git?rev=49a6b7a#49a6b7af25ce1fe54e8383e10980e9536821d286"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1189,7 +1200,7 @@ dependencies = [
 [[package]]
 name = "jwst-storage-migration"
 version = "0.1.0"
-source = "git+https://github.com/toeverything/OctoBase.git?rev=aad9e5b#aad9e5b7e9d6f479e6cf7555f5845bbbaaadbc66"
+source = "git+https://github.com/toeverything/OctoBase.git?rev=49a6b7a#49a6b7af25ce1fe54e8383e10980e9536821d286"
 dependencies = [
  "sea-orm-migration",
  "tokio",
@@ -1241,12 +1252,12 @@ checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
 
 [[package]]
 name = "libloading"
-version = "0.7.4"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
+checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161"
 dependencies = [
  "cfg-if",
- "winapi",
+ "windows-sys",
 ]
 
 [[package]]
@@ -1257,9 +1268,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058"
 
 [[package]]
 name = "libsqlite3-sys"
-version = "0.26.0"
+version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"
+checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
 dependencies = [
  "cc",
  "pkg-config",
@@ -1337,6 +1348,15 @@ version = "2.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
 
+[[package]]
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -1354,9 +1374,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.8.8"
+version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
+checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0"
 dependencies = [
  "libc",
  "log",
@@ -1375,9 +1395,9 @@ dependencies = [
 
 [[package]]
 name = "napi"
-version = "2.13.3"
+version = "2.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd063c93b900149304e3ba96ce5bf210cd4f81ef5eb80ded0d100df3e85a3ac0"
+checksum = "1133249c46e92da921bafc8aba4912bf84d6c475f7625183772ed2d0844dc3a7"
 dependencies = [
  "anyhow",
  "bitflags 2.4.1",
@@ -1393,15 +1413,15 @@ dependencies = [
 
 [[package]]
 name = "napi-build"
-version = "2.0.1"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "882a73d9ef23e8dc2ebbffb6a6ae2ef467c0f18ac10711e4cc59c5485d41df0e"
+checksum = "d4b4532cf86bfef556348ac65e561e3123879f0e7566cca6d43a6ff5326f13df"
 
 [[package]]
 name = "napi-derive"
-version = "2.13.0"
+version = "2.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da1c6a8fa84d549aa8708fcd062372bf8ec6e849de39016ab921067d21bde367"
+checksum = "a0cca5738c6e81eb5ffd2c8ff2b4f05ece9c4c60c7e2b36cec6524492cf7f330"
 dependencies = [
  "cfg-if",
  "convert_case",
@@ -1413,9 +1433,9 @@ dependencies = [
 
 [[package]]
 name = "napi-derive-backend"
-version = "1.0.52"
+version = "1.0.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20bbc7c69168d06a848f925ec5f0e0997f98e8c8d4f2cc30157f0da51c009e17"
+checksum = "35960e5f33228192a9b661447d0dfe8f5a3790ff5b4058c4d67680ded4f65b91"
 dependencies = [
  "convert_case",
  "once_cell",
@@ -1428,13 +1448,26 @@ dependencies = [
 
 [[package]]
 name = "napi-sys"
-version = "2.2.3"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "166b5ef52a3ab5575047a9fe8d4a030cdd0f63c96f071cd6907674453b07bae3"
+checksum = "2503fa6af34dc83fb74888df8b22afe933b58d37daf7d80424b1c60c68196b8b"
 dependencies = [
  "libloading",
 ]
 
+[[package]]
+name = "nix"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "libc",
+ "memoffset",
+ "pin-utils",
+]
+
 [[package]]
 name = "no-std-compat"
 version = "0.4.1"
@@ -2292,18 +2325,18 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
 
 [[package]]
 name = "serde"
-version = "1.0.190"
+version = "1.0.193"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7"
+checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.190"
+version = "1.0.193"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3"
+checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2450,9 +2483,9 @@ dependencies = [
 
 [[package]]
 name = "sqlx"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e50c216e3624ec8e7ecd14c6a6a6370aad6ee5d8cfc3ab30b5162eeeef2ed33"
+checksum = "dba03c279da73694ef99763320dea58b51095dfe87d001b1d4b5fe78ba8763cf"
 dependencies = [
  "sqlx-core",
  "sqlx-macros",
@@ -2463,11 +2496,11 @@ dependencies = [
 
 [[package]]
 name = "sqlx-core"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d6753e460c998bbd4cd8c6f0ed9a64346fcca0723d6e75e52fdc351c5d2169d"
+checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd"
 dependencies = [
- "ahash 0.8.3",
+ "ahash 0.8.6",
  "atoi",
  "bigdecimal",
  "byteorder",
@@ -2511,9 +2544,9 @@ dependencies = [
 
 [[package]]
 name = "sqlx-macros"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a793bb3ba331ec8359c1853bd39eed32cdd7baaf22c35ccf5c92a7e8d1189ec"
+checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2524,10 +2557,11 @@ dependencies = [
 
 [[package]]
 name = "sqlx-macros-core"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a4ee1e104e00dedb6aa5ffdd1343107b0a4702e862a84320ee7cc74782d96fc"
+checksum = "d0bd4519486723648186a08785143599760f7cc81c52334a55d6a83ea1e20841"
 dependencies = [
+ "atomic-write-file",
  "dotenvy",
  "either",
  "heck",
@@ -2550,9 +2584,9 @@ dependencies = [
 
 [[package]]
 name = "sqlx-mysql"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "864b869fdf56263f4c95c45483191ea0af340f9f3e3e7b4d57a61c7c87a970db"
+checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4"
 dependencies = [
  "atoi",
  "base64",
@@ -2597,9 +2631,9 @@ dependencies = [
 
 [[package]]
 name = "sqlx-postgres"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb7ae0e6a97fb3ba33b23ac2671a5ce6e3cabe003f451abd5a56e7951d975624"
+checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24"
 dependencies = [
  "atoi",
  "base64",
@@ -2642,9 +2676,9 @@ dependencies = [
 
 [[package]]
 name = "sqlx-sqlite"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d59dc83cf45d89c555a577694534fcd1b55c545a816c816ce51f20bbe56a4f3f"
+checksum = "210976b7d948c7ba9fced8ca835b11cbb2d677c59c79de41ac0d397e14547490"
 dependencies = [
  "atoi",
  "chrono",
@@ -2662,6 +2696,7 @@ dependencies = [
  "time",
  "tracing",
  "url",
+ "urlencoding",
  "uuid",
 ]
 
@@ -2817,9 +2852,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.33.0"
+version = "1.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653"
+checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9"
 dependencies = [
  "backtrace",
  "bytes",
@@ -2836,9 +2871,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3024,6 +3059,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "urlencoding"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
+
 [[package]]
 name = "utf8parse"
 version = "0.2.1"
@@ -3032,9 +3073,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 
 [[package]]
 name = "uuid"
-version = "1.5.0"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc"
+checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560"
 dependencies = [
  "getrandom",
  "rand",
@@ -3141,12 +3182,9 @@ dependencies = [
 
 [[package]]
 name = "webpki-roots"
-version = "0.24.0"
+version = "0.25.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888"
-dependencies = [
- "rustls-webpki",
-]
+checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10"
 
 [[package]]
 name = "whoami"
@@ -3278,6 +3316,26 @@ dependencies = [
  "tap",
 ]
 
+[[package]]
+name = "zerocopy"
+version = "0.7.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.6.0"

README.md

@@ -107,12 +107,11 @@ If you have questions, you are welcome to contact us. One of the best places to
 
 ## Ecosystem
 
-| Name                                                                                            |                                         |                                                                                                                                                     |
-| ----------------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [@toeverything/component](https://github.com/toeverything/design/tree/main/packages/components) | Toeverything Shared Component Resources |                                                                                                                                                     |
-| [@affine/component](packages/frontend/component)                                                | AFFiNE Component Resources              | [![](https://img.shields.io/codecov/c/github/toeverything/affine?style=flat-square)](https://affine-storybook.vercel.app/)                          |
-| [@toeverything/y-indexeddb](packages/common/y-indexeddb)                                        | IndexedDB database adapter for Yjs      | [![](https://img.shields.io/npm/dm/@toeverything/y-indexeddb?style=flat-square&color=eee)](https://www.npmjs.com/package/@toeverything/y-indexeddb) |
-| [@toeverything/theme](packages/common/theme)                                                    | AFFiNE theme                            | [![](https://img.shields.io/npm/dm/@toeverything/theme?style=flat-square&color=eee)](https://www.npmjs.com/package/@toeverything/theme)             |
+| Name                                                     |                                    |                                                                                                                                                     |
+| -------------------------------------------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [@affine/component](packages/frontend/component)         | AFFiNE Component Resources         | [![](https://img.shields.io/codecov/c/github/toeverything/affine?style=flat-square)](https://affine-storybook.vercel.app/)                          |
+| [@toeverything/y-indexeddb](packages/common/y-indexeddb) | IndexedDB database adapter for Yjs | [![](https://img.shields.io/npm/dm/@toeverything/y-indexeddb?style=flat-square&color=eee)](https://www.npmjs.com/package/@toeverything/y-indexeddb) |
+| [@toeverything/theme](packages/common/theme)             | AFFiNE theme                       | [![](https://img.shields.io/npm/dm/@toeverything/theme?style=flat-square&color=eee)](https://www.npmjs.com/package/@toeverything/theme)             |
 
 ## Plugins
 
@@ -195,6 +194,13 @@ For feature request, please see [community.affine.pro](https://community.affine.
 
 ## Building
 
+### Codespaces
+
+From the GitHub repo main page, click the green "Code" button and select "Create codespace on master". This will open a new Codespace with the (supposedly auto-forked
+AFFiNE repo cloned, built, and ready to go.
+
+### Local
+
 See [BUILDING.md] for instructions on how to build AFFiNE from source code.
 
 ## Contributing
@@ -220,10 +226,10 @@ See [LICENSE] for details.
 [update page]: https://affine.pro/blog?tag=Release%20Note
 [jobs available]: ./docs/jobs.md
 [latest packages]: https://github.com/toeverything/AFFiNE/pkgs/container/affine-self-hosted
-[contributor license agreement]: https://github.com/toeverything/affine/edit/master/.github/CLA.md
-[rust-version-icon]: https://img.shields.io/badge/Rust-1.71.0-dea584
+[contributor license agreement]: https://github.com/toeverything/affine/edit/canary/.github/CLA.md
+[rust-version-icon]: https://img.shields.io/badge/Rust-1.74.1-dea584
 [stars-icon]: https://img.shields.io/github/stars/toeverything/AFFiNE.svg?style=flat&logo=github&colorB=red&label=stars
-[codecov]: https://codecov.io/gh/toeverything/affine/branch/master/graphs/badge.svg?branch=master
+[codecov]: https://codecov.io/gh/toeverything/affine/branch/canary/graphs/badge.svg?branch=canary
 [node-version-icon]: https://img.shields.io/badge/node-%3E=18.16.1-success
 [typescript-version-icon]: https://img.shields.io/github/package-json/dependency-version/toeverything/affine/dev/typescript
 [react-version-icon]: https://img.shields.io/github/package-json/dependency-version/toeverything/AFFiNE/react?filename=packages%2Ffrontend%2Fcore%2Fpackage.json&color=rgb(97%2C228%2C251)

docs/CONTRIBUTING.md

@@ -13,7 +13,7 @@ Use the table of contents icon on the top left corner of this document to get to
 Currently we have two versions of AFFiNE:
 
 - [AFFiNE Pre-Alpha](https://livedemo.affine.pro/). This version uses the branch `Pre-Alpha`, it is no longer actively developed but contains some different functions and features.
-- [AFFiNE Alpha](https://pathfinder.affine.pro/). This version uses the `master` branch, this is the latest version under active development.
+- [AFFiNE Alpha](https://pathfinder.affine.pro/). This version uses the `canary` branch, this is the latest version under active development.
 
 To get an overview of the project, read the [README](../README.md). Here are some resources to help you get started with open source contributions:
 

docs/contributing/releases.md

@@ -11,7 +11,7 @@ The AFFiNE core team gives release authorization. And also have the following re
 
 ## How to make a release?
 
-Before releasing, ensure you have the latest version of the `master` branch.
+Before releasing, ensure you have the latest version of the `canary` branch.
 
 And Read the semver specification to understand how to version your release. https://semver.org
 
@@ -21,13 +21,13 @@ And Read the semver specification to understand how to version your release. htt
 ./scripts/set-version.sh 0.5.4-canary.5
 ```
 
-### 2. Commit changes and push to `master`
+### 2. Commit changes and push to `canary`
 
 ```shell
 git add .
 # vx.y.z-canary.n
 git commit -m "v0.5.4-canary.5"
-git push origin master
+git push origin canary
 ```
 
 ### 3. Create a release action

docs/issue-triaging.md

@@ -0,0 +1,28 @@
+# Issues Triaging
+
+When we receive your issue, we will first triaging it. Triaging an issue usually takes around one business day but may take longer. Goal of triaging is to provide you with a clear understanding of what will happen to your issue. For example, after your feature request was triaged you know whether we plan to tackle the issue or whether we'll wait to hear what the broader community thinks about this request.
+
+Here are issue states and their descriptions:
+
+| State               | Description                                                                                                                                                                                                                           |
+| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Untriaged           | The team has not yet reviewed the issue. We usually do it within one business day.                                                                                                                                                    |
+| As designed         | The behavior described in the issue is intentional. If you find it seriously disruptive or if we’ve misunderstood you, please let us know in the issue’s comments section.                                                            |
+| Blocked             | We can’t work on this issue until another one (linked) is resolved.                                                                                                                                                                   |
+| Can’t Reproduce     | We have been unable to reproduce the issue on our side. It could be flaky or fixed already, or we may not have had all the details we needed. If you’re still experiencing the issue and have any further details, please share them. |
+| Duplicate           | The issue is the same (or has the same cause) as another one (linked).                                                                                                                                                                |
+| Fixed               | If the issue was a bug, it’s been fixed; if it was a missing feature, it’s been implemented.                                                                                                                                          |
+| Fixed In Branch     | If the issue was a bug, it’s been fixed; if it was a missing feature, it’s been implemented; the changes are now in a separate branch and haven’t been merged into the default branch yet.                                            |
+| In Progress         | We’re currently working on the issue.                                                                                                                                                                                                 |
+| Incomplete          | Unfortunately we don’t have enough information to proceed. If you’re willing to share any further details about the issue, please do so in the comments.                                                                              |
+| Obsolete            | The part of the product that was causing this issue has been removed or significantly reworked since it was created.                                                                                                                  |
+| Upvoting            | We are currently evaluating demand for the issue and checking whether it requires complicated or risky changes. Please leave a vote or comment if you think it should be prioritized.                                                 |
+| Open                | We want to implement the fix or feature in the near future. We can’t promise it will appear in the next public release, but it’s on our short list.                                                                                   |
+| Shelved             | We have reviewed the issue and decided that, even though it has merit, we cannot currently include it in our near-term plan.                                                                                                          |
+| Third Party Problem | The issue is caused by a third party. We've done our best to inform them about it.                                                                                                                                                    |
+| To be Discussed     | We need some time to discuss the issue.                                                                                                                                                                                               |
+| To Reproduce        | We will try to find the steps needed to reproduce the issue on our side.                                                                                                                                                              |
+| Under Investigation | We’ve triaged the issue, but now we need to investigate it more thoroughly. This may require processing additional information like logs or dumps.                                                                                    |
+| Waiting for Info    | We’ve requested additional information from the person who created the issue and are waiting for them to get back to us.                                                                                                              |
+| Declined            | We’ve reviewed the suggestion and, while we appreciate its value, we unfortunately do not have the resources to implement it.                                                                                                         |
+| Answered            | The issue actually turned out to be a question or a misunderstanding, and it has been answered or resolved.                                                                                                                           |

nx.json

@@ -11,7 +11,7 @@
     }
   },
   "affected": {
-    "defaultBase": "master"
+    "defaultBase": "canary"
   },
   "namedInputs": {
     "default": ["{projectRoot}/**/*", "sharedGlobals"],
@@ -56,7 +56,7 @@
           "env": "SENTRY_AUTH_TOKEN"
         },
         {
-          "env": "NEXT_PUBLIC_SENTRY_DSN"
+          "env": "SENTRY_DSN"
         },
         {
           "env": "DISTRIBUTION"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@affine/monorepo",
-  "version": "0.10.2",
+  "version": "0.10.4-beta.2",
   "private": true,
   "author": "toeverything",
   "license": "MIT",
@@ -33,91 +33,91 @@
     "lint:eslint:fix": "yarn lint:eslint --fix",
     "lint:prettier": "prettier --ignore-unknown --cache --check .",
     "lint:prettier:fix": "prettier --ignore-unknown --cache --write .",
+    "lint:ox": "oxlint --deny-warnings --import-plugin -D correctness -D nursery -D prefer-array-some -D no-useless-promise-resolve-reject -A no-undef -A consistent-type-exports -A default -A named -A ban-ts-comment",
     "lint": "yarn lint:eslint && yarn lint:prettier",
     "lint:fix": "yarn lint:eslint:fix && yarn lint:prettier:fix",
     "test": "vitest --run",
     "test:ui": "vitest --ui",
     "test:coverage": "vitest run --coverage",
-    "notify": "node scripts/notify.mjs",
-    "circular": "madge --circular --ts-config ./tsconfig.json ./packages/frontend/core/src/pages/**/*.tsx ./packages/frontend/core/src/index.tsx ./packages/frontend/electron/src/*/index.ts",
     "typecheck": "tsc -b tsconfig.json --diagnostics",
     "postinstall": "node ./scripts/check-version.mjs && yarn i18n-codegen gen && yarn husky install"
   },
   "lint-staged": {
     "*": "prettier --write --ignore-unknown --cache",
     "*.{ts,tsx,mjs,js,jsx}": [
-      "prettier . --ignore-unknown --write",
+      "prettier --ignore-unknown --write",
       "eslint --cache --fix"
     ],
     "*.toml": [
-      "prettier . --ignore-unknown --write",
       "taplo format"
+    ],
+    "*.rs": [
+      "cargo fmt --"
     ]
   },
   "devDependencies": {
     "@affine-test/kit": "workspace:*",
     "@affine/cli": "workspace:*",
     "@affine/plugin-cli": "workspace:*",
-    "@commitlint/cli": "^17.8.0",
-    "@commitlint/config-conventional": "^17.8.0",
-    "@faker-js/faker": "^8.2.0",
+    "@commitlint/cli": "^18.4.3",
+    "@commitlint/config-conventional": "^18.4.3",
+    "@faker-js/faker": "^8.3.1",
     "@istanbuljs/schema": "^0.1.3",
     "@magic-works/i18n-codegen": "^0.5.0",
-    "@nx/vite": "16.10.0",
+    "@nx/vite": "17.1.3",
     "@perfsee/sdk": "^1.9.0",
-    "@playwright/test": "^1.39.0",
+    "@playwright/test": "^1.40.0",
     "@taplo/cli": "^0.5.2",
-    "@testing-library/react": "^14.0.0",
+    "@testing-library/react": "^14.1.2",
     "@toeverything/infra": "workspace:*",
     "@types/affine__env": "workspace:*",
-    "@types/eslint": "^8.44.4",
-    "@types/node": "^18.18.5",
-    "@typescript-eslint/eslint-plugin": "^6.7.5",
-    "@typescript-eslint/parser": "^6.7.5",
-    "@vanilla-extract/vite-plugin": "^3.9.0",
+    "@types/eslint": "^8.44.7",
+    "@types/node": "^20.9.3",
+    "@typescript-eslint/eslint-plugin": "^6.13.1",
+    "@typescript-eslint/parser": "^6.13.1",
+    "@vanilla-extract/vite-plugin": "^3.9.2",
     "@vanilla-extract/webpack-plugin": "^2.3.1",
-    "@vitejs/plugin-react-swc": "^3.4.0",
+    "@vitejs/plugin-react-swc": "^3.5.0",
     "@vitest/coverage-istanbul": "0.34.6",
     "@vitest/ui": "0.34.6",
-    "electron": "^27.0.0",
-    "eslint": "^8.51.0",
+    "electron": "^27.1.0",
+    "eslint": "^8.54.0",
     "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-i": "^2.28.1",
-    "eslint-plugin-prettier": "^5.0.1",
+    "eslint-plugin-i": "^2.29.0",
     "eslint-plugin-react": "^7.33.2",
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint-plugin-simple-import-sort": "^10.0.0",
-    "eslint-plugin-sonarjs": "^0.21.0",
-    "eslint-plugin-unicorn": "^48.0.1",
+    "eslint-plugin-sonarjs": "^0.23.0",
+    "eslint-plugin-unicorn": "^49.0.0",
     "eslint-plugin-unused-imports": "^3.0.0",
-    "eslint-plugin-vue": "^9.17.0",
-    "fake-indexeddb": "5.0.0",
-    "happy-dom": "^12.9.1",
+    "eslint-plugin-vue": "^9.18.1",
+    "fake-indexeddb": "5.0.1",
+    "happy-dom": "^12.10.3",
     "husky": "^8.0.3",
-    "lint-staged": "^15.0.0",
-    "madge": "^6.1.0",
-    "msw": "^1.3.2",
-    "nanoid": "^5.0.1",
-    "nx": "^16.10.0",
+    "lint-staged": "^15.1.0",
+    "msw": "^2.0.8",
+    "nanoid": "^5.0.3",
+    "nx": "^17.1.3",
     "nx-cloud": "^16.5.2",
     "nyc": "^15.1.0",
-    "prettier": "^3.0.3",
+    "oxlint": "^0.0.18",
+    "prettier": "^3.1.0",
     "semver": "^7.5.4",
     "serve": "^14.2.1",
-    "string-width": "^6.1.0",
+    "string-width": "^7.0.0",
     "ts-node": "^10.9.1",
-    "typescript": "^5.2.2",
-    "vite": "^4.4.11",
+    "typescript": "^5.3.2",
+    "vite": "^5.0.6",
     "vite-plugin-istanbul": "^5.0.0",
-    "vite-plugin-static-copy": "^0.17.0",
+    "vite-plugin-static-copy": "^1.0.0",
     "vite-tsconfig-paths": "^4.2.1",
     "vitest": "0.34.6",
     "vitest-fetch-mock": "^0.2.2",
     "vitest-mock-extended": "^1.3.1"
   },
-  "packageManager": "yarn@4.0.1",
+  "packageManager": "yarn@4.0.2",
   "resolutions": {
-    "vite": "^4.4.11",
+    "vite": "^5.0.6",
     "array-buffer-byte-length": "npm:@nolyfill/array-buffer-byte-length@latest",
     "array-includes": "npm:@nolyfill/array-includes@latest",
     "array.prototype.flat": "npm:@nolyfill/array.prototype.flat@latest",
@@ -173,9 +173,8 @@
     "unbox-primitive": "npm:@nolyfill/unbox-primitive@latest",
     "which-boxed-primitive": "npm:@nolyfill/which-boxed-primitive@latest",
     "which-typed-array": "npm:@nolyfill/which-typed-array@latest",
-    "next-auth@^4.23.2": "patch:next-auth@npm%3A4.23.2#./.yarn/patches/next-auth-npm-4.23.2-5f0e551bc7.patch",
-    "@electron-forge/core@^6.4.2": "patch:@electron-forge/core@npm%3A6.4.2#./.yarn/patches/@electron-forge-core-npm-6.4.2-ab60c87e75.patch",
-    "@electron-forge/core@6.4.2": "patch:@electron-forge/core@npm%3A6.4.2#./.yarn/patches/@electron-forge-core-npm-6.4.2-ab60c87e75.patch",
+    "next-auth@^4.24.5": "patch:next-auth@npm%3A4.24.5#~/.yarn/patches/next-auth-npm-4.24.5-8428e11927.patch",
+    "@reforged/maker-appimage/@electron-forge/maker-base": "7.2.0",
     "macos-alias": "npm:macos-alias-building@latest",
     "fs-xattr": "npm:@napi-rs/xattr@latest"
   }

packages/backend/server/migrations/20231121033532_history/migration.sql

@@ -0,0 +1,14 @@
+-- AlterTable
+ALTER TABLE "blobs" ADD COLUMN     "deleted_at" TIMESTAMPTZ(6);
+
+-- CreateTable
+CREATE TABLE "snapshot_histories" (
+    "workspace_id" VARCHAR(36) NOT NULL,
+    "guid" VARCHAR(36) NOT NULL,
+    "timestamp" TIMESTAMPTZ(6) NOT NULL,
+    "blob" BYTEA NOT NULL,
+    "state" BYTEA,
+    "expired_at" TIMESTAMPTZ(6) NOT NULL,
+
+    CONSTRAINT "snapshot_histories_pkey" PRIMARY KEY ("workspace_id","guid","timestamp")
+);

packages/backend/server/migrations/20231124091123_soft_delete_opt_blobs/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "optimized_blobs" ADD COLUMN     "deleted_at" TIMESTAMPTZ(6);

packages/backend/server/migrations/migration_lock.toml

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
 # It should be added in your version-control system (i.e. Git)
-provider = "postgresql"
+provider = "postgresql"

packages/backend/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/server",
   "private": true,
-  "version": "0.10.2",
+  "version": "0.10.4-beta.2",
   "description": "Affine Node.js server",
   "type": "module",
   "bin": {
@@ -18,42 +18,47 @@
     "predeploy": "yarn prisma migrate deploy && node --es-module-specifier-resolution node ./dist/data/app.js run"
   },
   "dependencies": {
-    "@apollo/server": "^4.9.4",
-    "@auth/prisma-adapter": "^1.0.3",
-    "@aws-sdk/client-s3": "^3.433.0",
+    "@apollo/server": "^4.9.5",
+    "@auth/prisma-adapter": "^1.0.7",
+    "@aws-sdk/client-s3": "^3.454.0",
     "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.17.0",
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.1.0",
     "@keyv/redis": "^2.8.0",
-    "@nestjs/apollo": "^12.0.9",
-    "@nestjs/common": "^10.2.7",
-    "@nestjs/core": "^10.2.7",
-    "@nestjs/event-emitter": "^2.0.2",
-    "@nestjs/graphql": "^12.0.9",
-    "@nestjs/platform-express": "^10.2.7",
-    "@nestjs/platform-socket.io": "^10.2.7",
-    "@nestjs/throttler": "^5.0.0",
-    "@nestjs/websockets": "^10.2.7",
+    "@nestjs/apollo": "^12.0.11",
+    "@nestjs/common": "^10.2.10",
+    "@nestjs/core": "^10.2.10",
+    "@nestjs/event-emitter": "^2.0.3",
+    "@nestjs/graphql": "^12.0.11",
+    "@nestjs/platform-express": "^10.2.10",
+    "@nestjs/platform-socket.io": "^10.2.10",
+    "@nestjs/schedule": "^4.0.0",
+    "@nestjs/throttler": "^5.0.1",
+    "@nestjs/websockets": "^10.2.10",
     "@node-rs/argon2": "^1.5.2",
     "@node-rs/crc32": "^1.7.2",
     "@node-rs/jsonwebtoken": "^0.2.3",
-    "@opentelemetry/api": "^1.6.0",
-    "@opentelemetry/core": "^1.17.1",
-    "@opentelemetry/instrumentation": "^0.44.0",
-    "@opentelemetry/instrumentation-graphql": "^0.35.2",
-    "@opentelemetry/instrumentation-http": "^0.44.0",
-    "@opentelemetry/instrumentation-ioredis": "^0.35.2",
-    "@opentelemetry/instrumentation-nestjs-core": "^0.33.2",
-    "@opentelemetry/instrumentation-socket.io": "^0.34.2",
-    "@opentelemetry/sdk-metrics": "^1.17.1",
-    "@opentelemetry/sdk-node": "^0.44.0",
-    "@opentelemetry/sdk-trace-node": "^1.17.1",
-    "@prisma/client": "^5.4.2",
-    "@prisma/instrumentation": "^5.4.2",
+    "@opentelemetry/api": "^1.7.0",
+    "@opentelemetry/core": "^1.18.1",
+    "@opentelemetry/exporter-prometheus": "^0.45.1",
+    "@opentelemetry/exporter-zipkin": "^1.18.1",
+    "@opentelemetry/host-metrics": "^0.34.0",
+    "@opentelemetry/instrumentation": "^0.45.1",
+    "@opentelemetry/instrumentation-graphql": "^0.36.0",
+    "@opentelemetry/instrumentation-http": "^0.45.1",
+    "@opentelemetry/instrumentation-ioredis": "^0.36.0",
+    "@opentelemetry/instrumentation-nestjs-core": "^0.33.3",
+    "@opentelemetry/instrumentation-socket.io": "^0.34.3",
+    "@opentelemetry/resources": "^1.18.1",
+    "@opentelemetry/sdk-metrics": "^1.18.1",
+    "@opentelemetry/sdk-node": "^0.45.1",
+    "@opentelemetry/sdk-trace-node": "^1.18.1",
+    "@prisma/client": "^5.6.0",
+    "@prisma/instrumentation": "^5.6.0",
     "@socket.io/redis-adapter": "^8.2.1",
     "cookie-parser": "^1.4.6",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
-    "file-type": "^18.5.0",
+    "file-type": "^18.7.0",
     "get-stream": "^8.0.1",
     "graphql": "^16.8.1",
     "graphql-type-json": "^0.3.2",
@@ -61,49 +66,49 @@
     "ioredis": "^5.3.2",
     "keyv": "^4.5.4",
     "lodash-es": "^4.17.21",
-    "nanoid": "^5.0.1",
-    "nest-commander": "^3.12.0",
+    "nanoid": "^5.0.3",
+    "nest-commander": "^3.12.2",
     "nestjs-throttler-storage-redis": "^0.4.1",
-    "next-auth": "^4.23.2",
-    "nodemailer": "^6.9.6",
+    "next-auth": "^4.24.5",
+    "nodemailer": "^6.9.7",
     "on-headers": "^1.0.2",
     "parse-duration": "^1.1.0",
     "pretty-time": "^1.1.0",
-    "prisma": "^5.4.2",
+    "prisma": "^5.6.0",
     "prom-client": "^15.0.0",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
     "semver": "^7.5.4",
     "socket.io": "^4.7.2",
-    "stripe": "^14.1.0",
+    "stripe": "^14.5.0",
     "ws": "^8.14.2",
-    "yjs": "^13.6.8"
+    "yjs": "^13.6.10"
   },
   "devDependencies": {
     "@affine-test/kit": "workspace:*",
     "@affine/storage": "workspace:*",
     "@napi-rs/image": "^1.7.0",
-    "@nestjs/testing": "^10.2.7",
-    "@types/cookie-parser": "^1.4.4",
-    "@types/engine.io": "^3.1.8",
-    "@types/express": "^4.17.19",
-    "@types/graphql-upload": "^16.0.3",
+    "@nestjs/testing": "^10.2.10",
+    "@types/cookie-parser": "^1.4.6",
+    "@types/engine.io": "^3.1.10",
+    "@types/express": "^4.17.21",
+    "@types/graphql-upload": "^16.0.5",
     "@types/keyv": "^4.2.0",
-    "@types/lodash-es": "^4.17.9",
-    "@types/node": "^18.18.5",
-    "@types/nodemailer": "^6.4.11",
-    "@types/on-headers": "^1.0.1",
-    "@types/pretty-time": "^1.1.3",
-    "@types/sinon": "^10.0.19",
-    "@types/supertest": "^2.0.14",
-    "@types/ws": "^8.5.7",
-    "ava": "^5.3.1",
+    "@types/lodash-es": "^4.17.11",
+    "@types/node": "^20.9.3",
+    "@types/nodemailer": "^6.4.14",
+    "@types/on-headers": "^1.0.3",
+    "@types/pretty-time": "^1.1.5",
+    "@types/sinon": "^17.0.2",
+    "@types/supertest": "^2.0.16",
+    "@types/ws": "^8.5.10",
+    "ava": "^6.0.0",
     "c8": "^8.0.1",
     "nodemon": "^3.0.1",
-    "sinon": "^16.1.0",
+    "sinon": "^17.0.1",
     "supertest": "^6.3.3",
     "ts-node": "^10.9.1",
-    "typescript": "^5.2.2"
+    "typescript": "^5.3.2"
   },
   "ava": {
     "extensions": {

packages/backend/server/schema.prisma

@@ -164,25 +164,29 @@ model VerificationToken {
 }
 
 model Blob {
-  id          Int      @id @default(autoincrement()) @db.Integer
-  hash        String   @db.VarChar
-  workspaceId String   @map("workspace_id") @db.VarChar
-  blob        Bytes    @db.ByteA
+  id          Int       @id @default(autoincrement()) @db.Integer
+  hash        String    @db.VarChar
+  workspaceId String    @map("workspace_id") @db.VarChar
+  blob        Bytes     @db.ByteA
   length      BigInt
-  createdAt   DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
+  createdAt   DateTime  @default(now()) @map("created_at") @db.Timestamptz(6)
+  // not for keeping, but for snapshot history
+  deletedAt   DateTime? @map("deleted_at") @db.Timestamptz(6)
 
   @@unique([workspaceId, hash])
   @@map("blobs")
 }
 
 model OptimizedBlob {
-  id          Int      @id @default(autoincrement()) @db.Integer
-  hash        String   @db.VarChar
-  workspaceId String   @map("workspace_id") @db.VarChar
-  params      String   @db.VarChar
-  blob        Bytes    @db.ByteA
+  id          Int       @id @default(autoincrement()) @db.Integer
+  hash        String    @db.VarChar
+  workspaceId String    @map("workspace_id") @db.VarChar
+  params      String    @db.VarChar
+  blob        Bytes     @db.ByteA
   length      BigInt
-  createdAt   DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
+  createdAt   DateTime  @default(now()) @map("created_at") @db.Timestamptz(6)
+  // not for keeping, but for snapshot history
+  deletedAt   DateTime? @map("deleted_at") @db.Timestamptz(6)
 
   @@unique([workspaceId, hash, params])
   @@map("optimized_blobs")
@@ -191,8 +195,8 @@ model OptimizedBlob {
 // the latest snapshot of each doc that we've seen
 // Snapshot + Updates are the latest state of the doc
 model Snapshot {
-  id          String   @default(uuid()) @map("guid") @db.VarChar
   workspaceId String   @map("workspace_id") @db.VarChar
+  id          String   @default(uuid()) @map("guid") @db.VarChar
   blob        Bytes    @db.ByteA
   seq         Int      @default(0) @db.Integer
   state       Bytes?   @db.ByteA
@@ -214,6 +218,18 @@ model Update {
   @@map("updates")
 }
 
+model SnapshotHistory {
+  workspaceId String   @map("workspace_id") @db.VarChar(36)
+  id          String   @map("guid") @db.VarChar(36)
+  timestamp   DateTime @db.Timestamptz(6)
+  blob        Bytes    @db.ByteA
+  state       Bytes?   @db.ByteA
+  expiredAt   DateTime @map("expired_at") @db.Timestamptz(6)
+
+  @@id([workspaceId, id, timestamp])
+  @@map("snapshot_histories")
+}
+
 model NewFeaturesWaitingList {
   id        String   @id @default(uuid()) @db.VarChar
   email     String   @unique

packages/backend/server/src/app.ts

@@ -1,8 +1,9 @@
 import { Module } from '@nestjs/common';
 
 import { AppController } from './app.controller';
+import { CacheModule } from './cache';
 import { ConfigModule } from './config';
-import { MetricsModule } from './metrics';
+import { EventModule } from './event';
 import { BusinessModules } from './modules';
 import { AuthModule } from './modules/auth';
 import { PrismaModule } from './prisma';
@@ -10,17 +11,19 @@ import { SessionModule } from './session';
 import { StorageModule } from './storage';
 import { RateLimiterModule } from './throttler';
 
+const BasicModules = [
+  PrismaModule,
+  ConfigModule.forRoot(),
+  CacheModule,
+  EventModule,
+  StorageModule.forRoot(),
+  SessionModule,
+  RateLimiterModule,
+  AuthModule,
+];
+
 @Module({
-  imports: [
-    PrismaModule,
-    ConfigModule.forRoot(),
-    StorageModule.forRoot(),
-    MetricsModule,
-    SessionModule,
-    RateLimiterModule,
-    AuthModule,
-    ...BusinessModules,
-  ],
+  imports: [...BasicModules, ...BusinessModules],
   controllers: [AppController],
 })
 export class AppModule {}

packages/backend/server/src/cache/cache.ts

@@ -0,0 +1,330 @@
+import Keyv from 'keyv';
+
+export interface CacheSetOptions {
+  // in milliseconds
+  ttl?: number;
+}
+
+// extends if needed
+export interface Cache {
+  // standard operation
+  get<T = unknown>(key: string): Promise<T | undefined>;
+  set<T = unknown>(
+    key: string,
+    value: T,
+    opts?: CacheSetOptions
+  ): Promise<boolean>;
+  setnx<T = unknown>(
+    key: string,
+    value: T,
+    opts?: CacheSetOptions
+  ): Promise<boolean>;
+  increase(key: string, count?: number): Promise<number>;
+  decrease(key: string, count?: number): Promise<number>;
+  delete(key: string): Promise<boolean>;
+  has(key: string): Promise<boolean>;
+  ttl(key: string): Promise<number>;
+  expire(key: string, ttl: number): Promise<boolean>;
+
+  // list operations
+  pushBack<T = unknown>(key: string, ...values: T[]): Promise<number>;
+  pushFront<T = unknown>(key: string, ...values: T[]): Promise<number>;
+  len(key: string): Promise<number>;
+  list<T = unknown>(key: string, start: number, end: number): Promise<T[]>;
+  popFront<T = unknown>(key: string, count?: number): Promise<T[]>;
+  popBack<T = unknown>(key: string, count?: number): Promise<T[]>;
+
+  // map operations
+  mapSet<T = unknown>(
+    map: string,
+    key: string,
+    value: T,
+    opts: CacheSetOptions
+  ): Promise<boolean>;
+  mapIncrease(map: string, key: string, count?: number): Promise<number>;
+  mapDecrease(map: string, key: string, count?: number): Promise<number>;
+  mapGet<T = unknown>(map: string, key: string): Promise<T | undefined>;
+  mapDelete(map: string, key: string): Promise<boolean>;
+  mapKeys(map: string): Promise<string[]>;
+  mapRandomKey(map: string): Promise<string | undefined>;
+  mapLen(map: string): Promise<number>;
+}
+
+export class LocalCache implements Cache {
+  private readonly kv: Keyv;
+
+  constructor() {
+    this.kv = new Keyv();
+  }
+
+  // standard operation
+  async get<T = unknown>(key: string): Promise<T | undefined> {
+    return this.kv.get(key).catch(() => undefined);
+  }
+
+  async set<T = unknown>(
+    key: string,
+    value: T,
+    opts: CacheSetOptions = {}
+  ): Promise<boolean> {
+    return this.kv
+      .set(key, value, opts.ttl)
+      .then(() => true)
+      .catch(() => false);
+  }
+
+  async setnx<T = unknown>(
+    key: string,
+    value: T,
+    opts?: CacheSetOptions | undefined
+  ): Promise<boolean> {
+    if (!(await this.has(key))) {
+      return this.set(key, value, opts);
+    }
+    return false;
+  }
+
+  async increase(key: string, count: number = 1): Promise<number> {
+    const prev = (await this.get(key)) ?? 0;
+    if (typeof prev !== 'number') {
+      throw new Error(
+        `Expect a Number keyed by ${key}, but found ${typeof prev}`
+      );
+    }
+
+    const curr = prev + count;
+    return (await this.set(key, curr)) ? curr : prev;
+  }
+
+  async decrease(key: string, count: number = 1): Promise<number> {
+    return this.increase(key, -count);
+  }
+
+  async delete(key: string): Promise<boolean> {
+    return this.kv.delete(key).catch(() => false);
+  }
+
+  async has(key: string): Promise<boolean> {
+    return this.kv.has(key).catch(() => false);
+  }
+
+  async ttl(key: string): Promise<number> {
+    return this.kv
+      .get(key, { raw: true })
+      .then(raw => (raw?.expires ? raw.expires - Date.now() : Infinity))
+      .catch(() => 0);
+  }
+
+  async expire(key: string, ttl: number): Promise<boolean> {
+    const value = await this.kv.get(key);
+    return this.set(key, value, { ttl });
+  }
+
+  // list operations
+  private async getArray<T = unknown>(key: string) {
+    const raw = await this.kv.get(key, { raw: true });
+    if (raw && !Array.isArray(raw.value)) {
+      throw new Error(
+        `Expect an Array keyed by ${key}, but found ${raw.value}`
+      );
+    }
+
+    return raw as Keyv.DeserializedData<T[]>;
+  }
+
+  private async setArray<T = unknown>(
+    key: string,
+    value: T[],
+    opts: CacheSetOptions = {}
+  ) {
+    return this.set(key, value, opts).then(() => value.length);
+  }
+
+  async pushBack<T = unknown>(key: string, ...values: T[]): Promise<number> {
+    let list: any[] = [];
+    let ttl: number | undefined = undefined;
+    const raw = await this.getArray(key);
+    if (raw) {
+      list = raw.value;
+      if (raw.expires) {
+        ttl = raw.expires - Date.now();
+      }
+    }
+
+    list = list.concat(values);
+    return this.setArray(key, list, { ttl });
+  }
+
+  async pushFront<T = unknown>(key: string, ...values: T[]): Promise<number> {
+    let list: any[] = [];
+    let ttl: number | undefined = undefined;
+    const raw = await this.getArray(key);
+    if (raw) {
+      list = raw.value;
+      if (raw.expires) {
+        ttl = raw.expires - Date.now();
+      }
+    }
+
+    list = values.concat(list);
+    return this.setArray(key, list, { ttl });
+  }
+
+  async len(key: string): Promise<number> {
+    return this.getArray(key).then(v => v?.value.length ?? 0);
+  }
+
+  /**
+   * list array elements with `[start, end]`
+   * the end indice is inclusive
+   */
+  async list<T = unknown>(
+    key: string,
+    start: number,
+    end: number
+  ): Promise<T[]> {
+    const raw = await this.getArray<T>(key);
+    if (raw?.value) {
+      start = (raw.value.length + start) % raw.value.length;
+      end = ((raw.value.length + end) % raw.value.length) + 1;
+      return raw.value.slice(start, end);
+    } else {
+      return [];
+    }
+  }
+
+  private async trim<T = unknown>(key: string, start: number, end: number) {
+    const raw = await this.getArray<T>(key);
+    if (raw) {
+      start = (raw.value.length + start) % raw.value.length;
+      // make negative end index work, and end indice is inclusive
+      end = ((raw.value.length + end) % raw.value.length) + 1;
+      const result = raw.value.splice(start, end);
+
+      await this.set(key, raw.value, {
+        ttl: raw.expires ? raw.expires - Date.now() : undefined,
+      });
+
+      return result;
+    }
+
+    return [];
+  }
+
+  async popFront<T = unknown>(key: string, count: number = 1) {
+    return this.trim<T>(key, 0, count - 1);
+  }
+
+  async popBack<T = unknown>(key: string, count: number = 1) {
+    return this.trim<T>(key, -count, count - 1);
+  }
+
+  // map operations
+  private async getMap<T = unknown>(map: string) {
+    const raw = await this.kv.get(map, { raw: true });
+
+    if (raw) {
+      if (typeof raw.value !== 'object') {
+        throw new Error(
+          `Expect an Object keyed by ${map}, but found ${typeof raw}`
+        );
+      }
+
+      if (Array.isArray(raw.value)) {
+        throw new Error(`Expect an Object keyed by ${map}, but found an Array`);
+      }
+    }
+
+    return raw as Keyv.DeserializedData<Record<string, T>>;
+  }
+
+  private async setMap<T = unknown>(
+    map: string,
+    value: Record<string, T>,
+    opts: CacheSetOptions = {}
+  ) {
+    return this.kv.set(map, value, opts.ttl).then(() => true);
+  }
+
+  async mapGet<T = unknown>(map: string, key: string): Promise<T | undefined> {
+    const raw = await this.getMap<T>(map);
+    if (raw?.value) {
+      return raw.value[key];
+    }
+
+    return undefined;
+  }
+
+  async mapSet<T = unknown>(
+    map: string,
+    key: string,
+    value: T
+  ): Promise<boolean> {
+    const raw = await this.getMap(map);
+    const data = raw?.value ?? {};
+
+    data[key] = value;
+
+    return this.setMap(map, data, {
+      ttl: raw?.expires ? raw.expires - Date.now() : undefined,
+    });
+  }
+
+  async mapDelete(map: string, key: string): Promise<boolean> {
+    const raw = await this.getMap(map);
+
+    if (raw?.value) {
+      delete raw.value[key];
+      return this.setMap(map, raw.value, {
+        ttl: raw.expires ? raw.expires - Date.now() : undefined,
+      });
+    }
+
+    return false;
+  }
+
+  async mapIncrease(
+    map: string,
+    key: string,
+    count: number = 1
+  ): Promise<number> {
+    const prev = (await this.mapGet(map, key)) ?? 0;
+
+    if (typeof prev !== 'number') {
+      throw new Error(
+        `Expect a Number keyed by ${key}, but found ${typeof prev}`
+      );
+    }
+
+    const curr = prev + count;
+
+    return (await this.mapSet(map, key, curr)) ? curr : prev;
+  }
+
+  async mapDecrease(
+    map: string,
+    key: string,
+    count: number = 1
+  ): Promise<number> {
+    return this.mapIncrease(map, key, -count);
+  }
+
+  async mapKeys(map: string): Promise<string[]> {
+    const raw = await this.getMap(map);
+    if (raw) {
+      return Object.keys(raw.value);
+    }
+
+    return [];
+  }
+
+  async mapRandomKey(map: string): Promise<string | undefined> {
+    const keys = await this.mapKeys(map);
+    return keys[Math.floor(Math.random() * keys.length)];
+  }
+
+  async mapLen(map: string): Promise<number> {
+    const raw = await this.getMap(map);
+    return raw ? Object.keys(raw.value).length : 0;
+  }
+}

packages/backend/server/src/cache/index.ts

@@ -0,0 +1,24 @@
+import { FactoryProvider, Global, Module } from '@nestjs/common';
+import { Redis } from 'ioredis';
+
+import { Config } from '../config';
+import { LocalCache } from './cache';
+import { RedisCache } from './redis';
+
+const CacheProvider: FactoryProvider = {
+  provide: LocalCache,
+  useFactory: (config: Config) => {
+    return config.redis.enabled
+      ? new RedisCache(new Redis(config.redis))
+      : new LocalCache();
+  },
+  inject: [Config],
+};
+
+@Global()
+@Module({
+  providers: [CacheProvider],
+  exports: [CacheProvider],
+})
+export class CacheModule {}
+export { LocalCache as Cache };

packages/backend/server/src/cache/redis.ts

@@ -0,0 +1,194 @@
+import { Redis } from 'ioredis';
+
+import { Cache, CacheSetOptions } from './cache';
+
+export class RedisCache implements Cache {
+  constructor(private readonly redis: Redis) {}
+
+  // standard operation
+  async get<T = unknown>(key: string): Promise<T> {
+    return this.redis
+      .get(key)
+      .then(v => {
+        if (v) {
+          return JSON.parse(v);
+        }
+        return undefined;
+      })
+      .catch(() => undefined);
+  }
+
+  async set<T = unknown>(
+    key: string,
+    value: T,
+    opts: CacheSetOptions = {}
+  ): Promise<boolean> {
+    if (opts.ttl) {
+      return this.redis
+        .set(key, JSON.stringify(value), 'PX', opts.ttl)
+        .then(() => true)
+        .catch(() => false);
+    }
+
+    return this.redis
+      .set(key, JSON.stringify(value))
+      .then(() => true)
+      .catch(() => false);
+  }
+
+  async increase(key: string, count: number = 1): Promise<number> {
+    return this.redis.incrby(key, count).catch(() => 0);
+  }
+
+  async decrease(key: string, count: number = 1): Promise<number> {
+    return this.redis.decrby(key, count).catch(() => 0);
+  }
+
+  async setnx<T = unknown>(
+    key: string,
+    value: T,
+    opts: CacheSetOptions = {}
+  ): Promise<boolean> {
+    if (opts.ttl) {
+      return this.redis
+        .set(key, JSON.stringify(value), 'PX', opts.ttl, 'NX')
+        .then(v => !!v)
+        .catch(() => false);
+    }
+
+    return this.redis
+      .set(key, JSON.stringify(value), 'NX')
+      .then(v => !!v)
+      .catch(() => false);
+  }
+
+  async delete(key: string): Promise<boolean> {
+    return this.redis
+      .del(key)
+      .then(v => v > 0)
+      .catch(() => false);
+  }
+
+  async has(key: string): Promise<boolean> {
+    return this.redis
+      .exists(key)
+      .then(v => v > 0)
+      .catch(() => false);
+  }
+
+  async ttl(key: string): Promise<number> {
+    return this.redis.ttl(key).catch(() => 0);
+  }
+
+  async expire(key: string, ttl: number): Promise<boolean> {
+    return this.redis
+      .pexpire(key, ttl)
+      .then(v => v > 0)
+      .catch(() => false);
+  }
+
+  // list operations
+  async pushBack<T = unknown>(key: string, ...values: T[]): Promise<number> {
+    return this.redis
+      .rpush(key, ...values.map(v => JSON.stringify(v)))
+      .catch(() => 0);
+  }
+
+  async pushFront<T = unknown>(key: string, ...values: T[]): Promise<number> {
+    return this.redis
+      .lpush(key, ...values.map(v => JSON.stringify(v)))
+      .catch(() => 0);
+  }
+
+  async len(key: string): Promise<number> {
+    return this.redis.llen(key).catch(() => 0);
+  }
+
+  async list<T = unknown>(
+    key: string,
+    start: number,
+    end: number
+  ): Promise<T[]> {
+    return this.redis
+      .lrange(key, start, end)
+      .then(data => data.map(v => JSON.parse(v)))
+      .catch(() => []);
+  }
+
+  async popFront<T = unknown>(key: string, count: number = 1): Promise<T[]> {
+    return this.redis
+      .lpop(key, count)
+      .then(data => (data ?? []).map(v => JSON.parse(v)))
+      .catch(() => []);
+  }
+
+  async popBack<T = unknown>(key: string, count: number = 1): Promise<T[]> {
+    return this.redis
+      .rpop(key, count)
+      .then(data => (data ?? []).map(v => JSON.parse(v)))
+      .catch(() => []);
+  }
+
+  // map operations
+  async mapSet<T = unknown>(
+    map: string,
+    key: string,
+    value: T
+  ): Promise<boolean> {
+    return this.redis
+      .hset(map, key, JSON.stringify(value))
+      .then(v => v > 0)
+      .catch(() => false);
+  }
+
+  async mapIncrease(
+    map: string,
+    key: string,
+    count: number = 1
+  ): Promise<number> {
+    return this.redis.hincrby(map, key, count);
+  }
+
+  async mapDecrease(
+    map: string,
+    key: string,
+    count: number = 1
+  ): Promise<number> {
+    return this.redis.hincrby(map, key, -count);
+  }
+
+  async mapGet<T = unknown>(map: string, key: string): Promise<T | undefined> {
+    return this.redis
+      .hget(map, key)
+      .then(v => (v ? JSON.parse(v) : undefined))
+      .catch(() => undefined);
+  }
+
+  async mapDelete(map: string, key: string): Promise<boolean> {
+    return this.redis
+      .hdel(map, key)
+      .then(v => v > 0)
+      .catch(() => false);
+  }
+
+  async mapKeys(map: string): Promise<string[]> {
+    return this.redis.hkeys(map).catch(() => []);
+  }
+
+  async mapRandomKey(map: string): Promise<string | undefined> {
+    return this.redis
+      .hrandfield(map, 1)
+      .then(v =>
+        typeof v === 'string'
+          ? v
+          : Array.isArray(v)
+            ? (v[0] as string)
+            : undefined
+      )
+      .catch(() => undefined);
+  }
+
+  async mapLen(map: string): Promise<number> {
+    return this.redis.hlen(map).catch(() => 0);
+  }
+}

packages/backend/server/src/config/def.ts

@@ -50,17 +50,17 @@ function boolean(value: string) {
 }
 
 export function parseEnvValue(value: string | undefined, type?: EnvConfigType) {
-  if (typeof value === 'undefined') {
+  if (value === undefined) {
     return;
   }
 
   return type === 'int'
     ? int(value)
     : type === 'float'
-    ? float(value)
-    : type === 'boolean'
-    ? boolean(value)
-    : value;
+      ? float(value)
+      : type === 'boolean'
+        ? boolean(value)
+        : value;
 }
 
 /**
@@ -362,6 +362,14 @@ export interface AFFiNEConfig {
        */
       experimentalMergeWithJwstCodec: boolean;
     };
+    history: {
+      /**
+       * How long the buffer time of creating a new history snapshot when doc get updated.
+       *
+       * in {ms}
+       */
+      interval: number;
+    };
   };
 
   payment: {

packages/backend/server/src/config/default.ts

@@ -209,6 +209,9 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
         updatePollInterval: 3000,
         experimentalMergeWithJwstCodec: false,
       },
+      history: {
+        interval: 1000 * 60 * 10 /* 10 mins */,
+      },
     },
     payment: {
       stripe: {

packages/backend/server/src/config/env.ts

@@ -10,7 +10,7 @@ export function applyEnvToConfig(rawConfig: AFFiNEConfig) {
         ? [config, process.env[env]]
         : [config[0], parseEnvValue(process.env[env], config[1])];
 
-    if (typeof value !== 'undefined') {
+    if (value !== undefined) {
       set(rawConfig, path, value);
     }
   }

packages/backend/server/src/event/events.ts

@@ -0,0 +1,23 @@
+import type { Snapshot, Workspace } from '@prisma/client';
+
+import { Flatten, Payload } from './types';
+
+interface EventDefinitions {
+  workspace: {
+    deleted: Payload<Workspace['id']>;
+  };
+
+  snapshot: {
+    updated: Payload<
+      Pick<Snapshot, 'id' | 'workspaceId'> & {
+        previous: Pick<Snapshot, 'blob' | 'state' | 'updatedAt'>;
+      }
+    >;
+    deleted: Payload<Pick<Snapshot, 'id' | 'workspaceId'>>;
+  };
+}
+
+export type EventKV = Flatten<EventDefinitions>;
+
+export type Event = keyof EventKV;
+export type EventPayload<E extends Event> = EventKV[E];

packages/backend/server/src/event/index.ts

@@ -0,0 +1,45 @@
+import { Global, Injectable, Module } from '@nestjs/common';
+import {
+  EventEmitter2,
+  EventEmitterModule,
+  OnEvent as RawOnEvent,
+} from '@nestjs/event-emitter';
+
+import type { Event, EventPayload } from './events';
+
+@Injectable()
+export class EventEmitter {
+  constructor(private readonly emitter: EventEmitter2) {}
+
+  emit<E extends Event>(event: E, payload: EventPayload<E>) {
+    return this.emitter.emit(event, payload);
+  }
+
+  emitAsync<E extends Event>(event: E, payload: EventPayload<E>) {
+    return this.emitter.emitAsync(event, payload);
+  }
+
+  on<E extends Event>(event: E, handler: (payload: EventPayload<E>) => void) {
+    return this.emitter.on(event, handler);
+  }
+
+  once<E extends Event>(event: E, handler: (payload: EventPayload<E>) => void) {
+    return this.emitter.once(event, handler);
+  }
+}
+
+export const OnEvent = (
+  event: Event,
+  opts?: Parameters<typeof RawOnEvent>[1]
+) => {
+  return RawOnEvent(event, opts);
+};
+
+@Global()
+@Module({
+  imports: [EventEmitterModule.forRoot()],
+  providers: [EventEmitter],
+  exports: [EventEmitter],
+})
+export class EventModule {}
+export { EventPayload };

packages/backend/server/src/event/types.ts

@@ -0,0 +1,33 @@
+export type Payload<T> = {
+  __payload: true;
+  data: T;
+};
+
+export type Join<A extends string, B extends string> = A extends ''
+  ? B
+  : `${A}.${B}`;
+
+export type PathType<T, Path extends string> = string extends Path
+  ? unknown
+  : Path extends keyof T
+    ? T[Path]
+    : Path extends `${infer K}.${infer R}`
+      ? K extends keyof T
+        ? PathType<T[K], R>
+        : unknown
+      : unknown;
+
+export type Leaves<T, P extends string = ''> = T extends Payload<any>
+  ? P
+  : T extends Record<string, any>
+    ? {
+        [K in keyof T]: K extends string ? Leaves<T[K], Join<P, K>> : never;
+      }[keyof T]
+    : never;
+
+export type Flatten<T> = Leaves<T> extends infer R
+  ? {
+      // @ts-expect-error yo, ts can't make it
+      [K in R]: PathType<T, K> extends Payload<infer U> ? U : never;
+    }
+  : never;

packages/backend/server/src/graphql.module.ts

@@ -8,14 +8,13 @@ import { fileURLToPath } from 'url';
 
 import { Config } from './config';
 import { GQLLoggerPlugin } from './graphql/logger-plugin';
-import { Metrics } from './metrics/metrics';
 
 @Global()
 @Module({
   imports: [
     GraphQLModule.forRootAsync<ApolloDriverConfig>({
       driver: ApolloDriver,
-      useFactory: (config: Config, metrics: Metrics) => {
+      useFactory: (config: Config) => {
         return {
           ...config.graphql,
           path: `${config.path}/graphql`,
@@ -31,10 +30,10 @@ import { Metrics } from './metrics/metrics';
             req,
             res,
           }),
-          plugins: [new GQLLoggerPlugin(metrics)],
+          plugins: [new GQLLoggerPlugin()],
         };
       },
-      inject: [Config, Metrics],
+      inject: [Config],
     }),
   ],
 })

packages/backend/server/src/graphql/logger-plugin.ts

@@ -7,40 +7,43 @@ import { Plugin } from '@nestjs/apollo';
 import { Logger } from '@nestjs/common';
 import { Response } from 'express';
 
-import { Metrics } from '../metrics/metrics';
+import { metrics } from '../metrics/metrics';
 import { ReqContext } from '../types';
 
 @Plugin()
 export class GQLLoggerPlugin implements ApolloServerPlugin {
   protected logger = new Logger(GQLLoggerPlugin.name);
 
-  constructor(private readonly metrics: Metrics) {}
-
   requestDidStart(
     reqContext: GraphQLRequestContext<ReqContext>
   ): Promise<GraphQLRequestListener<GraphQLRequestContext<ReqContext>>> {
     const res = reqContext.contextValue.req.res as Response;
     const operation = reqContext.request.operationName;
 
-    this.metrics.gqlRequest(1, { operation });
-    const timer = this.metrics.gqlTimer({ operation });
+    metrics.gql.counter('query_counter').add(1, { operation });
+    const start = Date.now();
 
     return Promise.resolve({
       willSendResponse: () => {
-        const costInMilliseconds = timer() * 1000;
+        const costInMilliseconds = Date.now() - start;
         res.setHeader(
           'Server-Timing',
           `gql;dur=${costInMilliseconds};desc="GraphQL"`
         );
+        metrics.gql
+          .histogram('query_duration')
+          .record(costInMilliseconds, { operation });
         return Promise.resolve();
       },
       didEncounterErrors: () => {
-        this.metrics.gqlError(1, { operation });
-        const costInMilliseconds = timer() * 1000;
+        const costInMilliseconds = Date.now() - start;
         res.setHeader(
           'Server-Timing',
           `gql;dur=${costInMilliseconds};desc="GraphQL ${operation}"`
         );
+        metrics.gql
+          .histogram('query_duration')
+          .record(costInMilliseconds, { operation });
         return Promise.resolve();
       },
     });

packages/backend/server/src/index.ts

@@ -1,22 +1,9 @@
 /// <reference types="./global.d.ts" />
-import { MetricExporter } from '@google-cloud/opentelemetry-cloud-monitoring-exporter';
-import { TraceExporter } from '@google-cloud/opentelemetry-cloud-trace-exporter';
+import { start as startAutoMetrics } from './metrics';
+startAutoMetrics();
+
 import { NestFactory } from '@nestjs/core';
 import type { NestExpressApplication } from '@nestjs/platform-express';
-import {
-  CompositePropagator,
-  W3CBaggagePropagator,
-  W3CTraceContextPropagator,
-} from '@opentelemetry/core';
-import gql from '@opentelemetry/instrumentation-graphql';
-import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
-import ioredis from '@opentelemetry/instrumentation-ioredis';
-import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
-import socketIO from '@opentelemetry/instrumentation-socket.io';
-import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
-import { NodeSDK } from '@opentelemetry/sdk-node';
-import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-node';
-import { PrismaInstrumentation } from '@prisma/instrumentation';
 import cookieParser from 'cookie-parser';
 import { static as staticMiddleware } from 'express';
 import graphqlUploadExpress from 'graphql-upload/graphqlUploadExpress.mjs';
@@ -28,35 +15,6 @@ import { serverTimingAndCache } from './middleware/timing';
 import { RedisIoAdapter } from './modules/sync/redis-adapter';
 
 const { NODE_ENV, AFFINE_ENV } = process.env;
-
-if (NODE_ENV === 'production') {
-  const traceExporter = new TraceExporter();
-  const tracing = new NodeSDK({
-    traceExporter,
-    metricReader: new PeriodicExportingMetricReader({
-      exporter: new MetricExporter(),
-    }),
-    spanProcessor: new BatchSpanProcessor(traceExporter),
-    textMapPropagator: new CompositePropagator({
-      propagators: [
-        new W3CBaggagePropagator(),
-        new W3CTraceContextPropagator(),
-      ],
-    }),
-    instrumentations: [
-      new NestInstrumentation(),
-      new ioredis.IORedisInstrumentation(),
-      new socketIO.SocketIoInstrumentation({ traceReserved: true }),
-      new gql.GraphQLInstrumentation({ mergeItems: true }),
-      new HttpInstrumentation(),
-      new PrismaInstrumentation(),
-    ],
-    serviceName: 'affine-cloud',
-  });
-
-  tracing.start();
-}
-
 const app = await NestFactory.create<NestExpressApplication>(AppModule, {
   cors: true,
   rawBody: true,

packages/backend/server/src/metrics/controller.ts

@@ -1,18 +0,0 @@
-import { Controller, Get, Res } from '@nestjs/common';
-import type { Response } from 'express';
-import { register } from 'prom-client';
-
-import { PrismaService } from '../prisma';
-
-@Controller()
-export class MetricsController {
-  constructor(private readonly prisma: PrismaService) {}
-
-  @Get('/metrics')
-  async index(@Res() res: Response): Promise<void> {
-    res.header('Content-Type', register.contentType);
-    const prismaMetrics = await this.prisma.$metrics.prometheus();
-    const appMetrics = await register.metrics();
-    res.send(appMetrics + prismaMetrics);
-  }
-}

packages/backend/server/src/metrics/index.ts

@@ -1,12 +1,3 @@
-import { Global, Module } from '@nestjs/common';
-
-import { MetricsController } from '../metrics/controller';
-import { Metrics } from './metrics';
-
-@Global()
-@Module({
-  providers: [Metrics],
-  exports: [Metrics],
-  controllers: [MetricsController],
-})
-export class MetricsModule {}
+export * from './metrics';
+export { start } from './opentelemetry';
+export * from './utils';

packages/backend/server/src/metrics/metrics.ts

@@ -1,28 +1,129 @@
-import { Injectable, OnModuleDestroy } from '@nestjs/common';
-import { register } from 'prom-client';
+import {
+  Attributes,
+  Counter,
+  Histogram,
+  Meter,
+  MetricOptions,
+} from '@opentelemetry/api';
 
-import { metricsCreator } from './utils';
+import { getMeter } from './opentelemetry';
 
-@Injectable()
-export class Metrics implements OnModuleDestroy {
-  onModuleDestroy(): void {
-    register.clear();
+type MetricType = 'counter' | 'gauge' | 'histogram';
+type Metric<T extends MetricType> = T extends 'counter'
+  ? Counter
+  : T extends 'gauge'
+    ? Histogram
+    : T extends 'histogram'
+      ? Histogram
+      : never;
+
+export type ScopedMetrics = {
+  [T in MetricType]: (name: string, opts?: MetricOptions) => Metric<T>;
+};
+type MetricCreators = {
+  [T in MetricType]: (
+    meter: Meter,
+    name: string,
+    opts?: MetricOptions
+  ) => Metric<T>;
+};
+
+export type KnownMetricScopes =
+  | 'socketio'
+  | 'gql'
+  | 'jwst'
+  | 'auth'
+  | 'controllers'
+  | 'doc';
+
+const metricCreators: MetricCreators = {
+  counter(meter: Meter, name: string, opts?: MetricOptions) {
+    return meter.createCounter(name, opts);
+  },
+  gauge(meter: Meter, name: string, opts?: MetricOptions) {
+    let value: any;
+    let attrs: Attributes | undefined;
+    const ob = meter.createObservableGauge(name, opts);
+
+    ob.addCallback(result => {
+      result.observe(value, attrs);
+    });
+
+    return {
+      record: (newValue, newAttrs) => {
+        value = newValue;
+        attrs = newAttrs;
+      },
+    } satisfies Histogram;
+  },
+  histogram(meter: Meter, name: string, opts?: MetricOptions) {
+    return meter.createHistogram(name, opts);
+  },
+};
+
+const scopes = new Map<string, ScopedMetrics>();
+
+function make(scope: string) {
+  const meter = getMeter();
+  const metrics = new Map<string, { type: MetricType; metric: any }>();
+  const prefix = scope + '/';
+
+  function getOrCreate<T extends MetricType>(
+    type: T,
+    name: string,
+    opts?: MetricOptions
+  ): Metric<T> {
+    name = prefix + name;
+    const metric = metrics.get(name);
+    if (metric) {
+      if (type !== metric.type) {
+        throw new Error(
+          `Metric ${name} has already been registered as ${metric.type} mode, but get as ${type} again.`
+        );
+      }
+
+      return metric.metric;
+    } else {
+      const metric = metricCreators[type](meter, name, opts);
+      metrics.set(name, { type, metric });
+      return metric;
+    }
   }
 
-  socketIOEventCounter = metricsCreator.counter('socket_io_counter', ['event']);
-  socketIOEventTimer = metricsCreator.timer('socket_io_timer', ['event']);
-  socketIOConnectionGauge = metricsCreator.gauge(
-    'socket_io_connection_counter'
-  );
-
-  gqlRequest = metricsCreator.counter('gql_request', ['operation']);
-  gqlError = metricsCreator.counter('gql_error', ['operation']);
-  gqlTimer = metricsCreator.timer('gql_timer', ['operation']);
-
-  jwstCodecMerge = metricsCreator.counter('jwst_codec_merge');
-  jwstCodecDidnotMatch = metricsCreator.counter('jwst_codec_didnot_match');
-  jwstCodecFail = metricsCreator.counter('jwst_codec_fail');
-
-  authCounter = metricsCreator.counter('auth');
-  authFailCounter = metricsCreator.counter('auth_fail', ['reason']);
+  return {
+    counter(name, opts) {
+      return getOrCreate('counter', name, opts);
+    },
+    gauge(name, opts) {
+      return getOrCreate('gauge', name, opts);
+    },
+    histogram(name, opts) {
+      return getOrCreate('histogram', name, opts);
+    },
+  } satisfies ScopedMetrics;
 }
+
+/**
+ * @example
+ *
+ * ```
+ * metrics.scope.counter('example_count').add(1, {
+ *   attr1: 'example-event'
+ * })
+ * ```
+ */
+export const metrics = new Proxy<Record<KnownMetricScopes, ScopedMetrics>>(
+  // @ts-expect-error proxied
+  {},
+  {
+    get(_, scopeName: string) {
+      let scope = scopes.get(scopeName);
+      if (!scope) {
+        scope = make(scopeName);
+        scopes.set(scopeName, scope);
+      }
+
+      return scope;
+    },
+  }
+);

packages/backend/server/src/metrics/opentelemetry.ts

@@ -0,0 +1,162 @@
+import { MetricExporter } from '@google-cloud/opentelemetry-cloud-monitoring-exporter';
+import { TraceExporter } from '@google-cloud/opentelemetry-cloud-trace-exporter';
+import { metrics } from '@opentelemetry/api';
+import {
+  CompositePropagator,
+  W3CBaggagePropagator,
+  W3CTraceContextPropagator,
+} from '@opentelemetry/core';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { ZipkinExporter } from '@opentelemetry/exporter-zipkin';
+import { HostMetrics } from '@opentelemetry/host-metrics';
+import { Instrumentation } from '@opentelemetry/instrumentation';
+import { GraphQLInstrumentation } from '@opentelemetry/instrumentation-graphql';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
+import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
+import { SocketIoInstrumentation } from '@opentelemetry/instrumentation-socket.io';
+import {
+  ConsoleMetricExporter,
+  type MeterProvider,
+  MetricProducer,
+  MetricReader,
+  PeriodicExportingMetricReader,
+} from '@opentelemetry/sdk-metrics';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import {
+  BatchSpanProcessor,
+  ConsoleSpanExporter,
+  SpanExporter,
+  TraceIdRatioBasedSampler,
+} from '@opentelemetry/sdk-trace-node';
+import { PrismaInstrumentation } from '@prisma/instrumentation';
+
+import { PrismaMetricProducer } from './prisma';
+
+abstract class OpentelemetryFactor {
+  abstract getMetricReader(): MetricReader;
+  abstract getSpanExporter(): SpanExporter;
+
+  getInstractions(): Instrumentation[] {
+    return [
+      new NestInstrumentation(),
+      new IORedisInstrumentation(),
+      new SocketIoInstrumentation({ traceReserved: true }),
+      new GraphQLInstrumentation({ mergeItems: true }),
+      new HttpInstrumentation(),
+      new PrismaInstrumentation(),
+    ];
+  }
+
+  getMetricsProducers(): MetricProducer[] {
+    return [new PrismaMetricProducer()];
+  }
+
+  create() {
+    const traceExporter = this.getSpanExporter();
+    return new NodeSDK({
+      sampler: new TraceIdRatioBasedSampler(0.1),
+      traceExporter,
+      metricReader: this.getMetricReader(),
+      spanProcessor: new BatchSpanProcessor(traceExporter),
+      textMapPropagator: new CompositePropagator({
+        propagators: [
+          new W3CBaggagePropagator(),
+          new W3CTraceContextPropagator(),
+        ],
+      }),
+      instrumentations: this.getInstractions(),
+      serviceName: 'affine-cloud',
+    });
+  }
+}
+
+class GCloudOpentelemetryFactor extends OpentelemetryFactor {
+  override getMetricReader(): MetricReader {
+    return new PeriodicExportingMetricReader({
+      exportIntervalMillis: 30000,
+      exportTimeoutMillis: 10000,
+      exporter: new MetricExporter({
+        prefix: 'custom.googleapis.com',
+      }),
+      metricProducers: this.getMetricsProducers(),
+    });
+  }
+
+  override getSpanExporter(): SpanExporter {
+    return new TraceExporter();
+  }
+}
+
+class LocalOpentelemetryFactor extends OpentelemetryFactor {
+  override getMetricReader(): MetricReader {
+    return new PrometheusExporter({
+      metricProducers: this.getMetricsProducers(),
+    });
+  }
+
+  override getSpanExporter(): SpanExporter {
+    return new ZipkinExporter();
+  }
+}
+
+class DebugOpentelemetryFactor extends OpentelemetryFactor {
+  override getMetricReader(): MetricReader {
+    return new PeriodicExportingMetricReader({
+      exporter: new ConsoleMetricExporter(),
+      metricProducers: this.getMetricsProducers(),
+    });
+  }
+
+  override getSpanExporter(): SpanExporter {
+    return new ConsoleSpanExporter();
+  }
+}
+
+function createSDK() {
+  let factor: OpentelemetryFactor | null = null;
+  if (process.env.NODE_ENV === 'production') {
+    factor = new GCloudOpentelemetryFactor();
+  } else if (process.env.DEBUG_METRICS) {
+    factor = new DebugOpentelemetryFactor();
+  } else {
+    factor = new LocalOpentelemetryFactor();
+  }
+
+  return factor?.create();
+}
+
+let OPENTELEMETRY_STARTED = false;
+
+function ensureStarted() {
+  if (!OPENTELEMETRY_STARTED) {
+    OPENTELEMETRY_STARTED = true;
+    start();
+  }
+}
+
+function getMeterProvider() {
+  ensureStarted();
+  return metrics.getMeterProvider();
+}
+
+function registerCustomMetrics() {
+  const hostMetricsMonitoring = new HostMetrics({
+    name: 'instance-host-metrics',
+    meterProvider: getMeterProvider() as MeterProvider,
+  });
+  hostMetricsMonitoring.start();
+}
+
+export function getMeter(name = 'business') {
+  return getMeterProvider().getMeter(name);
+}
+
+export function start() {
+  const sdk = createSDK();
+
+  if (sdk) {
+    sdk.start();
+    registerCustomMetrics();
+  }
+}

packages/backend/server/src/metrics/prisma.ts

@@ -0,0 +1,132 @@
+import { HrTime, ValueType } from '@opentelemetry/api';
+import { hrTime } from '@opentelemetry/core';
+import { Resource } from '@opentelemetry/resources';
+import {
+  AggregationTemporality,
+  CollectionResult,
+  DataPointType,
+  InstrumentType,
+  MetricProducer,
+  ScopeMetrics,
+} from '@opentelemetry/sdk-metrics';
+
+import { PrismaService } from '../prisma';
+
+function transformPrismaKey(key: string) {
+  // replace first '_' to '/' as a scope prefix
+  // example: prisma_client_query_duration_seconds_sum -> prisma/client_query_duration_seconds_sum
+  return key.replace(/_/, '/');
+}
+
+export class PrismaMetricProducer implements MetricProducer {
+  private readonly startTime: HrTime = hrTime();
+
+  async collect(): Promise<CollectionResult> {
+    const result: CollectionResult = {
+      resourceMetrics: {
+        resource: Resource.EMPTY,
+        scopeMetrics: [],
+      },
+      errors: [],
+    };
+
+    if (!PrismaService.INSTANCE) {
+      return result;
+    }
+
+    const prisma = PrismaService.INSTANCE;
+
+    const endTime = hrTime();
+
+    const metrics = await prisma.$metrics.json();
+    const scopeMetrics: ScopeMetrics = {
+      scope: {
+        name: '',
+      },
+      metrics: [],
+    };
+    for (const counter of metrics.counters) {
+      scopeMetrics.metrics.push({
+        descriptor: {
+          name: transformPrismaKey(counter.key),
+          description: counter.description,
+          unit: '1',
+          type: InstrumentType.COUNTER,
+          valueType: ValueType.INT,
+        },
+        dataPointType: DataPointType.SUM,
+        aggregationTemporality: AggregationTemporality.CUMULATIVE,
+        dataPoints: [
+          {
+            startTime: this.startTime,
+            endTime: endTime,
+            value: counter.value,
+            attributes: counter.labels,
+          },
+        ],
+        isMonotonic: true,
+      });
+    }
+
+    for (const gauge of metrics.gauges) {
+      scopeMetrics.metrics.push({
+        descriptor: {
+          name: transformPrismaKey(gauge.key),
+          description: gauge.description,
+          unit: '1',
+          type: InstrumentType.UP_DOWN_COUNTER,
+          valueType: ValueType.INT,
+        },
+        dataPointType: DataPointType.GAUGE,
+        aggregationTemporality: AggregationTemporality.CUMULATIVE,
+        dataPoints: [
+          {
+            startTime: this.startTime,
+            endTime: endTime,
+            value: gauge.value,
+            attributes: gauge.labels,
+          },
+        ],
+      });
+    }
+
+    for (const histogram of metrics.histograms) {
+      const boundaries = [];
+      const counts = [];
+      for (const [boundary, count] of histogram.value.buckets) {
+        boundaries.push(boundary);
+        counts.push(count);
+      }
+      scopeMetrics.metrics.push({
+        descriptor: {
+          name: transformPrismaKey(histogram.key),
+          description: histogram.description,
+          unit: 'ms',
+          type: InstrumentType.HISTOGRAM,
+          valueType: ValueType.DOUBLE,
+        },
+        dataPointType: DataPointType.HISTOGRAM,
+        aggregationTemporality: AggregationTemporality.CUMULATIVE,
+        dataPoints: [
+          {
+            startTime: this.startTime,
+            endTime: endTime,
+            value: {
+              buckets: {
+                boundaries,
+                counts,
+              },
+              count: histogram.value.count,
+              sum: histogram.value.sum,
+            },
+            attributes: histogram.labels,
+          },
+        ],
+      });
+    }
+
+    result.resourceMetrics.scopeMetrics.push(scopeMetrics);
+
+    return result;
+  }
+}

packages/backend/server/src/metrics/utils.ts

@@ -1,99 +1,12 @@
-import { Counter, Gauge, register, Summary } from 'prom-client';
+import { Attributes } from '@opentelemetry/api';
 
-function getOr<T>(name: string, or: () => T): T {
-  return (register.getSingleMetric(name) as T) || or();
-}
-
-type LabelValues<T extends string> = Partial<Record<T, string | number>>;
-type MetricsCreator<T extends string> = (
-  value: number,
-  labels: LabelValues<T>
-) => void;
-type TimerMetricsCreator<T extends string> = (
-  labels: LabelValues<T>
-) => () => number;
-
-export const metricsCreatorGenerator = () => {
-  const counterCreator = <T extends string>(
-    name: string,
-    labelNames?: T[]
-  ): MetricsCreator<T> => {
-    const counter = getOr(
-      name,
-      () =>
-        new Counter({
-          name,
-          help: name,
-          ...(labelNames ? { labelNames } : {}),
-        })
-    );
-
-    return (value: number, labels: LabelValues<T>) => {
-      counter.inc(labels, value);
-    };
-  };
-
-  const gaugeCreator = <T extends string>(
-    name: string,
-    labelNames?: T[]
-  ): MetricsCreator<T> => {
-    const gauge = getOr(
-      name,
-      () =>
-        new Gauge({
-          name,
-          help: name,
-          ...(labelNames ? { labelNames } : {}),
-        })
-    );
-
-    return (value: number, labels: LabelValues<T>) => {
-      gauge.set(labels, value);
-    };
-  };
-
-  const timerCreator = <T extends string>(
-    name: string,
-    labelNames?: T[]
-  ): TimerMetricsCreator<T> => {
-    const summary = getOr(
-      name,
-      () =>
-        new Summary({
-          name,
-          help: name,
-          ...(labelNames ? { labelNames } : {}),
-        })
-    );
-
-    return (labels: LabelValues<T>) => {
-      const now = process.hrtime();
-
-      return () => {
-        const delta = process.hrtime(now);
-        const value = delta[0] + delta[1] / 1e9;
-
-        summary.observe(labels, value);
-        return value;
-      };
-    };
-  };
-
-  return {
-    counter: counterCreator,
-    gauge: gaugeCreator,
-    timer: timerCreator,
-  };
-};
-
-export const metricsCreator = metricsCreatorGenerator();
+import { KnownMetricScopes, metrics } from './metrics';
 
 export const CallTimer = (
+  scope: KnownMetricScopes,
   name: string,
-  labels: Record<string, any> = {}
+  attrs?: Attributes
 ): MethodDecorator => {
-  const timer = metricsCreator.timer(name, Object.keys(labels));
-
   // @ts-expect-error allow
   return (
     _target,
@@ -106,19 +19,29 @@ export const CallTimer = (
     }
 
     desc.value = function (...args: any[]) {
-      const endTimer = timer(labels);
+      const timer = metrics[scope].histogram(name, {
+        description: `function call time costs of ${name}`,
+        unit: 'ms',
+      });
+
+      const start = Date.now();
+
+      const end = () => {
+        timer.record(Date.now() - start, attrs);
+      };
+
       let result: any;
       try {
         result = originalMethod.apply(this, args);
       } catch (e) {
-        endTimer();
+        end();
         throw e;
       }
 
       if (result instanceof Promise) {
-        return result.finally(endTimer);
+        return result.finally(end);
       } else {
-        endTimer();
+        end();
         return result;
       }
     };
@@ -128,11 +51,10 @@ export const CallTimer = (
 };
 
 export const CallCounter = (
+  scope: KnownMetricScopes,
   name: string,
-  labels: Record<string, any> = {}
+  attrs?: Attributes
 ): MethodDecorator => {
-  const count = metricsCreator.counter(name, Object.keys(labels));
-
   // @ts-expect-error allow
   return (
     _target,
@@ -145,7 +67,11 @@ export const CallCounter = (
     }
 
     desc.value = function (...args: any[]) {
-      count(1, labels);
+      const count = metrics[scope].counter(name, {
+        description: `function call counter of ${name}`,
+      });
+
+      count.add(1, attrs);
       return originalMethod.apply(this, args);
     };
 

packages/backend/server/src/middleware/exception-logger.ts

@@ -14,7 +14,7 @@ const TrivialExceptions = [NotFoundException];
 
 @Catch()
 export class ExceptionLogger implements ExceptionFilter {
-  private logger = new Logger('ExceptionLogger');
+  private readonly logger = new Logger('ExceptionLogger');
 
   catch(exception: Error, host: ArgumentsHost) {
     // with useGlobalFilters, the context is always HTTP

packages/backend/server/src/modules/auth/guard.ts

@@ -53,8 +53,8 @@ class AuthGuard implements CanActivate {
   constructor(
     @Inject(NextAuthOptionsProvide)
     private readonly nextAuthOptions: NextAuthOptions,
-    private auth: AuthService,
-    private prisma: PrismaService,
+    private readonly auth: AuthService,
+    private readonly prisma: PrismaService,
     private readonly reflector: Reflector
   ) {}
 

packages/backend/server/src/modules/auth/next-auth.controller.ts

@@ -23,7 +23,7 @@ import type { AuthAction, CookieOption, NextAuthOptions } from 'next-auth';
 import { AuthHandler } from 'next-auth/core';
 
 import { Config } from '../../config';
-import { Metrics } from '../../metrics/metrics';
+import { metrics } from '../../metrics';
 import { PrismaService } from '../../prisma/service';
 import { SessionService } from '../../session';
 import { AuthThrottlerGuard, Throttle } from '../../throttler';
@@ -46,7 +46,6 @@ export class NextAuthController {
     private readonly authService: AuthService,
     @Inject(NextAuthOptionsProvide)
     private readonly nextAuthOptions: NextAuthOptions,
-    private readonly metrics: Metrics,
     private readonly session: SessionService
   ) {
     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -90,12 +89,13 @@ export class NextAuthController {
       res.redirect(`/signin${query}`);
       return;
     }
-    this.metrics.authCounter(1, {});
     const [action, providerId] = req.url // start with request url
       .slice(BASE_URL.length) // make relative to baseUrl
       .replace(/\?.*/, '') // remove query part, use only path part
       .split('/') as [AuthAction, string]; // as array of strings;
 
+    metrics.auth.counter('call_counter').add(1, { action, providerId });
+
     const credentialsSignIn =
       req.method === 'POST' && providerId === 'credentials';
     let userId: string | undefined;
@@ -127,7 +127,9 @@ export class NextAuthController {
     const options = this.nextAuthOptions;
     if (req.method === 'POST' && action === 'session') {
       if (typeof req.body !== 'object' || typeof req.body.data !== 'object') {
-        this.metrics.authFailCounter(1, { reason: 'invalid_session_data' });
+        metrics.auth
+          .counter('call_fails_counter')
+          .add(1, { reason: 'invalid_session_data' });
         throw new BadRequestException(`Invalid new session data`);
       }
       const user = await this.updateSession(req, req.body.data);
@@ -210,9 +212,10 @@ export class NextAuthController {
 
     if (redirect?.endsWith('api/auth/error?error=AccessDenied')) {
       this.logger.log(`Early access redirect headers: ${req.headers}`);
-      this.metrics.authFailCounter(1, {
-        reason: 'no_early_access_permission',
-      });
+      metrics.auth
+        .counter('call_fails_counter')
+        .add(1, { reason: 'no_early_access_permission' });
+
       if (
         !req.headers?.referer ||
         checkUrlOrigin(req.headers.referer, 'https://accounts.google.com')

packages/backend/server/src/modules/auth/service.ts

@@ -23,14 +23,14 @@ export type UserClaim = Pick<
   hasPassword?: boolean;
 };
 
-export const getUtcTimestamp = () => Math.floor(new Date().getTime() / 1000);
+export const getUtcTimestamp = () => Math.floor(Date.now() / 1000);
 
 @Injectable()
 export class AuthService {
   constructor(
-    private config: Config,
-    private prisma: PrismaService,
-    private mailer: MailService
+    private readonly config: Config,
+    private readonly prisma: PrismaService,
+    private readonly mailer: MailService
   ) {}
 
   sign(user: UserClaim) {

packages/backend/server/src/modules/config.ts

@@ -0,0 +1,30 @@
+import { Module } from '@nestjs/common';
+import { Field, ObjectType, Query } from '@nestjs/graphql';
+
+export const { SERVER_FLAVOR } = process.env;
+
+@ObjectType()
+export class ServerConfigType {
+  @Field({ description: 'server version' })
+  version!: string;
+
+  @Field({ description: 'server flavor' })
+  flavor!: string;
+}
+
+export class ServerConfigResolver {
+  @Query(() => ServerConfigType, {
+    description: 'server config',
+  })
+  serverConfig(): ServerConfigType {
+    return {
+      version: AFFiNE.version,
+      flavor: SERVER_FLAVOR || 'allinone',
+    };
+  }
+}
+
+@Module({
+  providers: [ServerConfigResolver],
+})
+export class ServerConfigModule {}

packages/backend/server/src/modules/doc/history.ts

@@ -0,0 +1,275 @@
+import { isDeepStrictEqual } from 'node:util';
+
+import { Injectable, Logger } from '@nestjs/common';
+import { Cron, CronExpression } from '@nestjs/schedule';
+
+import { Config } from '../../config';
+import { type EventPayload, OnEvent } from '../../event';
+import { metrics } from '../../metrics';
+import { PrismaService } from '../../prisma';
+import { SubscriptionStatus } from '../payment/service';
+import { Permission } from '../workspaces/types';
+import { isEmptyBuffer } from './manager';
+
+@Injectable()
+export class DocHistoryManager {
+  private readonly logger = new Logger(DocHistoryManager.name);
+  constructor(
+    private readonly config: Config,
+    private readonly db: PrismaService
+  ) {}
+
+  @OnEvent('workspace.deleted')
+  onWorkspaceDeleted(workspaceId: EventPayload<'workspace.deleted'>) {
+    return this.db.snapshotHistory.deleteMany({
+      where: {
+        workspaceId,
+      },
+    });
+  }
+
+  @OnEvent('snapshot.deleted')
+  onSnapshotDeleted({ workspaceId, id }: EventPayload<'snapshot.deleted'>) {
+    return this.db.snapshotHistory.deleteMany({
+      where: {
+        workspaceId,
+        id,
+      },
+    });
+  }
+
+  @OnEvent('snapshot.updated')
+  async onDocUpdated(
+    { workspaceId, id, previous }: EventPayload<'snapshot.updated'>,
+    forceCreate = false
+  ) {
+    const last = await this.last(workspaceId, id);
+
+    let shouldCreateHistory = false;
+
+    if (!last) {
+      // never created
+      shouldCreateHistory = true;
+    } else if (last.timestamp === previous.updatedAt) {
+      // no change
+      shouldCreateHistory = false;
+    } else if (
+      // force
+      forceCreate ||
+      // last history created before interval in configs
+      last.timestamp.getTime() <
+        previous.updatedAt.getTime() - this.config.doc.history.interval
+    ) {
+      shouldCreateHistory = true;
+    }
+
+    if (shouldCreateHistory) {
+      // skip the history recording when no actual update on snapshot happended
+      if (last && isDeepStrictEqual(last.state, previous.state)) {
+        this.logger.debug(
+          `State matches, skip creating history record for ${id} in workspace ${workspaceId}`
+        );
+        return;
+      }
+
+      if (isEmptyBuffer(previous.blob)) {
+        this.logger.debug(
+          `Doc is empty, skip creating history record for ${id} in workspace ${workspaceId}`
+        );
+        return;
+      }
+
+      await this.db.snapshotHistory
+        .create({
+          select: {
+            timestamp: true,
+          },
+          data: {
+            workspaceId,
+            id,
+            timestamp: previous.updatedAt,
+            blob: previous.blob,
+            state: previous.state,
+            expiredAt: await this.getExpiredDateFromNow(workspaceId),
+          },
+        })
+        .catch(() => {
+          // safe to ignore
+          // only happens when duplicated history record created in multi processes
+        });
+      metrics.doc
+        .counter('history_created_counter', {
+          description: 'How many times the snapshot history created',
+        })
+        .add(1);
+      this.logger.log(`History created for ${id} in workspace ${workspaceId}.`);
+    }
+  }
+
+  async list(
+    workspaceId: string,
+    id: string,
+    before: Date = new Date(),
+    take: number = 10
+  ) {
+    return this.db.snapshotHistory.findMany({
+      select: {
+        timestamp: true,
+      },
+      where: {
+        workspaceId,
+        id,
+        timestamp: {
+          lt: before,
+        },
+        // only include the ones has not expired
+        expiredAt: {
+          gt: new Date(),
+        },
+      },
+      orderBy: {
+        timestamp: 'desc',
+      },
+      take,
+    });
+  }
+
+  async count(workspaceId: string, id: string) {
+    return this.db.snapshotHistory.count({
+      where: {
+        workspaceId,
+        id,
+        expiredAt: {
+          gt: new Date(),
+        },
+      },
+    });
+  }
+
+  async get(workspaceId: string, id: string, timestamp: Date) {
+    return this.db.snapshotHistory.findUnique({
+      where: {
+        workspaceId_id_timestamp: {
+          workspaceId,
+          id,
+          timestamp,
+        },
+        expiredAt: {
+          gt: new Date(),
+        },
+      },
+    });
+  }
+
+  async last(workspaceId: string, id: string) {
+    return this.db.snapshotHistory.findFirst({
+      where: {
+        workspaceId,
+        id,
+      },
+      select: {
+        timestamp: true,
+        state: true,
+      },
+      orderBy: {
+        timestamp: 'desc',
+      },
+    });
+  }
+
+  async recover(workspaceId: string, id: string, timestamp: Date) {
+    const history = await this.db.snapshotHistory.findUnique({
+      where: {
+        workspaceId_id_timestamp: {
+          workspaceId,
+          id,
+          timestamp,
+        },
+      },
+    });
+
+    if (!history) {
+      throw new Error('Given history not found');
+    }
+
+    const oldSnapshot = await this.db.snapshot.findUnique({
+      where: {
+        id_workspaceId: {
+          id,
+          workspaceId,
+        },
+      },
+    });
+
+    if (!oldSnapshot) {
+      // unreachable actually
+      throw new Error('Given Doc not found');
+    }
+
+    // save old snapshot as one history record
+    await this.onDocUpdated({ workspaceId, id, previous: oldSnapshot }, true);
+    // WARN:
+    //  we should never do the snapshot updating in recovering,
+    //  which is not the solution in CRDT.
+    //  let user revert in client and update the data in sync system
+    //    `await this.db.snapshot.update();`
+    metrics.doc
+      .counter('history_recovered_counter', {
+        description: 'How many times history recovered request happened',
+      })
+      .add(1);
+
+    return history.timestamp;
+  }
+
+  /**
+   * @todo(@darkskygit) refactor with [Usage Control] system
+   */
+  async getExpiredDateFromNow(workspaceId: string) {
+    const permission = await this.db.workspaceUserPermission.findFirst({
+      select: {
+        userId: true,
+      },
+      where: {
+        workspaceId,
+        type: Permission.Owner,
+      },
+    });
+
+    if (!permission) {
+      // unreachable actually
+      throw new Error('Workspace owner not found');
+    }
+
+    const sub = await this.db.userSubscription.findFirst({
+      select: {
+        id: true,
+      },
+      where: {
+        userId: permission.userId,
+        status: SubscriptionStatus.Active,
+      },
+    });
+
+    return new Date(
+      Date.now() +
+        1000 *
+          60 *
+          60 *
+          24 *
+          // 30 days for subscription user, 7 days for free user
+          (sub ? 30 : 7)
+    );
+  }
+
+  @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT /* everyday at 12am */)
+  async cleanupExpiredHistory() {
+    await this.db.snapshotHistory.deleteMany({
+      where: {
+        expiredAt: {
+          lte: new Date(),
+        },
+      },
+    });
+  }
+}

packages/backend/server/src/modules/doc/index.ts

@@ -1,7 +1,7 @@
 import { DynamicModule } from '@nestjs/common';
 
+import { DocHistoryManager } from './history';
 import { DocManager } from './manager';
-import { RedisDocManager } from './redis-manager';
 
 export class DocModule {
   /**
@@ -15,14 +15,10 @@ export class DocModule {
           provide: 'DOC_MANAGER_AUTOMATION',
           useValue: automation,
         },
-        {
-          provide: DocManager,
-          useClass: globalThis.AFFiNE.redis.enabled
-            ? RedisDocManager
-            : DocManager,
-        },
+        DocManager,
+        DocHistoryManager,
       ],
-      exports: [DocManager],
+      exports: [DocManager, DocHistoryManager],
     };
   }
 
@@ -39,4 +35,4 @@ export class DocModule {
   }
 }
 
-export { DocManager };
+export { DocHistoryManager, DocManager };

packages/backend/server/src/modules/doc/manager.ts

@@ -10,14 +10,17 @@ import { chunk } from 'lodash-es';
 import { defer, retry } from 'rxjs';
 import {
   applyUpdate,
+  decodeStateVector,
   Doc,
   encodeStateAsUpdate,
   encodeStateVector,
   transact,
 } from 'yjs';
 
+import { Cache } from '../../cache';
 import { Config } from '../../config';
-import { Metrics } from '../../metrics/metrics';
+import { EventEmitter, type EventPayload, OnEvent } from '../../event';
+import { metrics } from '../../metrics/metrics';
 import { PrismaService } from '../../prisma';
 import { mergeUpdatesInApplyWay as jwstMergeUpdates } from '../../storage';
 
@@ -38,9 +41,39 @@ function compare(yBinary: Buffer, jwstBinary: Buffer, strict = false): boolean {
   return compare(yBinary, yBinary2, true);
 }
 
-function isEmptyBuffer(buf: Buffer): boolean {
+/**
+ * Detect whether rhs state is newer than lhs state.
+ *
+ * How could we tell a state is newer:
+ *
+ *   i. if the state vector size is larger, it's newer
+ *  ii. if the state vector size is same, compare each client's state
+ */
+function isStateNewer(lhs: Buffer, rhs: Buffer): boolean {
+  const lhsVector = decodeStateVector(lhs);
+  const rhsVector = decodeStateVector(rhs);
+
+  if (lhsVector.size < rhsVector.size) {
+    return true;
+  }
+
+  for (const [client, state] of lhsVector) {
+    const rstate = rhsVector.get(client);
+    if (!rstate) {
+      return false;
+    }
+
+    if (state < rstate) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+export function isEmptyBuffer(buf: Buffer): boolean {
   return (
-    buf.length == 0 ||
+    buf.length === 0 ||
     // 0x0000
     (buf.length === 2 && buf[0] === 0 && buf[1] === 0)
   );
@@ -58,17 +91,18 @@ const MAX_SEQ_NUM = 0x3fffffff; // u31
  */
 @Injectable()
 export class DocManager implements OnModuleInit, OnModuleDestroy {
-  protected logger = new Logger(DocManager.name);
+  private readonly logger = new Logger(DocManager.name);
   private job: NodeJS.Timeout | null = null;
-  private seqMap = new Map<string, number>();
+  private readonly seqMap = new Map<string, number>();
   private busy = false;
 
   constructor(
-    protected readonly db: PrismaService,
     @Inject('DOC_MANAGER_AUTOMATION')
-    protected readonly automation: boolean,
-    protected readonly config: Config,
-    protected readonly metrics: Metrics
+    private readonly automation: boolean,
+    private readonly db: PrismaService,
+    private readonly config: Config,
+    private readonly cache: Cache,
+    private readonly event: EventEmitter
   ) {}
 
   onModuleInit() {
@@ -82,7 +116,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     this.destroy();
   }
 
-  protected recoverDoc(...updates: Buffer[]): Promise<Doc> {
+  private recoverDoc(...updates: Buffer[]): Promise<Doc> {
     const doc = new Doc();
     const chunks = chunk(updates, 10);
 
@@ -95,11 +129,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
               try {
                 applyUpdate(doc, u);
               } catch (e) {
-                this.logger.error(
-                  `Failed to apply update: ${updates
-                    .map(u => u.toString('hex'))
-                    .join('\n')}`
-                );
+                this.logger.error('Failed to apply update', e);
               }
             });
           });
@@ -117,24 +147,22 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     });
   }
 
-  protected async applyUpdates(
-    guid: string,
-    ...updates: Buffer[]
-  ): Promise<Doc> {
+  private async applyUpdates(guid: string, ...updates: Buffer[]): Promise<Doc> {
     const doc = await this.recoverDoc(...updates);
 
     // test jwst codec
     if (
+      this.config.affine.canary &&
       this.config.doc.manager.experimentalMergeWithJwstCodec &&
       updates.length < 100 /* avoid overloading */
     ) {
-      this.metrics.jwstCodecMerge(1, {});
+      metrics.jwst.counter('codec_merge_counter').add(1);
       const yjsResult = Buffer.from(encodeStateAsUpdate(doc));
       let log = false;
       try {
         const jwstResult = jwstMergeUpdates(updates);
         if (!compare(yjsResult, jwstResult)) {
-          this.metrics.jwstCodecDidnotMatch(1, {});
+          metrics.jwst.counter('codec_not_match').add(1);
           this.logger.warn(
             `jwst codec result doesn't match yjs codec result for: ${guid}`
           );
@@ -145,11 +173,11 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
           }
         }
       } catch (e) {
-        this.metrics.jwstCodecFail(1, {});
+        metrics.jwst.counter('codec_fails_counter').add(1);
         this.logger.warn(`jwst apply update failed for ${guid}: ${e}`);
         log = true;
       } finally {
-        if (log) {
+        if (log && this.config.node.dev) {
           this.logger.warn(
             `Updates: ${updates.map(u => u.toString('hex')).join('\n')}`
           );
@@ -196,6 +224,33 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     }
   }
 
+  @OnEvent('workspace.deleted')
+  async onWorkspaceDeleted(workspaceId: string) {
+    await this.db.snapshot.deleteMany({
+      where: {
+        workspaceId,
+      },
+    });
+    await this.db.update.deleteMany({
+      where: {
+        workspaceId,
+      },
+    });
+  }
+
+  @OnEvent('snapshot.deleted')
+  async onSnapshotDeleted({
+    id,
+    workspaceId,
+  }: EventPayload<'snapshot.deleted'>) {
+    await this.db.update.deleteMany({
+      where: {
+        id,
+        workspaceId,
+      },
+    });
+  }
+
   /**
    * add update to manager for later processing.
    */
@@ -223,8 +278,8 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
         .pipe(retry(retryTimes)) // retry until seq num not conflict
         .subscribe({
           next: () => {
-            this.logger.verbose(
-              `pushed update for workspace: ${workspaceId}, guid: ${guid}`
+            this.logger.debug(
+              `pushed 1 update for ${guid} in workspace ${workspaceId}`
             );
             resolve();
           },
@@ -233,6 +288,8 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
             reject(new Error('Failed to push update'));
           },
         });
+    }).then(() => {
+      return this.updateCachedUpdatesCount(workspaceId, guid, 1);
     });
   }
 
@@ -267,8 +324,8 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
         .pipe(retry(retryTimes)) // retry until seq num not conflict
         .subscribe({
           next: () => {
-            this.logger.verbose(
-              `pushed updates for workspace: ${workspaceId}, guid: ${guid}`
+            this.logger.debug(
+              `pushed ${updates.length} updates for ${guid} in workspace ${workspaceId}`
             );
             resolve();
           },
@@ -277,6 +334,8 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
             reject(new Error('Failed to push update'));
           },
         });
+    }).then(() => {
+      return this.updateCachedUpdatesCount(workspaceId, guid, updates.length);
     });
   }
 
@@ -363,70 +422,115 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
   /**
    * apply pending updates to snapshot
    */
-  protected async autoSquash() {
+  private async autoSquash() {
     // find the first update and batch process updates with same id
-    const first = await this.db.update.findFirst({
+    const candidate = await this.getAutoSquashCandidate();
+
+    // no pending updates
+    if (!candidate) {
+      return;
+    }
+
+    const { id, workspaceId } = candidate;
+
+    await this.lockUpdatesForAutoSquash(workspaceId, id, async () => {
+      try {
+        await this._get(workspaceId, id);
+      } catch (e) {
+        this.logger.error(
+          `Failed to apply updates for workspace: ${workspaceId}, guid: ${id}`
+        );
+        this.logger.error(e);
+      }
+    });
+  }
+
+  private async getAutoSquashCandidate() {
+    const cache = await this.getAutoSquashCandidateFromCache();
+
+    if (cache) {
+      return cache;
+    }
+
+    return this.db.update.findFirst({
       select: {
         id: true,
         workspaceId: true,
       },
     });
-
-    // no pending updates
-    if (!first) {
-      return;
-    }
-
-    const { id, workspaceId } = first;
-
-    try {
-      await this._get(workspaceId, id);
-    } catch (e) {
-      this.logger.error(
-        `Failed to apply updates for workspace: ${workspaceId}, guid: ${id}`
-      );
-      this.logger.error(e);
-    }
   }
 
-  protected async upsert(
+  private async upsert(
     workspaceId: string,
     guid: string,
     doc: Doc,
-    seq?: number
+    initialSeq?: number
   ) {
-    const blob = Buffer.from(encodeStateAsUpdate(doc));
-    const state = Buffer.from(encodeStateVector(doc));
+    return this.lockSnapshotForUpsert(workspaceId, guid, async () => {
+      const blob = Buffer.from(encodeStateAsUpdate(doc));
 
-    if (isEmptyBuffer(blob)) {
-      return;
-    }
+      if (isEmptyBuffer(blob)) {
+        return false;
+      }
 
-    await this.db.snapshot.upsert({
-      select: {
-        seq: true,
-      },
-      where: {
-        id_workspaceId: {
-          id: guid,
-          workspaceId,
-        },
-      },
-      create: {
-        id: guid,
-        workspaceId,
-        blob,
-        state,
-        seq,
-      },
-      update: {
-        blob,
-        state,
-      },
+      const state = Buffer.from(encodeStateVector(doc));
+
+      return await this.db.$transaction(async db => {
+        const snapshot = await db.snapshot.findUnique({
+          where: {
+            id_workspaceId: {
+              id: guid,
+              workspaceId,
+            },
+          },
+        });
+
+        // update
+        if (snapshot) {
+          // only update if state is newer
+          if (isStateNewer(snapshot.state ?? Buffer.from([0]), state)) {
+            await db.snapshot.update({
+              select: {
+                seq: true,
+              },
+              where: {
+                id_workspaceId: {
+                  workspaceId,
+                  id: guid,
+                },
+              },
+              data: {
+                blob,
+                state,
+              },
+            });
+
+            return true;
+          } else {
+            return false;
+          }
+        } else {
+          // create
+          await db.snapshot.create({
+            select: {
+              seq: true,
+            },
+            data: {
+              id: guid,
+              workspaceId,
+              blob,
+              state,
+              seq: initialSeq,
+            },
+          });
+
+          return true;
+        }
+      });
     });
   }
 
-  protected async _get(
+  private async _get(
     workspaceId: string,
     guid: string
   ): Promise<{ doc: Doc } | { snapshot: Buffer } | null> {
@@ -446,23 +550,44 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
    * Squash updates into a single update and save it as snapshot,
    * and delete the updates records at the same time.
    */
-  protected async squash(updates: Update[], snapshot: Snapshot | null) {
+  private async squash(updates: Update[], snapshot: Snapshot | null) {
     if (!updates.length) {
       throw new Error('No updates to squash');
     }
     const first = updates[0];
     const last = updates[updates.length - 1];
 
+    const { id, workspaceId } = first;
+
     const doc = await this.applyUpdates(
       first.id,
       snapshot ? snapshot.blob : Buffer.from([0, 0]),
       ...updates.map(u => u.blob)
     );
 
-    const { id, workspaceId } = first;
+    const done = await this.upsert(workspaceId, id, doc, last.seq);
 
-    await this.upsert(workspaceId, id, doc, last.seq);
-    await this.db.update.deleteMany({
+    if (done) {
+      if (snapshot) {
+        this.event.emit('snapshot.updated', {
+          id,
+          workspaceId,
+          previous: {
+            blob: snapshot.blob,
+            state: snapshot.state,
+            updatedAt: snapshot.updatedAt,
+          },
+        });
+      }
+
+      this.logger.debug(
+        `Squashed ${updates.length} updates for ${id} in workspace ${workspaceId}`
+      );
+    }
+
+    // always delete updates
+    // the upsert will return false if the state is not newer, so we don't need to worry about it
+    const { count } = await this.db.update.deleteMany({
       where: {
         id,
         workspaceId,
@@ -471,6 +596,9 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
         },
       },
     });
+
+    await this.updateCachedUpdatesCount(workspaceId, id, -count);
+
     return doc;
   }
 
@@ -496,6 +624,9 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
       // reset
       if (seq >= MAX_SEQ_NUM) {
         await this.db.snapshot.update({
+          select: {
+            seq: true,
+          },
           where: {
             id_workspaceId: {
               workspaceId,
@@ -516,4 +647,78 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
       return last + batch;
     }
   }
+
+  private async updateCachedUpdatesCount(
+    workspaceId: string,
+    guid: string,
+    count: number
+  ) {
+    const result = await this.cache.mapIncrease(
+      `doc:manager:updates`,
+      `${workspaceId}::${guid}`,
+      count
+    );
+
+    if (result <= 0) {
+      await this.cache.mapDelete(
+        `doc:manager:updates`,
+        `${workspaceId}::${guid}`
+      );
+    }
+  }
+
+  private async getAutoSquashCandidateFromCache() {
+    const key = await this.cache.mapRandomKey('doc:manager:updates');
+
+    if (key) {
+      const count = await this.cache.mapGet<number>('doc:manager:updates', key);
+      if (typeof count === 'number' && count > 0) {
+        const [workspaceId, id] = key.split('::');
+        return { id, workspaceId };
+      }
+    }
+
+    return null;
+  }
+
+  private async doWithLock<T>(lock: string, job: () => Promise<T>) {
+    const acquired = await this.cache.setnx(lock, 1, {
+      ttl: 60 * 1000,
+    });
+
+    if (!acquired) {
+      return;
+    }
+
+    try {
+      return await job();
+    } finally {
+      await this.cache.delete(lock).catch(e => {
+        // safe, the lock will be expired when ttl ends
+        this.logger.error(`Failed to release lock ${lock}`, e);
+      });
+    }
+  }
+
+  private async lockUpdatesForAutoSquash<T>(
+    workspaceId: string,
+    guid: string,
+    job: () => Promise<T>
+  ) {
+    return this.doWithLock(
+      `doc:manager:updates-lock:${workspaceId}::${guid}`,
+      job
+    );
+  }
+
+  async lockSnapshotForUpsert<T>(
+    workspaceId: string,
+    guid: string,
+    job: () => Promise<T>
+  ) {
+    return this.doWithLock(
+      `doc:manager:snapshot-lock:${workspaceId}::${guid}`,
+      job
+    );
+  }
 }

packages/backend/server/src/modules/doc/redis-manager.ts

@@ -1,129 +0,0 @@
-import { Inject, Injectable } from '@nestjs/common';
-import Redis from 'ioredis';
-
-import { Config } from '../../config';
-import { Metrics } from '../../metrics/metrics';
-import { PrismaService } from '../../prisma';
-import { DocID } from '../../utils/doc';
-import { DocManager } from './manager';
-
-function makeKey(prefix: string) {
-  return (parts: TemplateStringsArray, ...args: any[]) => {
-    return parts.reduce((prev, curr, i) => {
-      return prev + curr + (args[i] || '');
-    }, prefix);
-  };
-}
-
-const pending = 'um_pending:';
-const updates = makeKey('um_u:');
-const lock = makeKey('um_l:');
-
-const pushUpdateLua = `
-    redis.call('sadd', KEYS[1], ARGV[1])
-    redis.call('rpush', KEYS[2], ARGV[2])
-`;
-
-/**
- * @deprecated unstable
- */
-@Injectable()
-export class RedisDocManager extends DocManager {
-  private readonly redis: Redis;
-
-  constructor(
-    protected override readonly db: PrismaService,
-    @Inject('DOC_MANAGER_AUTOMATION')
-    protected override readonly automation: boolean,
-    protected override readonly config: Config,
-    protected override readonly metrics: Metrics
-  ) {
-    super(db, automation, config, metrics);
-    this.redis = new Redis(config.redis);
-    this.redis.defineCommand('pushDocUpdate', {
-      numberOfKeys: 2,
-      lua: pushUpdateLua,
-    });
-  }
-
-  override onModuleInit(): void {
-    if (this.automation) {
-      this.setup();
-    }
-  }
-
-  override async autoSquash(): Promise<void> {
-    // incase some update fallback to db
-    await super.autoSquash();
-
-    // consume rest updates in redis queue
-    const pendingDoc = await this.redis.spop(pending).catch(() => null); // safe
-
-    if (!pendingDoc) {
-      return;
-    }
-
-    const docId = new DocID(pendingDoc);
-    const updateKey = updates`${pendingDoc}`;
-    const lockKey = lock`${pendingDoc}`;
-
-    // acquire the lock
-    const lockResult = await this.redis
-      .set(
-        lockKey,
-        '1',
-        'EX',
-        // 10mins, incase progress exit in between lock require & release, which is a rare.
-        // if the lock is really hold more then 10mins, we should check the merge logic correctness
-        600,
-        'NX'
-      )
-      .catch(() => null); // safe;
-
-    if (!lockResult) {
-      // we failed to acquire the lock, put the pending doc back to queue.
-      await this.redis.sadd(pending, pendingDoc).catch(() => null); // safe
-      return;
-    }
-
-    try {
-      // fetch pending updates
-      const updates = await this.redis
-        .lrangeBuffer(updateKey, 0, -1)
-        .catch(() => []); // safe
-
-      if (!updates.length) {
-        return;
-      }
-
-      this.logger.verbose(
-        `applying ${updates.length} updates for workspace: ${docId}`
-      );
-
-      const snapshot = await this.getSnapshot(docId.workspace, docId.guid);
-
-      // merge
-      const doc = await (snapshot
-        ? this.applyUpdates(docId.full, snapshot.blob, ...updates)
-        : this.applyUpdates(docId.full, ...updates));
-
-      // update snapshot
-      await this.upsert(docId.workspace, docId.guid, doc, snapshot?.seq);
-
-      // delete merged updates
-      await this.redis
-        .ltrim(updateKey, updates.length, -1)
-        // safe, fallback to mergeUpdates
-        .catch(e => {
-          this.logger.error(`Failed to remove merged updates from Redis: ${e}`);
-        });
-    } catch (e) {
-      this.logger.error(
-        `Failed to merge updates with snapshot for ${docId}: ${e}`
-      );
-      await this.redis.sadd(pending, docId.toString()).catch(() => null); // safe
-    } finally {
-      await this.redis.del(lockKey);
-    }
-  }
-}

packages/backend/server/src/modules/index.ts

@@ -1,31 +1,40 @@
 import { DynamicModule, Type } from '@nestjs/common';
-import { EventEmitterModule } from '@nestjs/event-emitter';
+import { ScheduleModule } from '@nestjs/schedule';
 
 import { GqlModule } from '../graphql.module';
-import { AuthModule } from './auth';
+import { SERVER_FLAVOR, ServerConfigModule } from './config';
 import { DocModule } from './doc';
 import { PaymentModule } from './payment';
+import { SelfHostedModule } from './self-hosted';
 import { SyncModule } from './sync';
 import { UsersModule } from './users';
 import { WorkspaceModule } from './workspaces';
 
-const { SERVER_FLAVOR } = process.env;
-
 const BusinessModules: (Type | DynamicModule)[] = [];
 
 switch (SERVER_FLAVOR) {
   case 'sync':
     BusinessModules.push(SyncModule, DocModule.forSync());
     break;
-  case 'graphql':
+  case 'selfhosted':
     BusinessModules.push(
-      EventEmitterModule.forRoot({
-        global: true,
-      }),
+      ServerConfigModule,
+      SelfHostedModule,
+      ScheduleModule.forRoot(),
+      GqlModule,
+      WorkspaceModule,
+      UsersModule,
+      SyncModule,
+      DocModule.forRoot()
+    );
+    break;
+  case 'graphql':
+    BusinessModules.push(
+      ServerConfigModule,
+      ScheduleModule.forRoot(),
       GqlModule,
       WorkspaceModule,
       UsersModule,
-      AuthModule,
       DocModule.forRoot(),
       PaymentModule
     );
@@ -33,13 +42,11 @@ switch (SERVER_FLAVOR) {
   case 'allinone':
   default:
     BusinessModules.push(
-      EventEmitterModule.forRoot({
-        global: true,
-      }),
+      ServerConfigModule,
+      ScheduleModule.forRoot(),
       GqlModule,
       WorkspaceModule,
       UsersModule,
-      AuthModule,
       SyncModule,
       DocModule.forRoot(),
       PaymentModule
@@ -47,4 +54,4 @@ switch (SERVER_FLAVOR) {
     break;
 }
 
-export { BusinessModules };
+export { BusinessModules, SERVER_FLAVOR };

packages/backend/server/src/modules/payment/resolver.ts

@@ -52,7 +52,7 @@ class SubscriptionPrice {
 }
 
 @ObjectType('UserSubscription')
-class UserSubscriptionType implements Partial<UserSubscription> {
+export class UserSubscriptionType implements Partial<UserSubscription> {
   @Field({ name: 'id' })
   stripeSubscriptionId!: string;
 

packages/backend/server/src/modules/payment/service.ts

@@ -30,6 +30,7 @@ export enum SubscriptionPlan {
   Pro = 'pro',
   Team = 'team',
   Enterprise = 'enterprise',
+  SelfHosted = 'selfhosted',
 }
 
 export function encodeLookupKey(

packages/backend/server/src/modules/self-hosted.ts

@@ -0,0 +1,38 @@
+import { Module } from '@nestjs/common';
+import { ResolveField, Resolver } from '@nestjs/graphql';
+
+import { UserSubscriptionType } from './payment/resolver';
+import {
+  SubscriptionPlan,
+  SubscriptionRecurring,
+  SubscriptionStatus,
+} from './payment/service';
+import { UserType } from './users';
+
+const YEAR = 1000 * 60 * 60 * 24 * 30 * 12;
+
+@Resolver(() => UserType)
+export class SelfHostedDummyResolver {
+  private readonly start = new Date();
+  private readonly end = new Date(Number(this.start) + YEAR);
+  constructor() {}
+
+  @ResolveField(() => UserSubscriptionType)
+  async subscription() {
+    return {
+      stripeSubscriptionId: 'dummy',
+      plan: SubscriptionPlan.SelfHosted,
+      recurring: SubscriptionRecurring.Yearly,
+      status: SubscriptionStatus.Active,
+      start: this.start,
+      end: this.end,
+      createdAt: this.start,
+      updatedAt: this.start,
+    };
+  }
+}
+
+@Module({
+  providers: [SelfHostedDummyResolver],
+})
+export class SelfHostedModule {}

packages/backend/server/src/modules/sync/events/events.gateway.ts

@@ -11,8 +11,8 @@ import {
 import { Server, Socket } from 'socket.io';
 import { encodeStateAsUpdate, encodeStateVector } from 'yjs';
 
-import { Metrics } from '../../../metrics/metrics';
-import { CallCounter, CallTimer } from '../../../metrics/utils';
+import { metrics } from '../../../metrics';
+import { CallTimer } from '../../../metrics/utils';
 import { DocID } from '../../../utils/doc';
 import { Auth, CurrentUser } from '../../auth';
 import { DocManager } from '../../doc';
@@ -25,13 +25,51 @@ import {
   EventError,
   InternalError,
   NotInWorkspaceError,
-  WorkspaceNotFoundError,
 } from './error';
 
+export const GatewayErrorWrapper = (): MethodDecorator => {
+  // @ts-expect-error allow
+  return (
+    _target,
+    _key,
+    desc: TypedPropertyDescriptor<(...args: any[]) => any>
+  ) => {
+    const originalMethod = desc.value;
+    if (!originalMethod) {
+      return desc;
+    }
+
+    desc.value = function (...args: any[]) {
+      let result: any;
+      try {
+        result = originalMethod.apply(this, args);
+      } catch (e) {
+        metrics.socketio.counter('unhandled_errors').add(1);
+        return {
+          error: new InternalError(e as Error),
+        };
+      }
+
+      if (result instanceof Promise) {
+        return result.catch(e => {
+          metrics.socketio.counter('unhandled_errors').add(1);
+          return {
+            error: new InternalError(e),
+          };
+        });
+      } else {
+        return result;
+      }
+    };
+
+    return desc;
+  };
+};
+
 const SubscribeMessage = (event: string) =>
   applyDecorators(
-    CallCounter('socket_io_counter', { event }),
-    CallTimer('socket_io_timer', { event }),
+    GatewayErrorWrapper(),
+    CallTimer('socketio', 'event_duration', { event }),
     RawSubscribeMessage(event)
   );
 
@@ -59,7 +97,6 @@ export class EventsGateway implements OnGatewayConnection, OnGatewayDisconnect {
 
   constructor(
     private readonly docManager: DocManager,
-    private readonly metric: Metrics,
     private readonly permissions: PermissionService
   ) {}
 
@@ -68,12 +105,12 @@ export class EventsGateway implements OnGatewayConnection, OnGatewayDisconnect {
 
   handleConnection() {
     this.connectionCount++;
-    this.metric.socketIOConnectionGauge(this.connectionCount, {});
+    metrics.socketio.gauge('realtime_connections').record(this.connectionCount);
   }
 
   handleDisconnect() {
     this.connectionCount--;
-    this.metric.socketIOConnectionGauge(this.connectionCount, {});
+    metrics.socketio.gauge('realtime_connections').record(this.connectionCount);
   }
 
   @Auth()
@@ -233,25 +270,19 @@ export class EventsGateway implements OnGatewayConnection, OnGatewayDisconnect {
       };
     }
 
-    try {
-      const docId = new DocID(guid, workspaceId);
-      client
-        .to(docId.workspace)
-        .emit('server-updates', { workspaceId, guid, updates });
+    const docId = new DocID(guid, workspaceId);
+    client
+      .to(docId.workspace)
+      .emit('server-updates', { workspaceId, guid, updates });
 
-      const buffers = updates.map(update => Buffer.from(update, 'base64'));
+    const buffers = updates.map(update => Buffer.from(update, 'base64'));
 
-      await this.docManager.batchPush(docId.workspace, docId.guid, buffers);
-      return {
-        data: {
-          accepted: true,
-        },
-      };
-    } catch (e) {
-      return {
-        error: new InternalError(e as Error),
-      };
-    }
+    await this.docManager.batchPush(docId.workspace, docId.guid, buffers);
+    return {
+      data: {
+        accepted: true,
+      },
+    };
   }
 
   @Auth()
@@ -287,9 +318,7 @@ export class EventsGateway implements OnGatewayConnection, OnGatewayDisconnect {
 
     if (!doc) {
       return {
-        error: docId.isWorkspace
-          ? new WorkspaceNotFoundError(workspaceId)
-          : new DocNotFoundError(workspaceId, docId.guid),
+        error: new DocNotFoundError(workspaceId, docId.guid),
       };
     }
 

packages/backend/server/src/modules/workspaces/controller.ts

@@ -4,35 +4,37 @@ import {
   ForbiddenException,
   Get,
   Inject,
-  Logger,
   NotFoundException,
   Param,
   Res,
 } from '@nestjs/common';
 import type { Response } from 'express';
-import format from 'pretty-time';
 
+import { CallTimer } from '../../metrics';
+import { PrismaService } from '../../prisma';
 import { StorageProvide } from '../../storage';
 import { DocID } from '../../utils/doc';
 import { Auth, CurrentUser, Publicable } from '../auth';
-import { DocManager } from '../doc';
+import { DocHistoryManager, DocManager } from '../doc';
 import { UserType } from '../users';
-import { PermissionService } from './permission';
+import { PermissionService, PublicPageMode } from './permission';
+import { Permission } from './types';
 
 @Controller('/api/workspaces')
 export class WorkspacesController {
-  private readonly logger = new Logger('WorkspacesController');
-
   constructor(
     @Inject(StorageProvide) private readonly storage: Storage,
     private readonly permission: PermissionService,
-    private readonly docManager: DocManager
+    private readonly docManager: DocManager,
+    private readonly historyManager: DocHistoryManager,
+    private readonly prisma: PrismaService
   ) {}
 
   // get workspace blob
   //
   // NOTE: because graphql can't represent a File, so we have to use REST API to get blob
   @Get('/:id/blobs/:name')
+  @CallTimer('controllers', 'workspace_get_blob')
   async blob(
     @Param('id') workspaceId: string,
     @Param('name') name: string,
@@ -57,13 +59,13 @@ export class WorkspacesController {
   @Get('/:id/docs/:guid')
   @Auth()
   @Publicable()
+  @CallTimer('controllers', 'workspace_get_doc')
   async doc(
     @CurrentUser() user: UserType | undefined,
     @Param('id') ws: string,
     @Param('guid') guid: string,
     @Res() res: Response
   ) {
-    const start = process.hrtime();
     const docId = new DocID(guid, ws);
     if (
       // if a user has the permission
@@ -82,8 +84,62 @@ export class WorkspacesController {
       throw new NotFoundException('Doc not found');
     }
 
+    if (!docId.isWorkspace) {
+      // fetch the publish page mode for publish page
+      const publishPage = await this.prisma.workspacePage.findUnique({
+        where: {
+          workspaceId_pageId: {
+            workspaceId: docId.workspace,
+            pageId: docId.guid,
+          },
+        },
+      });
+      const publishPageMode =
+        publishPage?.mode === PublicPageMode.Edgeless ? 'edgeless' : 'page';
+
+      res.setHeader('publish-mode', publishPageMode);
+    }
+
     res.setHeader('content-type', 'application/octet-stream');
     res.send(update);
-    this.logger.debug(`workspaces doc api: ${format(process.hrtime(start))}`);
+  }
+
+  @Get('/:id/docs/:guid/histories/:timestamp')
+  @Auth()
+  @CallTimer('controllers', 'workspace_get_history')
+  async history(
+    @CurrentUser() user: UserType,
+    @Param('id') ws: string,
+    @Param('guid') guid: string,
+    @Param('timestamp') timestamp: string,
+    @Res() res: Response
+  ) {
+    const docId = new DocID(guid, ws);
+    let ts;
+    try {
+      ts = new Date(timestamp);
+    } catch (e) {
+      throw new Error('Invalid timestamp');
+    }
+
+    await this.permission.checkPagePermission(
+      docId.workspace,
+      docId.guid,
+      user.id,
+      Permission.Write
+    );
+
+    const history = await this.historyManager.get(
+      docId.workspace,
+      docId.guid,
+      ts
+    );
+
+    if (history) {
+      res.setHeader('content-type', 'application/octet-stream');
+      res.send(history.blob);
+    } else {
+      throw new NotFoundException('Doc history not found');
+    }
   }
 }

packages/backend/server/src/modules/workspaces/history.resolver.ts

@@ -0,0 +1,92 @@
+import {
+  Args,
+  Field,
+  GraphQLISODateTime,
+  Int,
+  Mutation,
+  ObjectType,
+  Parent,
+  ResolveField,
+  Resolver,
+} from '@nestjs/graphql';
+import type { SnapshotHistory } from '@prisma/client';
+
+import { DocID } from '../../utils/doc';
+import { Auth, CurrentUser } from '../auth';
+import { DocHistoryManager } from '../doc/history';
+import { UserType } from '../users';
+import { PermissionService } from './permission';
+import { WorkspaceType } from './resolver';
+import { Permission } from './types';
+
+@ObjectType()
+class DocHistoryType implements Partial<SnapshotHistory> {
+  @Field()
+  workspaceId!: string;
+
+  @Field()
+  id!: string;
+
+  @Field(() => GraphQLISODateTime)
+  timestamp!: Date;
+}
+
+@Resolver(() => WorkspaceType)
+export class DocHistoryResolver {
+  constructor(
+    private readonly historyManager: DocHistoryManager,
+    private readonly permission: PermissionService
+  ) {}
+
+  @ResolveField(() => [DocHistoryType])
+  async histories(
+    @Parent() workspace: WorkspaceType,
+    @Args('guid') guid: string,
+    @Args({ name: 'before', type: () => GraphQLISODateTime, nullable: true })
+    timestamp: Date = new Date(),
+    @Args({ name: 'take', type: () => Int, nullable: true })
+    take?: number
+  ): Promise<DocHistoryType[]> {
+    const docId = new DocID(guid, workspace.id);
+
+    if (docId.isWorkspace) {
+      throw new Error('Invalid guid for listing doc histories.');
+    }
+
+    return this.historyManager
+      .list(workspace.id, docId.guid, timestamp, take)
+      .then(rows =>
+        rows.map(({ timestamp }) => {
+          return {
+            workspaceId: workspace.id,
+            id: docId.guid,
+            timestamp,
+          };
+        })
+      );
+  }
+
+  @Auth()
+  @Mutation(() => Date)
+  async recoverDoc(
+    @CurrentUser() user: UserType,
+    @Args('workspaceId') workspaceId: string,
+    @Args('guid') guid: string,
+    @Args({ name: 'timestamp', type: () => GraphQLISODateTime }) timestamp: Date
+  ): Promise<Date> {
+    const docId = new DocID(guid, workspaceId);
+
+    if (docId.isWorkspace) {
+      throw new Error('Invalid guid for recovering doc from history.');
+    }
+
+    await this.permission.checkPagePermission(
+      docId.workspace,
+      docId.guid,
+      user.id,
+      Permission.Write
+    );
+
+    return this.historyManager.recover(docId.workspace, docId.guid, timestamp);
+  }
+}

packages/backend/server/src/modules/workspaces/index.ts

@@ -3,6 +3,7 @@ import { Module } from '@nestjs/common';
 import { DocModule } from '../doc';
 import { UsersService } from '../users';
 import { WorkspacesController } from './controller';
+import { DocHistoryResolver } from './history.resolver';
 import { PermissionService } from './permission';
 import { PagePermissionResolver, WorkspaceResolver } from './resolver';
 
@@ -14,6 +15,7 @@ import { PagePermissionResolver, WorkspaceResolver } from './resolver';
     PermissionService,
     UsersService,
     PagePermissionResolver,
+    DocHistoryResolver,
   ],
   exports: [PermissionService],
 })

packages/backend/server/src/modules/workspaces/permission.ts

@@ -244,18 +244,20 @@ export class PermissionService {
     permission = Permission.Read
   ) {
     // check whether page is public
-    const count = await this.prisma.workspacePage.count({
-      where: {
-        workspaceId: ws,
-        pageId: page,
-        public: true,
-      },
-    });
+    if (permission === Permission.Read) {
+      const count = await this.prisma.workspacePage.count({
+        where: {
+          workspaceId: ws,
+          pageId: page,
+          public: true,
+        },
+      });
 
-    // page is public
-    // accessible
-    if (count > 0) {
-      return true;
+      // page is public
+      // accessible
+      if (count > 0) {
+        return true;
+      }
     }
 
     if (user) {

packages/backend/server/src/modules/workspaces/resolver.ts

@@ -33,6 +33,7 @@ import type {
 import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';
 import { applyUpdate, Doc } from 'yjs';
 
+import { EventEmitter } from '../../event';
 import { PrismaService } from '../../prisma';
 import { StorageProvide } from '../../storage';
 import { CloudThrottlerGuard, Throttle } from '../../throttler';
@@ -146,6 +147,7 @@ export class WorkspaceResolver {
     private readonly prisma: PrismaService,
     private readonly permissions: PermissionService,
     private readonly users: UsersService,
+    private readonly event: EventEmitter,
     @Inject(StorageProvide) private readonly storage: Storage
   ) {}
 
@@ -308,22 +310,11 @@ export class WorkspaceResolver {
   })
   async createWorkspace(
     @CurrentUser() user: UserType,
-    @Args({ name: 'init', type: () => GraphQLUpload })
-    update: FileUpload
+    // we no longer support init workspace with a preload file
+    // use sync system to uploading them once created
+    @Args({ name: 'init', type: () => GraphQLUpload, nullable: true })
+    init: FileUpload | null
   ) {
-    // convert stream to buffer
-    const buffer = await new Promise<Buffer>((resolve, reject) => {
-      const stream = update.createReadStream();
-      const chunks: Uint8Array[] = [];
-      stream.on('data', chunk => {
-        chunks.push(chunk);
-      });
-      stream.on('error', reject);
-      stream.on('end', () => {
-        resolve(Buffer.concat(chunks));
-      });
-    });
-
     const workspace = await this.prisma.workspace.create({
       data: {
         public: false,
@@ -341,14 +332,31 @@ export class WorkspaceResolver {
       },
     });
 
-    if (buffer.length) {
-      await this.prisma.snapshot.create({
-        data: {
-          id: workspace.id,
-          workspaceId: workspace.id,
-          blob: buffer,
-        },
+    if (init) {
+      // convert stream to buffer
+      const buffer = await new Promise<Buffer>(resolve => {
+        const stream = init.createReadStream();
+        const chunks: Uint8Array[] = [];
+        stream.on('data', chunk => {
+          chunks.push(chunk);
+        });
+        stream.on('error', () => {
+          resolve(Buffer.from([]));
+        });
+        stream.on('end', () => {
+          resolve(Buffer.concat(chunks));
+        });
       });
+
+      if (buffer.length) {
+        await this.prisma.snapshot.create({
+          data: {
+            id: workspace.id,
+            workspaceId: workspace.id,
+            blob: buffer,
+          },
+        });
+      }
     }
 
     return workspace;
@@ -382,18 +390,7 @@ export class WorkspaceResolver {
       },
     });
 
-    await this.prisma.$transaction([
-      this.prisma.update.deleteMany({
-        where: {
-          workspaceId: id,
-        },
-      }),
-      this.prisma.snapshot.deleteMany({
-        where: {
-          workspaceId: id,
-        },
-      }),
-    ]);
+    this.event.emit('workspace.deleted', id);
 
     return true;
   }
@@ -885,9 +882,9 @@ export class PagePermissionResolver {
     }
 
     await this.permission.checkWorkspace(
-      workspaceId,
+      docId.workspace,
       user.id,
-      Permission.Admin
+      Permission.Read
     );
 
     return this.permission.publishPage(docId.workspace, docId.guid, mode);
@@ -924,7 +921,7 @@ export class PagePermissionResolver {
     await this.permission.checkWorkspace(
       docId.workspace,
       user.id,
-      Permission.Admin
+      Permission.Read
     );
 
     return this.permission.revokePublicPage(docId.workspace, docId.guid);