fix(workspace): avoid data loss (hot-fix) (#6114)

enhancement

___

- Introduced a new ESM module resolution setup using `ts-node` to enhance the development and deployment process.
- Implemented a dynamic loader script registration mechanism to facilitate ESM module loading.
- Simplified the predeploy script execution by refining environment variable handling and stdout configuration.
- Updated `package.json` to reflect changes in script commands for better ESM support and added necessary dependencies for `ts-node` and `typescript`.

___

<table><thead><tr><th></th><th align="left">Relevant files</th></tr></thead><tbody><tr><td><strong>Enhancement</strong></td><td><table>
<tr>
  <td>
    <details>
      <summary><strong>loader.js</strong><dd><code>Introduce ESM Module Resolution via ts-node</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary>
<hr>

packages/backend/server/scripts/loader.js

<li>Introduced <code>ts-node</code> configuration for ESM module resolution.<br> <li> Exported a <code>resolve</code> function for module resolution.<br>

</details>

  </td>
  <td><a href="https:/toeverything/AFFiNE/pull/5920/files#diff-9ed793897a493633028d510db0742ff38d2d86471c54b17513d4354c51597ef8">+11/-0</a>&nbsp; &nbsp; </td>

</tr>

<tr>
  <td>
    <details>
      <summary><strong>register.js</strong><dd><code>Implement Dynamic Loader Script Registration</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary>
<hr>

packages/backend/server/scripts/register.js

<li>Implemented dynamic registration of the loader script.<br> <li> Utilized <code>node:module</code> and <code>node:url</code> for script registration.<br>

</details>

  </td>
  <td><a href="https:/toeverything/AFFiNE/pull/5920/files#diff-64831012a09f2bc4bc5a611ddb8e0871b0e83588de6c5d4f2f5cb1dae8fff244">+4/-0</a>&nbsp; &nbsp; &nbsp; </td>

</tr>

<tr>
  <td>
    <details>
      <summary><strong>self-host-predeploy.js</strong><dd><code>Simplify Predeploy Script Execution</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary>
<hr>

packages/backend/server/scripts/self-host-predeploy.js

<li>Simplified environment variable passing to <code>execSync</code>.<br> <li> Changed stdout handling to inherit from the parent process.<br>

</details>

  </td>
  <td><a href="https:/toeverything/AFFiNE/pull/5920/files#diff-bd7b0be14c198018c21dadda6945a779c57d13e4c8584ee62da4baa99d370664">+3/-5</a>&nbsp; &nbsp; &nbsp; </td>

</tr>

<tr>
  <td>
    <details>
      <summary><strong>package.json</strong><dd><code>Update Scripts and Dependencies for ESM Support</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary>
<hr>

packages/backend/server/package.json

<li>Updated script commands for ESM compatibility.<br> <li> Added <code>ts-node</code> and <code>typescript</code> dependencies.<br> <li> Removed redundant <code>--es-module-specifier-resolution=node</code> flags.<br>

</details>

  </td>
  <td><a href="https:/toeverything/AFFiNE/pull/5920/files#diff-a6530c6fe539aaa49ff0a7a80bc4362c1d95c419fdd19125415dcc869b31a443">+6/-6</a>&nbsp; &nbsp; &nbsp; </td>

</tr>
</table></td></tr></tr></tbody></table>

___

> ✨ **PR-Agent usage**:
>Comment `/help` on the PR to get a list of all available PR-Agent tools and their descriptions

.eslintignore

@@ -12,3 +12,4 @@ static
 web-static
 public
 packages/frontend/i18n/src/i18n-generated.ts
+packages/frontend/templates/edgeless-templates.gen.ts

.eslintrc.js

@@ -217,6 +217,7 @@ const config = {
     'unicorn/no-useless-promise-resolve-reject': 'error',
     'unicorn/no-new-array': 'error',
     'unicorn/new-for-builtins': 'error',
+    'unicorn/prefer-node-protocol': 'error',
     'sonarjs/no-all-duplicated-branches': 'error',
     'sonarjs/no-element-overwrite': 'error',
     'sonarjs/no-empty-collection': 'error',

.github/actions/build-rust/action.yml

@@ -37,7 +37,7 @@ runs:
         echo "TARGET_CC=clang" >> "$GITHUB_ENV"
 
     - name: Cache cargo
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: |
           ~/.cargo/registry/index/

.github/actions/deploy/deploy.mjs

@@ -15,12 +15,13 @@ const {
   R2_SECRET_ACCESS_KEY,
   ENABLE_CAPTCHA,
   CAPTCHA_TURNSTILE_SECRET,
-  OAUTH_EMAIL_SENDER,
-  OAUTH_EMAIL_LOGIN,
-  OAUTH_EMAIL_PASSWORD,
+  MAILER_SENDER,
+  MAILER_USER,
+  MAILER_PASSWORD,
   AFFINE_GOOGLE_CLIENT_ID,
   AFFINE_GOOGLE_CLIENT_SECRET,
   CLOUD_SQL_IAM_ACCOUNT,
+  CLOUD_LOGGER_IAM_ACCOUNT,
   GCLOUD_CONNECTION_NAME,
   GCLOUD_CLOUD_SQL_INTERNAL_ENDPOINT,
   REDIS_HOST,
@@ -59,14 +60,24 @@ const createHelmCommand = ({ isDryRun }) => {
       ? [
           `--set-json   web.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`,
           `--set-json   graphql.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`,
+          `--set-json   graphql.serviceAccount.annotations=\"{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_LOGGER_IAM_ACCOUNT}\\"}\"`,
           `--set-json   sync.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`,
+          `--set-json   sync.serviceAccount.annotations=\"{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_LOGGER_IAM_ACCOUNT}\\"}\"`,
           `--set-json   cloud-sql-proxy.serviceAccount.annotations=\"{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_SQL_IAM_ACCOUNT}\\" }\"`,
           `--set-json   cloud-sql-proxy.nodeSelector=\"{ \\"iam.gke.io/gke-metadata-server-enabled\\": \\"true\\" }\"`,
         ]
       : [];
   const webReplicaCount = isProduction ? 3 : isBeta ? 2 : 2;
-  const graphqlReplicaCount = isProduction ? 10 : isBeta ? 5 : 2;
-  const syncReplicaCount = isProduction ? 10 : isBeta ? 5 : 2;
+  const graphqlReplicaCount = isProduction
+    ? Number(process.env.PRODUCTION_GRAPHQL_REPLICA) || 3
+    : isBeta
+      ? Number(process.env.isBeta_GRAPHQL_REPLICA) || 2
+      : 2;
+  const syncReplicaCount = isProduction
+    ? Number(process.env.PRODUCTION_SYNC_REPLICA) || 3
+    : isBeta
+      ? Number(process.env.BETA_SYNC_REPLICA) || 2
+      : 2;
   const namespace = isProduction
     ? 'production'
     : isBeta
@@ -95,15 +106,15 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set-string graphql.app.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`,
     `--set-string graphql.app.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`,
     `--set-string graphql.app.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`,
-    `--set-string graphql.app.oauth.email.sender="${OAUTH_EMAIL_SENDER}"`,
-    `--set-string graphql.app.oauth.email.login="${OAUTH_EMAIL_LOGIN}"`,
-    `--set-string graphql.app.oauth.email.password="${OAUTH_EMAIL_PASSWORD}"`,
+    `--set-string graphql.app.mailer.sender="${MAILER_SENDER}"`,
+    `--set-string graphql.app.mailer.user="${MAILER_USER}"`,
+    `--set-string graphql.app.mailer.password="${MAILER_PASSWORD}"`,
     `--set-string graphql.app.oauth.google.enabled=true`,
     `--set-string graphql.app.oauth.google.clientId="${AFFINE_GOOGLE_CLIENT_ID}"`,
     `--set-string graphql.app.oauth.google.clientSecret="${AFFINE_GOOGLE_CLIENT_SECRET}"`,
     `--set-string graphql.app.payment.stripe.apiKey="${STRIPE_API_KEY}"`,
     `--set-string graphql.app.payment.stripe.webhookKey="${STRIPE_WEBHOOK_KEY}"`,
-    `--set        graphql.app.experimental.enableJwstCodec=true`,
+    `--set        graphql.app.experimental.enableJwstCodec=${isInternal}`,
     `--set        graphql.app.features.earlyAccessPreview=false`,
     `--set        sync.replicaCount=${syncReplicaCount}`,
     `--set-string sync.image.tag="${imageTag}"`,

.github/actions/setup-node/action.yml

@@ -63,7 +63,7 @@ runs:
       run: node -e "const p = $(yarn config cacheFolder --json).effective; console.log('yarn_global_cache=' + p)" >> $GITHUB_OUTPUT
 
     - name: Cache non-full yarn cache on Linux
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       if: ${{ inputs.full-cache != 'true' && runner.os == 'Linux' }}
       with:
         path: |
@@ -75,7 +75,7 @@ runs:
     # and the decompression performance on Windows is very terrible
     # so we reduce the number of cached files on non-Linux systems by remove node_modules from cache path.
     - name: Cache non-full yarn cache on non-Linux
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       if: ${{ inputs.full-cache != 'true' && runner.os != 'Linux' }}
       with:
         path: |
@@ -83,7 +83,7 @@ runs:
         key: node_modules-cache-${{ github.job }}-${{ runner.os }}
 
     - name: Cache full yarn cache on Linux
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       if: ${{ inputs.full-cache == 'true' && runner.os == 'Linux' }}
       with:
         path: |
@@ -92,7 +92,7 @@ runs:
         key: node_modules-cache-full-${{ runner.os }}
 
     - name: Cache full yarn cache on non-Linux
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       if: ${{ inputs.full-cache == 'true' && runner.os != 'Linux' }}
       with:
         path: |
@@ -134,7 +134,7 @@ runs:
       # Note: Playwright's cache directory is hard coded because that's what it
       # says to do in the docs. There doesn't appear to be a command that prints
       # it out for us.
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       id: playwright-cache
       if: ${{ inputs.playwright-install == 'true' }}
       with:
@@ -167,7 +167,7 @@ runs:
       run: |
         echo "version=$(yarn why --json electron | grep -h 'workspace:.' | jq --raw-output '.children[].locator' | sed -e 's/@playwright\/test@.*://' | head -n 1)" >> $GITHUB_OUTPUT
 
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       id: electron-cache
       if: ${{ inputs.electron-install == 'true' }}
       with:

.github/deployment/node/Dockerfile

@@ -1,10 +1,11 @@
 FROM node:18-bookworm-slim
 
 COPY ./packages/backend/server /app
+COPY ./packages/frontend/core/dist /app/static
 WORKDIR /app
 
 RUN apt-get update && \
   apt-get install -y --no-install-recommends openssl && \
   rm -rf /var/lib/apt/lists/*
 
-CMD ["node", "--es-module-specifier-resolution=node", "./dist/index.js"]
+CMD ["node", "--import", "./scripts/register.js", "./dist/index.js"]

.github/deployment/self-host/compose.yaml

@@ -0,0 +1,59 @@
+services:
+  affine:
+    image: ghcr.io/toeverything/affine-graphql:stable
+    container_name: affine_selfhosted
+    command:
+      ['sh', '-c', 'node ./scripts/self-host-predeploy && node ./dist/index.js']
+    ports:
+      - '3010:3010'
+      - '5555:5555'
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+    volumes:
+      # custom configurations
+      - ~/.affine/self-host/config:/root/.affine/config
+      # blob storage
+      - ~/.affine/self-host/storage:/root/.affine/storage
+    logging:
+      driver: 'json-file'
+      options:
+        max-size: '1000m'
+    restart: unless-stopped
+    environment:
+      - NODE_OPTIONS="--import=./scripts/register.js"
+      - AFFINE_CONFIG_PATH=/root/.affine/config
+      - REDIS_SERVER_HOST=redis
+      - DATABASE_URL=postgres://affine:affine@postgres:5432/affine
+      - NODE_ENV=production
+      - AFFINE_ADMIN_EMAIL=${AFFINE_ADMIN_EMAIL}
+      - AFFINE_ADMIN_PASSWORD=${AFFINE_ADMIN_PASSWORD}
+  redis:
+    image: redis
+    container_name: affine_redis
+    restart: unless-stopped
+    volumes:
+      - ~/.affine/self-host/redis:/data
+    healthcheck:
+      test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  postgres:
+    image: postgres
+    container_name: affine_postgres
+    restart: unless-stopped
+    volumes:
+      - ~/.affine/self-host/postgres:/var/lib/postgresql/data
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U affine']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    environment:
+      POSTGRES_USER: affine
+      POSTGRES_PASSWORD: affine
+      POSTGRES_DB: affine
+      PGDATA: /var/lib/postgresql/data/pgdata

.github/helm/affine/Chart.yaml

@@ -3,4 +3,4 @@ name: affine
 description: AFFiNE cloud chart
 type: application
 version: 0.0.0
-appVersion: "0.11.0"
+appVersion: "0.12.0"

.github/helm/affine/charts/graphql/Chart.yaml

@@ -3,7 +3,7 @@ name: graphql
 description: AFFiNE GraphQL server
 type: application
 version: 0.0.0
-appVersion: "0.11.0"
+appVersion: "0.12.0"
 dependencies:
   - name: gcloud-sql-proxy
     version: 0.0.0

.github/helm/affine/charts/graphql/templates/deployment.yaml

@@ -39,6 +39,8 @@ spec:
             value: "--max-old-space-size=4096"
           - name: NO_COLOR
             value: "1"
+          - name: DEPLOYMENT_TYPE
+            value: "affine"
           - name: SERVER_FLAVOR
             value: "graphql"
           - name: AFFINE_ENV
@@ -73,37 +75,41 @@ spec:
             value: "{{ .Values.app.path }}"
           - name: AFFINE_SERVER_HOST
             value: "{{ .Values.app.host }}"
+          - name: AFFINE_SERVER_HTTPS
+            value: "{{ .Values.app.https }}"
           - name: ENABLE_R2_OBJECT_STORAGE
             value: "{{ .Values.app.objectStorage.r2.enabled }}"
           - name: ENABLE_CAPTCHA
             value: "{{ .Values.app.captcha.enabled }}"
           - name: FEATURES_EARLY_ACCESS_PREVIEW
             value: "{{ .Values.app.features.earlyAccessPreview }}"
-          - name: OAUTH_EMAIL_SENDER
+          - name: FEATURES_SYNC_CLIENT_VERSION_CHECK
+            value: "{{ .Values.app.features.syncClientVersionCheck }}"
+          - name: MAILER_HOST
             valueFrom:
               secretKeyRef:
-                name: "{{ .Values.app.oauth.email.secretName }}"
-                key: sender
-          - name: OAUTH_EMAIL_LOGIN
+                name: "{{ .Values.app.mailer.secretName }}"
+                key: host
+          - name: MAILER_PORT
             valueFrom:
               secretKeyRef:
-                name: "{{ .Values.app.oauth.email.secretName }}"
-                key: login
-          - name: OAUTH_EMAIL_SERVER
-            valueFrom:
-              secretKeyRef:
-                name: "{{ .Values.app.oauth.email.secretName }}"
-                key: server
-          - name: OAUTH_EMAIL_PORT
-            valueFrom:
-              secretKeyRef:
-                name: "{{ .Values.app.oauth.email.secretName }}"
+                name: "{{ .Values.app.mailer.secretName }}"
                 key: port
-          - name: OAUTH_EMAIL_PASSWORD
+          - name: MAILER_USER
             valueFrom:
               secretKeyRef:
-                name: "{{ .Values.app.oauth.email.secretName }}"
+                name: "{{ .Values.app.mailer.secretName }}"
+                key: user
+          - name: MAILER_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.app.mailer.secretName }}"
                 key: password
+          - name: MAILER_SENDER
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.app.mailer.secretName }}"
+                key: sender
           - name: STRIPE_API_KEY
             valueFrom:
               secretKeyRef:
@@ -145,6 +151,8 @@ spec:
                 key: turnstileSecret
           {{ end }}
           {{ if .Values.app.oauth.google.enabled }}
+          - name: OAUTH_GOOGLE_ENABLED
+            value: "true"
           - name: OAUTH_GOOGLE_CLIENT_ID
             valueFrom:
               secretKeyRef:

.github/helm/affine/charts/graphql/templates/mailer.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.app.mailer.secretName -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "{{ .Values.app.mailer.secretName }}"
+type: Opaque
+data:
+  host: "{{ .Values.app.mailer.host | b64enc }}"
+  port: "{{ .Values.app.mailer.port | b64enc }}"
+  user: "{{ .Values.app.mailer.user | b64enc }}"
+  password: "{{ .Values.app.mailer.password | b64enc }}"
+  sender: "{{ .Values.app.mailer.sender | b64enc }}"
+{{- end }}

.github/helm/affine/charts/graphql/templates/oauth.yaml

@@ -1,15 +1,3 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "{{ .Values.app.oauth.email.secretName }}"
-type: Opaque
-data:
-  sender: "{{ .Values.app.oauth.email.sender | b64enc }}"
-  login: "{{ .Values.app.oauth.email.login | b64enc }}"
-  password: "{{ .Values.app.oauth.email.password | b64enc }}"
-  server: "{{ .Values.app.oauth.email.server | b64enc }}"
-  port: "{{ .Values.app.oauth.email.port | b64enc }}"
----
 {{- if .Values.app.oauth.google.enabled -}}
 apiVersion: v1
 kind: Secret

.github/helm/affine/charts/graphql/values.yaml

@@ -16,6 +16,7 @@ app:
   path: ''
   # AFFINE_SERVER_HOST
   host: '0.0.0.0'
+  https: true
   doc:
     mergeInterval: "3000"
   jwt:
@@ -34,14 +35,7 @@ app:
       accountId: ''
       accessKeyId: ''
       secretAccessKey: ''
-  oauth:
-    email:
-      secretName: 'oauth-email'
-      sender: 'noreply@toeverything.info'
-      login: ''
-      password: ''
-      server: 'smtp.gmail.com'
-      port: '465'
+  oauth: 
     google:
       enabled: false
       secretName: oauth-google
@@ -52,6 +46,13 @@ app:
       secretName: oauth-github
       clientId: ''
       clientSecret: ''
+  mailer:
+    secretName: 'mailer'
+    host: 'smtp.gmail.com'
+    port: '465'
+    user: ''
+    password: ''
+    sender: 'noreply@toeverything.info'
   payment:
     stripe:
       secretName: 'stripe'

.github/helm/affine/charts/sync/Chart.yaml

@@ -3,7 +3,7 @@ name: sync
 description: AFFiNE Sync Server
 type: application
 version: 0.0.0
-appVersion: "0.11.0"
+appVersion: "0.12.0"
 dependencies:
   - name: gcloud-sql-proxy
     version: 0.0.0

.github/helm/affine/charts/sync/templates/deployment.yaml

@@ -36,6 +36,8 @@ spec:
             value: "{{ .Values.env }}"
           - name: NO_COLOR
             value: "1"
+          - name: DEPLOYMENT_TYPE
+            value: "affine"
           - name: SERVER_FLAVOR
             value: "sync"
           - name: NEXTAUTH_URL

.github/renovate.json

@@ -47,11 +47,11 @@
       "groupName": "electron-forge"
     },
     {
-      "groupName": "blocksuite-nightly",
+      "groupName": "blocksuite-canary",
       "matchPackagePatterns": ["^@blocksuite"],
       "excludePackageNames": ["@blocksuite/icons"],
       "rangeStrategy": "replace",
-      "followTag": "nightly"
+      "followTag": "canary"
     },
     {
       "groupName": "all non-major dependencies",
@@ -70,6 +70,7 @@
   "commitMessageAction": "bump up",
   "commitMessageTopic": "{{depName}} version",
   "ignoreDeps": [],
+  "postUpdateOptions": ["yarnDedupeHighest"],
   "lockFileMaintenance": {
     "enabled": true,
     "extends": ["schedule:weekly"]

.github/workflows/build-test.yml

@@ -19,6 +19,7 @@ env:
   MACOSX_DEPLOYMENT_TARGET: '10.13'
   NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
   PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/node_modules/.cache/ms-playwright
+  DEPLOYMENT_TYPE: affine
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -95,6 +96,8 @@ jobs:
         run: |
           git checkout .yarnrc.yml
           yarn lint:prettier
+      - name: Yarn Dedupe
+        run: yarn dedupe --check
       - name: Run Type Check
         run: yarn typecheck
 
@@ -288,6 +291,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-storage
     env:
+      NODE_ENV: test
       DISTRIBUTION: browser
     services:
       postgres:
@@ -444,7 +448,6 @@ jobs:
           ${{ matrix.tests.script }}
         env:
           DEV_SERVER_URL: http://localhost:8080
-          ENABLE_LOCAL_EMAIL: true
 
       - name: Upload test results
         if: ${{ failure() }}

.github/workflows/deploy.yml

@@ -29,6 +29,7 @@ jobs:
         uses: ./.github/actions/setup-node
         with:
           electron-install: false
+          extra-flags: workspaces focus @affine/server
       - name: Build Server
         run: yarn workspace @affine/server build
       - name: Upload server dist
@@ -62,6 +63,7 @@ jobs:
           SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+          PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
       - name: Upload core artifact
         uses: actions/upload-artifact@v4
         with:
@@ -69,10 +71,10 @@ jobs:
           path: ./packages/frontend/core/dist
           if-no-files-found: error
 
-  build-storage:
-    name: Build Storage
+  build-core-selfhost:
+    name: Build @affine/core
     runs-on: ubuntu-latest
-
+    environment: ${{ github.event.inputs.flavor }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Version
@@ -80,22 +82,33 @@ jobs:
         uses: ./.github/actions/setup-version
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
-      - name: Build Rust
-        uses: ./.github/actions/build-rust
-        with:
-          target: 'x86_64-unknown-linux-gnu'
-          package: '@affine/storage'
-          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-      - name: Upload storage.node
+      - name: Build Core
+        run: yarn nx build @affine/core --skip-nx-cache
+        env:
+          BUILD_TYPE: ${{ github.event.inputs.flavor }}
+          SHOULD_REPORT_TRACE: false
+          PUBLIC_PATH: '/'
+      - name: Download selfhost fonts
+        run: node ./scripts/download-blocksuite-fonts.mjs
+      - name: Upload core artifact
         uses: actions/upload-artifact@v4
         with:
-          name: storage.node
-          path: ./packages/backend/storage/storage.node
+          name: selfhost-core
+          path: ./packages/frontend/core/dist
           if-no-files-found: error
 
-  build-storage-arm64:
-    name: Build Storage arm64
+  build-storage:
+    name: Build Storage - ${{ matrix.targets.name }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        targets:
+          - name: x86_64-unknown-linux-gnu
+            file: storage.node
+          - name: aarch64-unknown-linux-gnu
+            file: storage.arm64.node
+          - name: armv7-unknown-linux-gnueabihf
+            file: storage.armv7.node
 
     steps:
       - uses: actions/checkout@v4
@@ -104,27 +117,34 @@ jobs:
         uses: ./.github/actions/setup-version
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
+        with:
+          electron-install: false
+          extra-flags: workspaces focus @affine/storage
       - name: Build Rust
         uses: ./.github/actions/build-rust
         with:
-          target: 'aarch64-unknown-linux-gnu'
+          target: ${{ matrix.targets.name }}
           package: '@affine/storage'
           nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-      - name: Upload storage.node
+      - name: Upload ${{ matrix.targets.file }}
         uses: actions/upload-artifact@v4
         with:
-          name: storage.arm64.node
+          name: ${{ matrix.targets.file }}
           path: ./packages/backend/storage/storage.node
           if-no-files-found: error
 
   build-docker:
     name: Build Docker
     runs-on: ubuntu-latest
+    permissions:
+      contents: 'write'
+      id-token: 'write'
+      packages: 'write'
     needs:
       - build-server
       - build-core
+      - build-core-selfhost
       - build-storage
-      - build-storage-arm64
     steps:
       - uses: actions/checkout@v4
       - name: Download core artifact
@@ -147,8 +167,15 @@ jobs:
         with:
           name: storage.arm64.node
           path: ./packages/backend/storage
-      - name: move storage.arm64.node
-        run: mv ./packages/backend/storage/storage.node ./packages/backend/server/storage.arm64.node
+      - name: Download storage.node arm64
+        uses: actions/download-artifact@v4
+        with:
+          name: storage.armv7.node
+          path: .
+      - name: move storage files
+        run: |
+          mv ./packages/backend/storage/storage.node ./packages/backend/server/storage.arm64.node
+          mv storage.node ./packages/backend/server/storage.armv7.node
       - name: Setup env
         run: |
           echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
@@ -190,9 +217,19 @@ jobs:
           registry-url: https://npm.pkg.github.com
           scope: '@toeverything'
 
+      - name: Remove core dist
+        run: rm -rf ./packages/frontend/core/dist
+
+      - name: Download selfhost core artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: selfhost-core
+          path: ./packages/frontend/core/dist
+
       - name: Install Node.js dependencies
         run: |
-          yarn config set --json supportedArchitectures.cpu '["x64", "arm64"]'
+          yarn config set --json supportedArchitectures.cpu '["x64", "arm64", "arm"]'
+          yarn config set --json supportedArchitectures.libc '["glibc"]'
           yarn workspaces focus @affine/server --production
 
       - name: Generate Prisma client
@@ -204,7 +241,7 @@ jobs:
           context: .
           push: true
           pull: true
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
           provenance: true
           file: .github/deployment/node/Dockerfile
           tags: ghcr.io/toeverything/affine-graphql:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-graphql:${{env.RELEASE_FLAVOR}}
@@ -242,9 +279,9 @@ jobs:
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           ENABLE_CAPTCHA: true
           CAPTCHA_TURNSTILE_SECRET: ${{ secrets.CAPTCHA_TURNSTILE_SECRET }}
-          OAUTH_EMAIL_SENDER: ${{ secrets.OAUTH_EMAIL_SENDER }}
-          OAUTH_EMAIL_LOGIN: ${{ secrets.OAUTH_EMAIL_LOGIN }}
-          OAUTH_EMAIL_PASSWORD: ${{ secrets.OAUTH_EMAIL_PASSWORD }}
+          MAILER_SENDER: ${{ secrets.OAUTH_EMAIL_SENDER }}
+          MAILER_USER: ${{ secrets.OAUTH_EMAIL_LOGIN }}
+          MAILER_PASSWORD: ${{ secrets.OAUTH_EMAIL_PASSWORD }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           AFFINE_GOOGLE_CLIENT_ID: ${{ secrets.AFFINE_GOOGLE_CLIENT_ID }}
           AFFINE_GOOGLE_CLIENT_SECRET: ${{ secrets.AFFINE_GOOGLE_CLIENT_SECRET }}
@@ -257,6 +294,7 @@ jobs:
           REDIS_HOST: ${{ secrets.REDIS_HOST }}
           REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
           CLOUD_SQL_IAM_ACCOUNT: ${{ secrets.CLOUD_SQL_IAM_ACCOUNT }}
+          CLOUD_LOGGER_IAM_ACCOUNT: ${{ secrets.CLOUD_LOGGER_IAM_ACCOUNT }}
           STRIPE_API_KEY: ${{ secrets.STRIPE_API_KEY }}
           STRIPE_WEBHOOK_KEY: ${{ secrets.STRIPE_WEBHOOK_KEY }}
           STATIC_IP_NAME: ${{ secrets.STATIC_IP_NAME }}

.github/workflows/publish-ui-storybook.yml

@@ -0,0 +1,51 @@
+name: Publish UI Storybook
+
+env:
+  NODE_OPTIONS: --max-old-space-size=4096
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - canary
+  pull_request:
+    branches:
+      - canary
+    paths-ignore:
+      - README.md
+      - .github/**
+      - packages/backend/server
+      - packages/frontend/electron
+      - '!.github/workflows/publish-storybook.yml'
+
+jobs:
+  publish-ui-storybook:
+    name: Publish UI Storybook
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
+          # This is required to fetch all commits for chromatic
+          fetch-depth: 0
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          electron-install: false
+      - uses: chromaui/action-next@v1
+        with:
+          workingDir: packages/frontend/component
+          buildScriptName: build:storybook
+          exitOnceUploaded: true
+          onlyChanged: false
+          diagnostics: true
+        env:
+          CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_UI_PROJECT_TOKEN }}
+          NODE_OPTIONS: ${{ env.NODE_OPTIONS }}
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: chromatic-build-artifacts-${{ github.run_id }}
+          path: |
+            chromatic-diagnostics.json
+            **/build-storybook.log

.github/workflows/release-desktop.yml

@@ -143,7 +143,7 @@ jobs:
         run: |
           mkdir -p builds
           mv packages/frontend/electron/out/*/make/zip/linux/x64/*.zip ./builds/affine-${{ env.BUILD_TYPE }}-linux-x64.zip
-          mv packages/frontend/electron/out/*/make/AppImage/x64/*.AppImage ./builds/affine-${{ env.BUILD_TYPE }}-linux-x64.AppImage
+          mv packages/frontend/electron/out/*/make/*.AppImage ./builds/affine-${{ env.BUILD_TYPE }}-linux-x64.AppImage
 
       - name: Upload Artifact
         uses: actions/upload-artifact@v4

.gitignore

@@ -79,3 +79,6 @@ lib
 affine.db
 apps/web/next-routes.conf
 .nx
+
+packages/frontend/templates/edgeless
+packages/frontend/core/public/static/templates

.husky/pre-commit

@@ -1,4 +1 @@
-#!/usr/bin/env sh
-. "$(dirname -- "$0")/_/husky.sh"
-
 yarn lint-staged && yarn lint:ox

.prettierignore

@@ -16,6 +16,7 @@ packages/frontend/i18n/src/i18n-generated.ts
 packages/frontend/graphql/src/graphql/index.ts
 tests/affine-legacy/**/static
 .yarnrc.yml
+packages/frontend/templates/edgeless-templates.gen.ts
 packages/frontend/templates/templates.gen.ts
 packages/frontend/templates/onboarding
 

README.md

@@ -138,24 +138,6 @@ We would like to express our gratitude to all the individuals who have already c
   <img alt="contributors" src="https://opencollective.com/affine/contributors.svg?width=890&button=false" />
 </a>
 
-## Data Compatibility
-
-Data compatibility is a very important issue for us. We will try our best to ensure that the data is compatible with the previous version.
-
-If you encounter any problems when upgrading the version, please feel free to [contact us](mailto:developer@toeverything.info).
-
-| AFFiNE Version  | Export/Import workspace | Data auto migration |
-| --------------- | ----------------------- | ------------------- |
-| <= 0.5.4        | ❌️                     | ❌                  |
-| 0.6.x           | ✅️                     | ✅                  |
-| 0.7.x           | ✅️                     | ✅                  |
-| 0.8.x (current) | ✅                      | ✅                  |
-| 0.9.x (next)    | 🚧                      | 🚧                  |
-
-- ❌️: Not compatible
-- ✅: Compatible
-- 🚧: Work in progress
-
 ## Self-Host
 
 > We know that the self-host version has been out of date for a long time.

docs/developing-server.md

@@ -59,9 +59,9 @@ You may need additional env for auth login. You may want to put your own one if
 For email login & password, please refer to https://nodemailer.com/usage/using-gmail/
 
 ```
-OAUTH_EMAIL_SENDER=
-OAUTH_EMAIL_LOGIN=
-OAUTH_EMAIL_PASSWORD=
+MAILER_SENDER=
+MAILER_USER=
+MAILER_PASSWORD=
 OAUTH_GOOGLE_ENABLED="true"
 OAUTH_GOOGLE_CLIENT_ID=
 OAUTH_GOOGLE_CLIENT_SECRET=

docs/reference/package.json

@@ -19,5 +19,5 @@
     ],
     "ext": "ts,md,json"
   },
-  "version": "0.10.3-canary.2"
+  "version": "0.12.0"
 }

nx.json

@@ -1,12 +1,13 @@
 {
   "$schema": "./node_modules/nx/schemas/nx-schema.json",
   "npmScope": "toeverything",
+  "nxCloudAccessToken": "MzUwNTU4YWItZGFhYi00YjE2LWIxODAtODk4NmIwYjMwYzZkfHJlYWQ=",
   "tasksRunnerOptions": {
     "default": {
       "runner": "nx-cloud",
       "options": {
         "cacheableOperations": ["build", "test", "e2e", "lint"],
-        "accessToken": "YmQ2NTg1ODktZTk5Mi00YzhiLTk2ZmUtNWQzMDg0NDBkOWM3fHJlYWQtb25seQ=="
+        "runtimeCacheInputs": ["node -v"]
       }
     }
   },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@affine/monorepo",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "private": true,
   "author": "toeverything",
   "license": "MIT",
@@ -37,7 +37,8 @@
     "test:ui": "vitest --ui",
     "test:coverage": "vitest run --coverage",
     "typecheck": "tsc -b tsconfig.json --diagnostics",
-    "postinstall": "node ./scripts/check-version.mjs && yarn i18n-codegen gen && yarn husky install"
+    "postinstall": "node ./scripts/check-version.mjs && yarn i18n-codegen gen && yarn husky install",
+    "prepare": "husky"
   },
   "lint-staged": {
     "*": "prettier --write --ignore-unknown --cache",
@@ -61,8 +62,7 @@
     "@istanbuljs/schema": "^0.1.3",
     "@magic-works/i18n-codegen": "^0.5.0",
     "@nx/vite": "17.2.8",
-    "@perfsee/sdk": "^1.9.0",
-    "@playwright/test": "^1.40.0",
+    "@playwright/test": "^1.41.0",
     "@taplo/cli": "^0.5.2",
     "@testing-library/react": "^14.1.2",
     "@toeverything/infra": "workspace:*",
@@ -76,7 +76,7 @@
     "@vitejs/plugin-react-swc": "^3.5.0",
     "@vitest/coverage-istanbul": "1.1.3",
     "@vitest/ui": "1.1.3",
-    "electron": "^27.1.0",
+    "electron": "^28.2.1",
     "eslint": "^8.54.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-i": "^2.29.0",
@@ -88,13 +88,12 @@
     "eslint-plugin-unused-imports": "^3.0.0",
     "eslint-plugin-vue": "^9.18.1",
     "fake-indexeddb": "5.0.2",
-    "happy-dom": "^12.10.3",
-    "husky": "^8.0.3",
+    "happy-dom": "^13.0.0",
+    "husky": "^9.0.6",
     "lint-staged": "^15.1.0",
     "msw": "^2.0.8",
     "nanoid": "^5.0.3",
-    "nx": "^17.1.3",
-    "nx-cloud": "^16.5.2",
+    "nx": "^17.2.8",
     "nyc": "^15.1.0",
     "oxlint": "0.0.22",
     "prettier": "^3.1.0",

packages/backend/server/.env.example

@@ -1,8 +1,4 @@
-DATABASE_URL="postgresql://affine@localhost:5432/affine"
-NEXTAUTH_URL="http://localhost:8080"
-OAUTH_EMAIL_SENDER="noreply@toeverything.info"
-OAUTH_EMAIL_LOGIN=""
-OAUTH_EMAIL_PASSWORD=""
-ENABLE_LOCAL_EMAIL="true"
-STRIPE_API_KEY=
-STRIPE_WEBHOOK_KEY=
+# AFFINE_SERVER_PORT=3010
+# AFFINE_SERVER_HOST=app.affine.pro
+# AFFINE_SERVER_HTTPS=true
+# DATABASE_URL="postgres://affine@localhost:5432/affine"

packages/backend/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/server",
   "private": true,
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Affine Node.js server",
   "type": "module",
   "bin": {
@@ -9,20 +9,21 @@
   },
   "scripts": {
     "build": "tsc",
-    "start": "node --loader ts-node/esm/transpile-only.mjs --es-module-specifier-resolution node ./src/index.ts",
+    "start": "node --loader ts-node/esm/transpile-only.mjs ./src/index.ts",
     "dev": "nodemon ./src/index.ts",
     "test": "ava --concurrency 1 --serial",
     "test:coverage": "c8 ava --concurrency 1 --serial",
     "postinstall": "prisma generate",
-    "data-migration": "node --loader ts-node/esm/transpile-only.mjs --es-module-specifier-resolution node ./src/data/app.ts",
-    "predeploy": "yarn prisma migrate deploy && node --es-module-specifier-resolution node ./dist/data/app.js run"
+    "data-migration": "node --loader ts-node/esm/transpile-only.mjs ./src/data/index.ts",
+    "predeploy": "yarn prisma migrate deploy && node --import ./scripts/register.js ./dist/data/index.js run"
   },
   "dependencies": {
     "@apollo/server": "^4.9.5",
     "@auth/prisma-adapter": "^1.0.7",
-    "@aws-sdk/client-s3": "^3.454.0",
+    "@aws-sdk/client-s3": "^3.499.0",
     "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.17.0",
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.1.0",
+    "@google-cloud/opentelemetry-resource-util": "^2.1.0",
     "@keyv/redis": "^2.8.0",
     "@nestjs/apollo": "^12.0.11",
     "@nestjs/common": "^10.2.10",
@@ -32,35 +33,39 @@
     "@nestjs/platform-express": "^10.2.10",
     "@nestjs/platform-socket.io": "^10.2.10",
     "@nestjs/schedule": "^4.0.0",
+    "@nestjs/serve-static": "^4.0.0",
     "@nestjs/throttler": "^5.0.1",
     "@nestjs/websockets": "^10.2.10",
     "@node-rs/argon2": "^1.5.2",
     "@node-rs/crc32": "^1.7.2",
     "@node-rs/jsonwebtoken": "^0.3.0",
     "@opentelemetry/api": "^1.7.0",
-    "@opentelemetry/core": "^1.19.0",
-    "@opentelemetry/exporter-prometheus": "^0.46.0",
-    "@opentelemetry/exporter-zipkin": "^1.19.0",
-    "@opentelemetry/host-metrics": "^0.34.0",
-    "@opentelemetry/instrumentation": "^0.46.0",
-    "@opentelemetry/instrumentation-graphql": "^0.36.0",
-    "@opentelemetry/instrumentation-http": "^0.46.0",
-    "@opentelemetry/instrumentation-ioredis": "^0.36.0",
-    "@opentelemetry/instrumentation-nestjs-core": "^0.33.3",
-    "@opentelemetry/instrumentation-socket.io": "^0.35.0",
-    "@opentelemetry/resources": "^1.19.0",
-    "@opentelemetry/sdk-metrics": "^1.19.0",
-    "@opentelemetry/sdk-node": "^0.46.0",
-    "@opentelemetry/sdk-trace-node": "^1.19.0",
+    "@opentelemetry/core": "^1.21.0",
+    "@opentelemetry/exporter-prometheus": "^0.48.0",
+    "@opentelemetry/exporter-zipkin": "^1.21.0",
+    "@opentelemetry/host-metrics": "^0.35.0",
+    "@opentelemetry/instrumentation": "^0.48.0",
+    "@opentelemetry/instrumentation-graphql": "^0.37.0",
+    "@opentelemetry/instrumentation-http": "^0.48.0",
+    "@opentelemetry/instrumentation-ioredis": "^0.37.0",
+    "@opentelemetry/instrumentation-nestjs-core": "^0.34.0",
+    "@opentelemetry/instrumentation-socket.io": "^0.36.0",
+    "@opentelemetry/resources": "^1.21.0",
+    "@opentelemetry/sdk-metrics": "^1.21.0",
+    "@opentelemetry/sdk-node": "^0.48.0",
+    "@opentelemetry/sdk-trace-node": "^1.21.0",
+    "@opentelemetry/semantic-conventions": "^1.21.0",
     "@prisma/client": "^5.7.1",
     "@prisma/instrumentation": "^5.7.1",
     "@socket.io/redis-adapter": "^8.2.1",
     "cookie-parser": "^1.4.6",
     "dotenv": "^16.3.1",
+    "dotenv-cli": "^7.3.0",
     "express": "^4.18.2",
     "file-type": "^19.0.0",
     "get-stream": "^8.0.1",
     "graphql": "^16.8.1",
+    "graphql-scalars": "^1.22.4",
     "graphql-type-json": "^0.3.2",
     "graphql-upload": "^16.0.2",
     "ioredis": "^5.3.2",
@@ -81,6 +86,8 @@
     "semver": "^7.5.4",
     "socket.io": "^4.7.2",
     "stripe": "^14.5.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3",
     "ws": "^8.14.2",
     "yjs": "^13.6.10",
     "zod": "^3.22.4"
@@ -107,11 +114,10 @@
     "c8": "^9.0.0",
     "nodemon": "^3.0.1",
     "sinon": "^17.0.1",
-    "supertest": "^6.3.3",
-    "ts-node": "^10.9.1",
-    "typescript": "^5.3.2"
+    "supertest": "^6.3.3"
   },
   "ava": {
+    "timeout": "1m",
     "extensions": {
       "ts": "module"
     },
@@ -133,10 +139,11 @@
     "environmentVariables": {
       "TS_NODE_PROJECT": "./tests/tsconfig.json",
       "NODE_ENV": "test",
-      "ENABLE_LOCAL_EMAIL": "true",
-      "OAUTH_EMAIL_LOGIN": "noreply@toeverything.info",
-      "OAUTH_EMAIL_PASSWORD": "affine",
-      "OAUTH_EMAIL_SENDER": "noreply@toeverything.info",
+      "MAILER_HOST": "0.0.0.0",
+      "MAILER_PORT": "1025",
+      "MAILER_USER": "noreply@toeverything.info",
+      "MAILER_PASSWORD": "affine",
+      "MAILER_SENDER": "noreply@toeverything.info",
       "FEATURES_EARLY_ACCESS_PREVIEW": "false"
     }
   },
@@ -156,7 +163,6 @@
     "env": {
       "TS_NODE_TRANSPILE_ONLY": true,
       "TS_NODE_PROJECT": "./tsconfig.json",
-      "NODE_ENV": "development",
       "DEBUG": "affine:*",
       "FORCE_COLOR": true,
       "DEBUG_COLORS": true

packages/backend/server/schema.prisma

@@ -265,7 +265,9 @@ model Snapshot {
   seq         Int      @default(0) @db.Integer
   state       Bytes?   @db.ByteA
   createdAt   DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
-  updatedAt   DateTime @updatedAt @map("updated_at") @db.Timestamptz(6)
+  // the `updated_at` field will not record the time of record changed,
+  // but the created time of last seen update that has been merged into snapshot.
+  updatedAt   DateTime @map("updated_at") @db.Timestamptz(6)
 
   @@id([id, workspaceId])
   @@map("snapshots")

packages/backend/server/scripts/loader.js

@@ -0,0 +1,11 @@
+import { create, createEsmHooks } from 'ts-node';
+
+const service = create({
+  experimentalSpecifierResolution: 'node',
+  transpileOnly: true,
+  logError: true,
+  skipProject: true,
+});
+const hooks = createEsmHooks(service);
+
+export const resolve = hooks.resolve;

packages/backend/server/scripts/register.js

@@ -0,0 +1,4 @@
+import { register } from 'node:module';
+import { pathToFileURL } from 'node:url';
+
+register('./scripts/loader.js', pathToFileURL('./'));

packages/backend/server/scripts/self-host-predeploy.js

@@ -0,0 +1,52 @@
+import { execSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const SELF_HOST_CONFIG_DIR = '/root/.affine/config';
+/**
+ * @type {Array<{ from: string; to?: string, modifier?: (content: string): string }>}
+ */
+const configFiles = [
+  { from: './.env.example', to: '.env' },
+  { from: './dist/config/affine.js', modifier: configCleaner },
+  { from: './dist/config/affine.env.js', modifier: configCleaner },
+];
+
+function configCleaner(content) {
+  return content.replace(
+    /(^\/\/#.*$)|(^\/\/\s+TODO.*$)|("use\sstrict";?)|(^.*eslint-disable.*$)/gm,
+    ''
+  );
+}
+
+function prepare() {
+  fs.mkdirSync(SELF_HOST_CONFIG_DIR, { recursive: true });
+
+  for (const { from, to, modifier } of configFiles) {
+    const targetFileName = to ?? path.parse(from).base;
+    const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, targetFileName);
+    if (!fs.existsSync(targetFilePath)) {
+      console.log(`creating config file [${targetFilePath}].`);
+      if (modifier) {
+        const content = fs.readFileSync(from, 'utf-8');
+        fs.writeFileSync(targetFilePath, modifier(content), 'utf-8');
+      } else {
+        fs.cpSync(from, targetFilePath, {
+          force: false,
+        });
+      }
+    }
+  }
+}
+
+function runPredeployScript() {
+  console.log('running predeploy script.');
+  execSync('yarn predeploy', {
+    encoding: 'utf-8',
+    env: process.env,
+    stdio: 'inherit',
+  });
+}
+
+prepare();
+runPredeployScript();

packages/backend/server/src/affine.config.ts

@@ -1,25 +0,0 @@
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
-// Custom configurations
-
-const env = process.env;
-const node = AFFiNE.node;
-
-// TODO: may be separate config overring in `affine.[env].config`?
-if (node.prod && env.R2_OBJECT_STORAGE_ACCOUNT_ID) {
-  AFFiNE.storage.providers.r2 = {
-    accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID,
-    credentials: {
-      accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!,
-      secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!,
-    },
-  };
-  AFFiNE.storage.storages.avatar.provider = 'r2';
-  AFFiNE.storage.storages.avatar.bucket = 'account-avatar';
-  AFFiNE.storage.storages.avatar.publicLinkFactory = key =>
-    `https://avatar.affineassets.com/${key}`;
-
-  AFFiNE.storage.storages.blob.provider = 'r2';
-  AFFiNE.storage.storages.blob.bucket = `workspace-blobs-${
-    AFFiNE.affine.canary ? 'canary' : 'prod'
-  }`;
-}

packages/backend/server/src/affine.ts

@@ -1,3 +0,0 @@
-import { getDefaultAFFiNEConfig } from './config/default';
-
-globalThis.AFFiNE = getDefaultAFFiNEConfig();

packages/backend/server/src/app.controller.ts

@@ -1,13 +1,18 @@
 import { Controller, Get } from '@nestjs/common';
 
+import { Config } from './fundamentals/config';
+
 @Controller('/')
 export class AppController {
+  constructor(private readonly config: Config) {}
+
   @Get()
   info() {
-    const version = AFFiNE.version;
     return {
-      compatibility: version,
-      message: `AFFiNE ${version} Server`,
+      compatibility: this.config.version,
+      message: `AFFiNE ${this.config.version} Server`,
+      type: this.config.type,
+      flavor: this.config.flavor,
     };
   }
 }

packages/backend/server/src/app.module.ts

@@ -0,0 +1,169 @@
+import { join } from 'node:path';
+
+import { Logger, Module } from '@nestjs/common';
+import { APP_INTERCEPTOR } from '@nestjs/core';
+import { ScheduleModule } from '@nestjs/schedule';
+import { ServeStaticModule } from '@nestjs/serve-static';
+import { get } from 'lodash-es';
+
+import { AppController } from './app.controller';
+import { AuthModule } from './core/auth';
+import { ADD_ENABLED_FEATURES, ServerConfigModule } from './core/config';
+import { DocModule } from './core/doc';
+import { FeatureModule } from './core/features';
+import { QuotaModule } from './core/quota';
+import { StorageModule } from './core/storage';
+import { SyncModule } from './core/sync';
+import { UsersModule } from './core/users';
+import { WorkspaceModule } from './core/workspaces';
+import { getOptionalModuleMetadata } from './fundamentals';
+import { CacheInterceptor, CacheModule } from './fundamentals/cache';
+import {
+  type AvailablePlugins,
+  Config,
+  ConfigModule,
+} from './fundamentals/config';
+import { EventModule } from './fundamentals/event';
+import { GqlModule } from './fundamentals/graphql';
+import { MailModule } from './fundamentals/mailer';
+import { MetricsModule } from './fundamentals/metrics';
+import { PrismaModule } from './fundamentals/prisma';
+import { SessionModule } from './fundamentals/session';
+import { RateLimiterModule } from './fundamentals/throttler';
+import { WebSocketModule } from './fundamentals/websocket';
+import { pluginsMap } from './plugins';
+
+export const FunctionalityModules = [
+  ConfigModule.forRoot(),
+  ScheduleModule.forRoot(),
+  EventModule,
+  CacheModule,
+  PrismaModule,
+  MetricsModule,
+  RateLimiterModule,
+  SessionModule,
+  MailModule,
+];
+
+export class AppModuleBuilder {
+  private readonly modules: AFFiNEModule[] = [];
+  constructor(private readonly config: Config) {}
+
+  use(...modules: AFFiNEModule[]): this {
+    modules.forEach(m => {
+      const requirements = getOptionalModuleMetadata(m, 'requires');
+      // if condition not set or condition met, include the module
+      if (requirements?.length) {
+        const nonMetRequirements = requirements.filter(c => {
+          const value = get(this.config, c);
+          return (
+            value === undefined ||
+            value === null ||
+            (typeof value === 'string' && value.trim().length === 0)
+          );
+        });
+
+        if (nonMetRequirements.length) {
+          const name = 'module' in m ? m.module.name : m.name;
+          new Logger(name).warn(
+            `${name} is not enabled because of the required configuration is not satisfied.`,
+            'Unsatisfied configuration:',
+            ...nonMetRequirements.map(config => `  AFFiNE.${config}`)
+          );
+          return;
+        }
+      }
+
+      const predicator = getOptionalModuleMetadata(m, 'if');
+      if (predicator && !predicator(this.config)) {
+        return;
+      }
+
+      const contribution = getOptionalModuleMetadata(m, 'contributesTo');
+      if (contribution) {
+        ADD_ENABLED_FEATURES(contribution);
+      }
+      this.modules.push(m);
+    });
+
+    return this;
+  }
+
+  useIf(
+    predicator: (config: Config) => boolean,
+    ...modules: AFFiNEModule[]
+  ): this {
+    if (predicator(this.config)) {
+      this.use(...modules);
+    }
+
+    return this;
+  }
+
+  compile() {
+    @Module({
+      providers: [
+        {
+          provide: APP_INTERCEPTOR,
+          useClass: CacheInterceptor,
+        },
+      ],
+      imports: this.modules,
+      controllers: this.config.isSelfhosted ? [] : [AppController],
+    })
+    class AppModule {}
+
+    return AppModule;
+  }
+}
+
+function buildAppModule() {
+  const factor = new AppModuleBuilder(AFFiNE);
+
+  factor
+    // common fundamental modules
+    .use(...FunctionalityModules)
+    // auth
+    .use(AuthModule)
+
+    // business modules
+    .use(DocModule)
+
+    // sync server only
+    .useIf(config => config.flavor.sync, SyncModule)
+
+    // graphql server only
+    .useIf(
+      config => config.flavor.graphql,
+      ServerConfigModule,
+      WebSocketModule,
+      GqlModule,
+      StorageModule,
+      UsersModule,
+      WorkspaceModule,
+      FeatureModule,
+      QuotaModule
+    )
+
+    // self hosted server only
+    .useIf(
+      config => config.isSelfhosted,
+      ServeStaticModule.forRoot({
+        rootPath: join('/app', 'static'),
+      })
+    );
+
+  // plugin modules
+  AFFiNE.plugins.enabled.forEach(name => {
+    const plugin = pluginsMap.get(name as AvailablePlugins);
+    if (!plugin) {
+      throw new Error(`Unknown plugin ${name}`);
+    }
+
+    factor.use(plugin);
+  });
+
+  return factor.compile();
+}
+
+export const AppModule = buildAppModule();

packages/backend/server/src/app.ts

@@ -1,34 +1,48 @@
-import { Module } from '@nestjs/common';
-import { APP_INTERCEPTOR } from '@nestjs/core';
+import { Type } from '@nestjs/common';
+import { NestFactory } from '@nestjs/core';
+import type { NestExpressApplication } from '@nestjs/platform-express';
+import cookieParser from 'cookie-parser';
+import graphqlUploadExpress from 'graphql-upload/graphqlUploadExpress.mjs';
 
-import { AppController } from './app.controller';
-import { CacheInterceptor, CacheModule } from './cache';
-import { ConfigModule } from './config';
-import { EventModule } from './event';
-import { BusinessModules } from './modules';
-import { AuthModule } from './modules/auth';
-import { PrismaModule } from './prisma';
-import { SessionModule } from './session';
-import { RateLimiterModule } from './throttler';
+import { SocketIoAdapter } from './fundamentals';
+import { SocketIoAdapterImpl } from './fundamentals/websocket';
+import { ExceptionLogger } from './middleware/exception-logger';
+import { serverTimingAndCache } from './middleware/timing';
 
-const BasicModules = [
-  PrismaModule,
-  ConfigModule.forRoot(),
-  CacheModule,
-  EventModule,
-  SessionModule,
-  RateLimiterModule,
-  AuthModule,
-];
+export async function createApp() {
+  const { AppModule } = await import('./app.module');
 
-@Module({
-  providers: [
-    {
-      provide: APP_INTERCEPTOR,
-      useClass: CacheInterceptor,
-    },
-  ],
-  imports: [...BasicModules, ...BusinessModules],
-  controllers: [AppController],
-})
-export class AppModule {}
+  const app = await NestFactory.create<NestExpressApplication>(AppModule, {
+    cors: true,
+    rawBody: true,
+    bodyParser: true,
+    logger: AFFiNE.affine.stable ? ['log'] : ['verbose'],
+  });
+
+  app.use(serverTimingAndCache);
+
+  app.use(
+    graphqlUploadExpress({
+      // TODO: dynamic limit by quota
+      maxFileSize: 100 * 1024 * 1024,
+      maxFiles: 5,
+    })
+  );
+
+  app.useGlobalFilters(new ExceptionLogger());
+  app.use(cookieParser());
+
+  if (AFFiNE.flavor.sync) {
+    const SocketIoAdapter = app.get<Type<SocketIoAdapter>>(
+      SocketIoAdapterImpl,
+      {
+        strict: false,
+      }
+    );
+
+    const adapter = new SocketIoAdapter(app);
+    app.useWebSocketAdapter(adapter);
+  }
+
+  return app;
+}

packages/backend/server/src/cache/index.ts

@@ -1,26 +0,0 @@
-import { FactoryProvider, Global, Module } from '@nestjs/common';
-import { Redis } from 'ioredis';
-
-import { Config } from '../config';
-import { LocalCache } from './cache';
-import { RedisCache } from './redis';
-
-const CacheProvider: FactoryProvider = {
-  provide: LocalCache,
-  useFactory: (config: Config) => {
-    return config.redis.enabled
-      ? new RedisCache(new Redis(config.redis))
-      : new LocalCache();
-  },
-  inject: [Config],
-};
-
-@Global()
-@Module({
-  providers: [CacheProvider],
-  exports: [CacheProvider],
-})
-export class CacheModule {}
-export { LocalCache as Cache };
-
-export { CacheInterceptor, MakeCache, PreventCache } from './interceptor';

packages/backend/server/src/config/affine.env.ts

@@ -0,0 +1,37 @@
+// Convenient way to map environment variables to config values.
+AFFiNE.ENV_MAP = {
+  AFFINE_SERVER_PORT: ['port', 'int'],
+  AFFINE_SERVER_HOST: 'host',
+  AFFINE_SERVER_SUB_PATH: 'path',
+  AFFINE_SERVER_HTTPS: ['https', 'boolean'],
+  DATABASE_URL: 'db.url',
+  ENABLE_CAPTCHA: ['auth.captcha.enable', 'boolean'],
+  CAPTCHA_TURNSTILE_SECRET: ['auth.captcha.turnstile.secret', 'string'],
+  OAUTH_GOOGLE_ENABLED: ['auth.oauthProviders.google.enabled', 'boolean'],
+  OAUTH_GOOGLE_CLIENT_ID: 'auth.oauthProviders.google.clientId',
+  OAUTH_GOOGLE_CLIENT_SECRET: 'auth.oauthProviders.google.clientSecret',
+  OAUTH_GITHUB_ENABLED: ['auth.oauthProviders.github.enabled', 'boolean'],
+  OAUTH_GITHUB_CLIENT_ID: 'auth.oauthProviders.github.clientId',
+  OAUTH_GITHUB_CLIENT_SECRET: 'auth.oauthProviders.github.clientSecret',
+  MAILER_HOST: 'mailer.host',
+  MAILER_PORT: ['mailer.port', 'int'],
+  MAILER_USER: 'mailer.auth.user',
+  MAILER_PASSWORD: 'mailer.auth.pass',
+  MAILER_SENDER: 'mailer.from.address',
+  MAILER_SECURE: ['mailer.secure', 'boolean'],
+  THROTTLE_TTL: ['rateLimiter.ttl', 'int'],
+  THROTTLE_LIMIT: ['rateLimiter.limit', 'int'],
+  REDIS_SERVER_HOST: 'plugins.redis.host',
+  REDIS_SERVER_PORT: ['plugins.redis.port', 'int'],
+  REDIS_SERVER_USER: 'plugins.redis.username',
+  REDIS_SERVER_PASSWORD: 'plugins.redis.password',
+  REDIS_SERVER_DATABASE: ['plugins.redis.db', 'int'],
+  DOC_MERGE_INTERVAL: ['doc.manager.updatePollInterval', 'int'],
+  DOC_MERGE_USE_JWST_CODEC: [
+    'doc.manager.experimentalMergeWithYOcto',
+    'boolean',
+  ],
+  STRIPE_API_KEY: 'plugins.payment.stripe.keys.APIKey',
+  STRIPE_WEBHOOK_KEY: 'plugins.payment.stripe.keys.webhookKey',
+  FEATURES_EARLY_ACCESS_PREVIEW: ['featureFlags.earlyAccessPreview', 'boolean'],
+};

packages/backend/server/src/config/affine.self.ts

@@ -0,0 +1,54 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+// Custom configurations for AFFiNE Cloud
+// ====================================================================================
+// Q: WHY THIS FILE EXISTS?
+// A: AFFiNE deployment environment may have a lot of custom environment variables,
+//    which are not suitable to be put in the `affine.ts` file.
+//    For example, AFFiNE Cloud Clusters are deployed on Google Cloud Platform.
+//    We need to enable the `gcloud` plugin to make sure the nodes working well,
+//    but the default selfhost version may not require it.
+//    So it's not a good idea to put such logic in the common `affine.ts` file.
+//
+//    ```
+//    if (AFFiNE.deploy) {
+//      AFFiNE.plugins.use('gcloud');
+//    }
+//    ```
+// ====================================================================================
+const env = process.env;
+
+AFFiNE.metrics.enabled = !AFFiNE.node.test;
+
+if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {
+  AFFiNE.storage.providers.r2 = {
+    accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID,
+    credentials: {
+      accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!,
+      secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!,
+    },
+  };
+  AFFiNE.storage.storages.avatar.provider = 'r2';
+  AFFiNE.storage.storages.avatar.bucket = 'account-avatar';
+  AFFiNE.storage.storages.avatar.publicLinkFactory = key =>
+    `https://avatar.affineassets.com/${key}`;
+
+  AFFiNE.storage.storages.blob.provider = 'r2';
+  AFFiNE.storage.storages.blob.bucket = `workspace-blobs-${
+    AFFiNE.affine.canary ? 'canary' : 'prod'
+  }`;
+}
+
+AFFiNE.plugins.use('redis');
+AFFiNE.plugins.use('payment');
+
+if (AFFiNE.deploy) {
+  AFFiNE.mailer = {
+    service: 'gmail',
+    auth: {
+      user: env.MAILER_USER,
+      pass: env.MAILER_PASSWORD,
+    },
+  };
+
+  AFFiNE.plugins.use('gcloud');
+}

packages/backend/server/src/config/affine.ts

@@ -0,0 +1,94 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+//
+// ###############################################################
+// ##                AFFiNE Configuration System                ##
+// ###############################################################
+// Here is the file of all AFFiNE configurations that will affect runtime behavior.
+// Override any configuration here and it will be merged when starting the server.
+// Any changes in this file won't take effect before server restarted.
+//
+//
+// > Configurations merge order
+//   1. load environment variables (`.env` if provided, and from system)
+//   2. load `src/fundamentals/config/default.ts` for all default settings
+//   3. apply `./affine.ts` patches (this file)
+//   4. apply `./affine.env.ts` patches
+//
+//
+// ###############################################################
+// ##                       General settings                    ##
+// ###############################################################
+//
+// /* The unique identity of the server */
+// AFFiNE.serverId = 'some-randome-uuid';
+//
+// /* The name of AFFiNE Server, may show on the UI */
+// AFFiNE.serverName = 'Your Cool AFFiNE Selfhosted Cloud';
+//
+// /* Whether the server is deployed behind a HTTPS proxied environment */
+AFFiNE.https = false;
+// /* Domain of your server that your server will be available at */
+AFFiNE.host = 'localhost';
+// /* The local port of your server that will listen on */
+AFFiNE.port = 3010;
+// /* The sub path of your server */
+// /* For example, if you set `AFFiNE.path = '/affine'`, then the server will be available at `${domain}/affine` */
+// AFFiNE.path = '/affine';
+//
+//
+// ###############################################################
+// ##                       Database settings                   ##
+// ###############################################################
+//
+// /* The URL of the database where most of AFFiNE server data will be stored in */
+// AFFiNE.db.url = 'postgres://user:passsword@localhost:5432/affine';
+//
+//
+// ###############################################################
+// ##                   Server Function settings                ##
+// ###############################################################
+//
+// /* Whether enable metrics and tracing while running the server */
+// /* The metrics will be available at `http://localhost:9464/metrics` with [Prometheus] format exported */
+// AFFiNE.metrics.enabled = true;
+//
+// /* GraphQL configurations that control the behavior of the Apollo Server behind */
+// /* @see https://www.apollographql.com/docs/apollo-server/api/apollo-server */
+// AFFiNE.graphql = {
+//   /* Path to mount GraphQL API */
+//   path: '/graphql',
+//   buildSchemaOptions: {
+//     numberScalarMode: 'integer',
+//   },
+//   /* Whether allow client to query the schema introspection */
+//   introspection: true,
+//   /* Whether enable GraphQL Playground UI */
+//   playground: true,
+// }
+//
+// /* Doc Store & Collaberation */
+// /* How long the buffer time of creating a new history snapshot when doc get updated */
+// AFFiNE.doc.history.interval = 1000 * 60 * 10; // 10 minutes
+//
+// /* Use `y-octo` to merge updates at the same time when merging using Yjs */
+// AFFiNE.doc.manager.experimentalMergeWithYOcto = true;
+//
+// /* How often the manager will start a new turn of merging pending updates into doc snapshot */
+// AFFiNE.doc.manager.updatePollInterval = 1000 * 3;
+//
+//
+// ###############################################################
+// ##                        Plugins settings                   ##
+// ###############################################################
+//
+// /* Redis Plugin */
+// /* Provide caching and session storing backed by Redis. */
+// /* Useful when you deploy AFFiNE server in a cluster. */
+AFFiNE.plugins.use('redis', {
+  /* override options */
+});
+// /* Payment Plugin */
+AFFiNE.plugins.use('payment', {
+  stripe: { keys: {}, apiVersion: '2023-10-16' },
+});
+//

packages/backend/server/src/config/default.ts

@@ -1,214 +0,0 @@
-/// <reference types="../global.d.ts" />
-
-import { createPrivateKey, createPublicKey } from 'node:crypto';
-
-import parse from 'parse-duration';
-
-import pkg from '../../package.json' assert { type: 'json' };
-import type { AFFiNEConfig, ServerFlavor } from './def';
-import { applyEnvToConfig } from './env';
-import { getDefaultAFFiNEStorageConfig } from './storage';
-
-export const SERVER_FLAVOR = (process.env.SERVER_FLAVOR ??
-  'allinone') as ServerFlavor;
-
-// Don't use this in production
-export const examplePrivateKey = `-----BEGIN EC PRIVATE KEY-----
-MHcCAQEEIEtyAJLIULkphVhqXqxk4Nr8Ggty3XLwUJWBxzAWCWTMoAoGCCqGSM49
-AwEHoUQDQgAEF3U/0wIeJ3jRKXeFKqQyBKlr9F7xaAUScRrAuSP33rajm3cdfihI
-3JvMxVNsS2lE8PSGQrvDrJZaDo0L+Lq9Gg==
------END EC PRIVATE KEY-----`;
-
-const jwtKeyPair = (function () {
-  const AUTH_PRIVATE_KEY = process.env.AUTH_PRIVATE_KEY ?? examplePrivateKey;
-  const privateKey = createPrivateKey({
-    key: Buffer.from(AUTH_PRIVATE_KEY),
-    format: 'pem',
-    type: 'sec1',
-  })
-    .export({
-      format: 'pem',
-      type: 'pkcs8',
-    })
-    .toString('utf8');
-  const publicKey = createPublicKey({
-    key: Buffer.from(AUTH_PRIVATE_KEY),
-    format: 'pem',
-    type: 'spki',
-  })
-    .export({
-      format: 'pem',
-      type: 'spki',
-    })
-    .toString('utf8');
-
-  return {
-    publicKey,
-    privateKey,
-  };
-})();
-
-export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
-  const defaultConfig = {
-    serverId: 'affine-nestjs-server',
-    version: pkg.version,
-    ENV_MAP: {
-      AFFINE_SERVER_PORT: ['port', 'int'],
-      AFFINE_SERVER_HOST: 'host',
-      AFFINE_SERVER_SUB_PATH: 'path',
-      AFFINE_ENV: 'affineEnv',
-      DATABASE_URL: 'db.url',
-      ENABLE_CAPTCHA: ['auth.captcha.enable', 'boolean'],
-      CAPTCHA_TURNSTILE_SECRET: ['auth.captcha.turnstile.secret', 'string'],
-      OAUTH_GOOGLE_ENABLED: ['auth.oauthProviders.google.enabled', 'boolean'],
-      OAUTH_GOOGLE_CLIENT_ID: 'auth.oauthProviders.google.clientId',
-      OAUTH_GOOGLE_CLIENT_SECRET: 'auth.oauthProviders.google.clientSecret',
-      OAUTH_GITHUB_ENABLED: ['auth.oauthProviders.github.enabled', 'boolean'],
-      OAUTH_GITHUB_CLIENT_ID: 'auth.oauthProviders.github.clientId',
-      OAUTH_GITHUB_CLIENT_SECRET: 'auth.oauthProviders.github.clientSecret',
-      OAUTH_EMAIL_LOGIN: 'auth.email.login',
-      OAUTH_EMAIL_SENDER: 'auth.email.sender',
-      OAUTH_EMAIL_SERVER: 'auth.email.server',
-      OAUTH_EMAIL_PORT: ['auth.email.port', 'int'],
-      OAUTH_EMAIL_PASSWORD: 'auth.email.password',
-      THROTTLE_TTL: ['rateLimiter.ttl', 'int'],
-      THROTTLE_LIMIT: ['rateLimiter.limit', 'int'],
-      REDIS_SERVER_ENABLED: ['redis.enabled', 'boolean'],
-      REDIS_SERVER_HOST: 'redis.host',
-      REDIS_SERVER_PORT: ['redis.port', 'int'],
-      REDIS_SERVER_USER: 'redis.username',
-      REDIS_SERVER_PASSWORD: 'redis.password',
-      REDIS_SERVER_DATABASE: ['redis.database', 'int'],
-      DOC_MERGE_INTERVAL: ['doc.manager.updatePollInterval', 'int'],
-      DOC_MERGE_USE_JWST_CODEC: [
-        'doc.manager.experimentalMergeWithJwstCodec',
-        'boolean',
-      ],
-      ENABLE_LOCAL_EMAIL: ['auth.localEmail', 'boolean'],
-      STRIPE_API_KEY: 'payment.stripe.keys.APIKey',
-      STRIPE_WEBHOOK_KEY: 'payment.stripe.keys.webhookKey',
-      FEATURES_EARLY_ACCESS_PREVIEW: [
-        'featureFlags.earlyAccessPreview',
-        'boolean',
-      ],
-    } satisfies AFFiNEConfig['ENV_MAP'],
-    affineEnv: 'dev',
-    get affine() {
-      const env = this.affineEnv;
-      return {
-        canary: env === 'dev',
-        beta: env === 'beta',
-        stable: env === 'production',
-      };
-    },
-    env: process.env.NODE_ENV ?? 'development',
-    get node() {
-      const env = this.env;
-      return {
-        prod: env === 'production',
-        dev: env === 'development',
-        test: env === 'test',
-      };
-    },
-    get deploy() {
-      return !this.node.dev && !this.node.test;
-    },
-    featureFlags: {
-      earlyAccessPreview: false,
-    },
-    get https() {
-      return !this.node.dev;
-    },
-    host: 'localhost',
-    port: 3010,
-    path: '',
-    db: {
-      url: '',
-    },
-    get origin() {
-      return this.node.dev
-        ? 'http://localhost:8080'
-        : `${this.https ? 'https' : 'http'}://${this.host}${
-            this.host === 'localhost' ? `:${this.port}` : ''
-          }`;
-    },
-    get baseUrl() {
-      return `${this.origin}${this.path}`;
-    },
-    graphql: {
-      buildSchemaOptions: {
-        numberScalarMode: 'integer',
-      },
-      introspection: true,
-      playground: true,
-    },
-    auth: {
-      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-      accessTokenExpiresIn: parse('1h')! / 1000,
-      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-      refreshTokenExpiresIn: parse('7d')! / 1000,
-      leeway: 60,
-      captcha: {
-        enable: false,
-        turnstile: {
-          secret: '1x0000000000000000000000000000000AA',
-        },
-        challenge: {
-          bits: 20,
-        },
-      },
-      privateKey: jwtKeyPair.privateKey,
-      publicKey: jwtKeyPair.publicKey,
-      enableSignup: true,
-      enableOauth: false,
-      get nextAuthSecret() {
-        return this.privateKey;
-      },
-      oauthProviders: {},
-      localEmail: false,
-      email: {
-        server: 'smtp.gmail.com',
-        port: 465,
-        login: '',
-        sender: '',
-        password: '',
-      },
-    },
-    storage: getDefaultAFFiNEStorageConfig(),
-    rateLimiter: {
-      ttl: 60,
-      limit: 60,
-    },
-    redis: {
-      enabled: false,
-      host: '127.0.0.1',
-      port: 6379,
-      username: '',
-      password: '',
-      database: 0,
-    },
-    doc: {
-      manager: {
-        enableUpdateAutoMerging: SERVER_FLAVOR !== 'sync',
-        updatePollInterval: 3000,
-        experimentalMergeWithJwstCodec: false,
-      },
-      history: {
-        interval: 1000 * 60 * 10 /* 10 mins */,
-      },
-    },
-    payment: {
-      stripe: {
-        keys: {
-          APIKey: '',
-          webhookKey: '',
-        },
-        apiVersion: '2023-10-16',
-      },
-    },
-  } satisfies AFFiNEConfig;
-
-  applyEnvToConfig(defaultConfig);
-
-  return defaultConfig;
-};

packages/backend/server/src/config/env.ts

@@ -1,17 +0,0 @@
-import { set } from 'lodash-es';
-
-import { type AFFiNEConfig, parseEnvValue } from './def';
-
-export function applyEnvToConfig(rawConfig: AFFiNEConfig) {
-  for (const env in rawConfig.ENV_MAP) {
-    const config = rawConfig.ENV_MAP[env];
-    const [path, value] =
-      typeof config === 'string'
-        ? [config, process.env[env]]
-        : [config[0], parseEnvValue(process.env[env], config[1])];
-
-    if (value !== undefined) {
-      set(rawConfig, path, value);
-    }
-  }
-}

packages/backend/server/src/constants.ts

@@ -1,3 +0,0 @@
-export const OPERATION_NAME = 'x-operation-name';
-
-export const REQUEST_ID = 'x-request-id';

packages/backend/server/src/core/auth/guard.ts

@@ -10,8 +10,10 @@ import { Reflector } from '@nestjs/core';
 import type { NextAuthOptions } from 'next-auth';
 import { AuthHandler } from 'next-auth/core';
 
-import { PrismaService } from '../../prisma';
-import { getRequestResponseFromContext } from '../../utils/nestjs';
+import {
+  getRequestResponseFromContext,
+  PrismaService,
+} from '../../fundamentals';
 import { NextAuthOptionsProvide } from './next-auth-options';
 import { AuthService } from './service';
 

packages/backend/server/src/core/auth/index.ts

@@ -1,7 +1,5 @@
 import { Global, Module } from '@nestjs/common';
 
-import { SessionModule } from '../../session';
-import { MAILER, MailService } from './mailer';
 import { NextAuthController } from './next-auth.controller';
 import { NextAuthOptionsProvider } from './next-auth-options';
 import { AuthResolver } from './resolver';
@@ -9,18 +7,12 @@ import { AuthService } from './service';
 
 @Global()
 @Module({
-  imports: [SessionModule],
-  providers: [
-    AuthService,
-    AuthResolver,
-    NextAuthOptionsProvider,
-    MAILER,
-    MailService,
-  ],
-  exports: [AuthService, NextAuthOptionsProvider, MailService],
+  providers: [AuthService, AuthResolver, NextAuthOptionsProvider],
+  exports: [AuthService, NextAuthOptionsProvider],
   controllers: [NextAuthController],
 })
 export class AuthModule {}
 
 export * from './guard';
 export { TokenType } from './resolver';
+export { AuthService };

packages/backend/server/src/core/auth/next-auth-options.ts

@@ -8,12 +8,14 @@ import Email from 'next-auth/providers/email';
 import Github from 'next-auth/providers/github';
 import Google from 'next-auth/providers/google';
 
-import { Config } from '../../config';
-import { PrismaService } from '../../prisma';
-import { SessionService } from '../../session';
+import {
+  Config,
+  MailService,
+  PrismaService,
+  SessionService,
+} from '../../fundamentals';
 import { FeatureType } from '../features';
-import { Quota_FreePlanV1 } from '../quota';
-import { MailService } from './mailer';
+import { Quota_FreePlanV1_1 } from '../quota';
 import {
   decode,
   encode,
@@ -50,7 +52,7 @@ export const NextAuthOptionsProvider: FactoryProvider<NextAuthOptions> = {
             activated: true,
             feature: {
               connect: {
-                feature_version: Quota_FreePlanV1,
+                feature_version: Quota_FreePlanV1_1,
               },
             },
           },
@@ -94,28 +96,28 @@ export const NextAuthOptionsProvider: FactoryProvider<NextAuthOptions> = {
       }
       return result;
     };
+
+    prismaAdapter.createVerificationToken = async data => {
+      await session.set(
+        `${data.identifier}:${data.token}`,
+        Date.now() + session.sessionTtl
+      );
+      return data;
+    };
+
+    prismaAdapter.useVerificationToken = async ({ identifier, token }) => {
+      const expires = await session.get(`${identifier}:${token}`);
+      if (expires) {
+        return { identifier, token, expires: new Date(expires) };
+      } else {
+        return null;
+      }
+    };
+
     const nextAuthOptions: NextAuthOptions = {
-      providers: [
-        // @ts-expect-error esm interop issue
-        Email.default({
-          server: {
-            host: config.auth.email.server,
-            port: config.auth.email.port,
-            auth: {
-              user: config.auth.email.login,
-              pass: config.auth.email.password,
-            },
-          },
-          from: config.auth.email.sender,
-          sendVerificationRequest: (params: SendVerificationRequestParams) =>
-            sendVerificationRequest(config, logger, mailer, session, params),
-        }),
-      ],
+      providers: [],
       adapter: prismaAdapter,
       debug: !config.node.prod,
-      session: {
-        strategy: 'database',
-      },
       logger: {
         debug(code, metadata) {
           logger.debug(`${code}: ${JSON.stringify(metadata)}`);
@@ -168,6 +170,16 @@ export const NextAuthOptionsProvider: FactoryProvider<NextAuthOptions> = {
       })
     );
 
+    if (config.mailer && mailer) {
+      nextAuthOptions.providers.push(
+        // @ts-expect-error esm interop issue
+        Email.default({
+          sendVerificationRequest: (params: SendVerificationRequestParams) =>
+            sendVerificationRequest(config, logger, mailer, session, params),
+        })
+      );
+    }
+
     if (config.auth.oauthProviders.github) {
       nextAuthOptions.providers.push(
         // @ts-expect-error esm interop issue
@@ -179,7 +191,7 @@ export const NextAuthOptionsProvider: FactoryProvider<NextAuthOptions> = {
       );
     }
 
-    if (config.auth.oauthProviders.google) {
+    if (config.auth.oauthProviders.google?.enabled) {
       nextAuthOptions.providers.push(
         // @ts-expect-error esm interop issue
         Google.default({
@@ -194,6 +206,11 @@ export const NextAuthOptionsProvider: FactoryProvider<NextAuthOptions> = {
       );
     }
 
+    if (nextAuthOptions.providers.length > 1) {
+      // not only credentials provider
+      nextAuthOptions.session = { strategy: 'database' };
+    }
+
     nextAuthOptions.jwt = {
       encode: async ({ token, maxAge }) =>
         encode(config, prisma, token, maxAge),

packages/backend/server/src/core/auth/next-auth.controller.ts

@@ -22,11 +22,14 @@ import { nanoid } from 'nanoid';
 import type { AuthAction, CookieOption, NextAuthOptions } from 'next-auth';
 import { AuthHandler } from 'next-auth/core';
 
-import { Config } from '../../config';
-import { metrics } from '../../metrics';
-import { PrismaService } from '../../prisma/service';
-import { SessionService } from '../../session';
-import { AuthThrottlerGuard, Throttle } from '../../throttler';
+import {
+  AuthThrottlerGuard,
+  Config,
+  metrics,
+  PrismaService,
+  SessionService,
+  Throttle,
+} from '../../fundamentals';
 import { NextAuthOptionsProvide } from './next-auth-options';
 import { AuthService } from './service';
 
@@ -186,15 +189,17 @@ export class NextAuthController {
     }
 
     let nextAuthTokenCookie: (CookieOption & { value: string }) | undefined;
-    const cookiePrefix = this.config.node.prod ? '__Secure-' : '';
-    const sessionCookieName = `${cookiePrefix}next-auth.session-token`;
+    const secureCookiePrefix = '__Secure-';
+    const sessionCookieName = `next-auth.session-token`;
     // next-auth credentials login only support JWT strategy
     // https://next-auth.js.org/configuration/providers/credentials
     // let's store the session token in the database
     if (
       credentialsSignIn &&
       (nextAuthTokenCookie = cookies?.find(
-        ({ name }) => name === sessionCookieName
+        ({ name }) =>
+          name === sessionCookieName ||
+          name === `${secureCookiePrefix}${sessionCookieName}`
       ))
     ) {
       const cookieExpires = new Date();

packages/backend/server/src/core/auth/resolver.ts

@@ -16,9 +16,12 @@ import {
 import type { Request } from 'express';
 import { nanoid } from 'nanoid';
 
-import { Config } from '../../config';
-import { SessionService } from '../../session';
-import { CloudThrottlerGuard, Throttle } from '../../throttler';
+import {
+  CloudThrottlerGuard,
+  Config,
+  SessionService,
+  Throttle,
+} from '../../fundamentals';
 import { UserType } from '../users';
 import { Auth, CurrentUser } from './guard';
 import { AuthService } from './service';
@@ -167,8 +170,13 @@ export class AuthResolver {
     @CurrentUser() user: UserType,
     @Args('token') token: string
   ) {
+    const key = await this.session.get(token);
+    if (!key) {
+      throw new ForbiddenException('Invalid token');
+    }
+
     // email has set token in `sendVerifyChangeEmail`
-    const [id, email] = (await this.session.get(token)).split(',');
+    const [id, email] = key.split(',');
     if (!id || id !== user.id || !email) {
       throw new ForbiddenException('Invalid token');
     }

packages/backend/server/src/core/auth/service.ts

@@ -11,11 +11,13 @@ import { Algorithm, sign, verify as jwtVerify } from '@node-rs/jsonwebtoken';
 import type { User } from '@prisma/client';
 import { nanoid } from 'nanoid';
 
-import { Config } from '../../config';
-import { PrismaService } from '../../prisma';
-import { verifyChallengeResponse } from '../../storage';
-import { Quota_FreePlanV1 } from '../quota';
-import { MailService } from './mailer';
+import {
+  Config,
+  MailService,
+  PrismaService,
+  verifyChallengeResponse,
+} from '../../fundamentals';
+import { Quota_FreePlanV1_1 } from '../quota';
 
 export type UserClaim = Pick<
   User,
@@ -134,7 +136,7 @@ export class AuthService {
     return (
       !!outcome.success &&
       // skip hostname check in dev mode
-      (this.config.affineEnv === 'dev' || outcome.hostname === this.config.host)
+      (this.config.node.dev || outcome.hostname === this.config.host)
     );
   }
 
@@ -192,13 +194,14 @@ export class AuthService {
         name,
         email,
         password: hashedPassword,
+        // TODO(@forehalo): handle in event system
         features: {
           create: {
             reason: 'created by api sign up',
             activated: true,
             feature: {
               connect: {
-                feature_version: Quota_FreePlanV1,
+                feature_version: Quota_FreePlanV1_1,
               },
             },
           },
@@ -228,7 +231,7 @@ export class AuthService {
             activated: true,
             feature: {
               connect: {
-                feature_version: Quota_FreePlanV1,
+                feature_version: Quota_FreePlanV1_1,
               },
             },
           },

packages/backend/server/src/core/auth/utils/jwt.ts

@@ -4,8 +4,7 @@ import { BadRequestException } from '@nestjs/common';
 import { Algorithm, sign, verify as jwtVerify } from '@node-rs/jsonwebtoken';
 import { JWT } from 'next-auth/jwt';
 
-import { Config } from '../../../config';
-import { PrismaService } from '../../../prisma';
+import { Config, PrismaService } from '../../../fundamentals';
 import { getUtcTimestamp, UserClaim } from '../service';
 
 export const jwtEncode = async (

packages/backend/server/src/core/auth/utils/send-mail.ts

@@ -2,9 +2,7 @@ import { Logger } from '@nestjs/common';
 import { nanoid } from 'nanoid';
 import type { SendVerificationRequestParams } from 'next-auth/providers/email';
 
-import { Config } from '../../../config';
-import { SessionService } from '../../../session';
-import { MailService } from '../mailer';
+import { Config, MailService, SessionService } from '../../../fundamentals';
 
 export async function sendVerificationRequest(
   config: Config,
@@ -13,7 +11,7 @@ export async function sendVerificationRequest(
   session: SessionService,
   params: SendVerificationRequestParams
 ) {
-  const { identifier, url, provider } = params;
+  const { identifier, url } = params;
   const urlWithToken = new URL(url);
   const callbackUrl = urlWithToken.searchParams.get('callbackUrl') || '';
   if (!callbackUrl) {
@@ -30,7 +28,6 @@ export async function sendVerificationRequest(
 
   const result = await mailer.sendSignInEmail(urlWithToken.toString(), {
     to: identifier,
-    from: provider.from,
   });
   logger.log(`send verification email success: ${result.accepted.join(', ')}`);
 

packages/backend/server/src/core/config.ts

@@ -0,0 +1,71 @@
+import { Module } from '@nestjs/common';
+import { Field, ObjectType, Query, registerEnumType } from '@nestjs/graphql';
+
+import { DeploymentType } from '../fundamentals';
+
+export enum ServerFeature {
+  Payment = 'payment',
+}
+
+registerEnumType(ServerFeature, {
+  name: 'ServerFeature',
+});
+
+registerEnumType(DeploymentType, {
+  name: 'ServerDeploymentType',
+});
+
+const ENABLED_FEATURES: ServerFeature[] = [];
+export function ADD_ENABLED_FEATURES(feature: ServerFeature) {
+  ENABLED_FEATURES.push(feature);
+}
+
+@ObjectType()
+export class ServerConfigType {
+  @Field({
+    description:
+      'server identical name could be shown as badge on user interface',
+  })
+  name!: string;
+
+  @Field({ description: 'server version' })
+  version!: string;
+
+  @Field({ description: 'server base url' })
+  baseUrl!: string;
+
+  @Field(() => DeploymentType, { description: 'server type' })
+  type!: DeploymentType;
+
+  /**
+   * @deprecated
+   */
+  @Field({ description: 'server flavor', deprecationReason: 'use `features`' })
+  flavor!: string;
+
+  @Field(() => [ServerFeature], { description: 'enabled server features' })
+  features!: ServerFeature[];
+}
+export class ServerConfigResolver {
+  @Query(() => ServerConfigType, {
+    description: 'server config',
+  })
+  serverConfig(): ServerConfigType {
+    return {
+      name: AFFiNE.serverName,
+      version: AFFiNE.version,
+      baseUrl: AFFiNE.baseUrl,
+      type: AFFiNE.type,
+      // BACKWARD COMPATIBILITY
+      // the old flavors contains `selfhosted` but it actually not flavor but deployment type
+      // this field should be removed after frontend feature flags implemented
+      flavor: AFFiNE.type,
+      features: ENABLED_FEATURES,
+    };
+  }
+}
+
+@Module({
+  providers: [ServerConfigResolver],
+})
+export class ServerConfigModule {}

packages/backend/server/src/core/doc/history.ts

@@ -3,10 +3,13 @@ import { isDeepStrictEqual } from 'node:util';
 import { Injectable, Logger } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 
-import { Config } from '../../config';
-import { type EventPayload, OnEvent } from '../../event';
-import { metrics } from '../../metrics';
-import { PrismaService } from '../../prisma';
+import {
+  Config,
+  type EventPayload,
+  metrics,
+  OnEvent,
+  PrismaService,
+} from '../../fundamentals';
 import { QuotaService } from '../quota';
 import { Permission } from '../workspaces/types';
 import { isEmptyBuffer } from './manager';

packages/backend/server/src/core/doc/manager.ts

@@ -4,24 +4,29 @@ import {
   OnModuleDestroy,
   OnModuleInit,
 } from '@nestjs/common';
+import { Cron, CronExpression } from '@nestjs/schedule';
 import { Snapshot, Update } from '@prisma/client';
 import { chunk } from 'lodash-es';
 import { defer, retry } from 'rxjs';
 import {
   applyUpdate,
-  decodeStateVector,
   Doc,
   encodeStateAsUpdate,
   encodeStateVector,
   transact,
 } from 'yjs';
 
-import { Cache } from '../../cache';
-import { Config } from '../../config';
-import { EventEmitter, type EventPayload, OnEvent } from '../../event';
-import { metrics } from '../../metrics/metrics';
-import { PrismaService } from '../../prisma';
-import { mergeUpdatesInApplyWay as jwstMergeUpdates } from '../../storage';
+import {
+  Cache,
+  CallTimer,
+  Config,
+  EventEmitter,
+  type EventPayload,
+  mergeUpdatesInApplyWay as jwstMergeUpdates,
+  metrics,
+  OnEvent,
+  PrismaService,
+} from '../../fundamentals';
 
 function compare(yBinary: Buffer, jwstBinary: Buffer, strict = false): boolean {
   if (yBinary.equals(jwstBinary)) {
@@ -40,36 +45,6 @@ function compare(yBinary: Buffer, jwstBinary: Buffer, strict = false): boolean {
   return compare(yBinary, yBinary2, true);
 }
 
-/**
- * Detect whether rhs state is newer than lhs state.
- *
- * How could we tell a state is newer:
- *
- *   i. if the state vector size is larger, it's newer
- *  ii. if the state vector size is same, compare each client's state
- */
-function isStateNewer(lhs: Buffer, rhs: Buffer): boolean {
-  const lhsVector = decodeStateVector(lhs);
-  const rhsVector = decodeStateVector(rhs);
-
-  if (lhsVector.size < rhsVector.size) {
-    return true;
-  }
-
-  for (const [client, state] of lhsVector) {
-    const rstate = rhsVector.get(client);
-    if (!rstate) {
-      return false;
-    }
-
-    if (state < rstate) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
 export function isEmptyBuffer(buf: Buffer): boolean {
   return (
     buf.length === 0 ||
@@ -79,6 +54,7 @@ export function isEmptyBuffer(buf: Buffer): boolean {
 }
 
 const MAX_SEQ_NUM = 0x3fffffff; // u31
+const UPDATES_QUEUE_CACHE_KEY = 'doc:manager:updates';
 
 /**
  * Since we can't directly save all client updates into database, in which way the database will overload,
@@ -113,6 +89,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     this.destroy();
   }
 
+  @CallTimer('doc', 'yjs_recover_updates_to_doc')
   private recoverDoc(...updates: Buffer[]): Promise<Doc> {
     const doc = new Doc();
     const chunks = chunk(updates, 10);
@@ -148,11 +125,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     const doc = await this.recoverDoc(...updates);
 
     // test jwst codec
-    if (
-      this.config.affine.canary &&
-      this.config.doc.manager.experimentalMergeWithJwstCodec &&
-      updates.length < 100 /* avoid overloading */
-    ) {
+    if (this.config.doc.manager.experimentalMergeWithYOcto) {
       metrics.jwst.counter('codec_merge_counter').add(1);
       const yjsResult = Buffer.from(encodeStateAsUpdate(doc));
       let log = false;
@@ -203,7 +176,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     }, this.config.doc.manager.updatePollInterval);
 
     this.logger.log('Automation started');
-    if (this.config.doc.manager.experimentalMergeWithJwstCodec) {
+    if (this.config.doc.manager.experimentalMergeWithYOcto) {
       this.logger.warn(
         'Experimental feature enabled: merge updates with jwst codec is enabled'
       );
@@ -376,7 +349,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     const updates = await this.getUpdates(workspaceId, guid);
 
     if (updates.length) {
-      const doc = await this.squash(updates, snapshot);
+      const doc = await this.squash(snapshot, updates);
       return Buffer.from(encodeStateVector(doc));
     }
 
@@ -409,7 +382,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
       // take it ease, we don't want to overload db and or cpu
       // if we limit the taken number here,
       // user will never see the latest doc if there are too many updates pending to be merged.
-      take: 100,
+      take: this.config.doc.manager.maxUpdatesPullCount,
     });
 
     // perf(memory): avoid sorting in db
@@ -457,80 +430,92 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     });
   }
 
+  /**
+   * @returns whether the snapshot is updated to the latest, `undefined` means the doc to be upserted is outdated.
+   */
+  @CallTimer('doc', 'upsert')
   private async upsert(
     workspaceId: string,
     guid: string,
     doc: Doc,
     // we always delay the snapshot update to avoid db overload,
-    // so the value of `updatedAt` will not be accurate to user's real action time
+    // so the value of auto updated `updatedAt` by db will never be accurate to user's real action time
     updatedAt: Date,
-    initialSeq?: number
+    seq: number
   ) {
-    return this.lockSnapshotForUpsert(workspaceId, guid, async () => {
-      const blob = Buffer.from(encodeStateAsUpdate(doc));
+    const blob = Buffer.from(encodeStateAsUpdate(doc));
 
-      if (isEmptyBuffer(blob)) {
-        return false;
+    if (isEmptyBuffer(blob)) {
+      return undefined;
+    }
+
+    const state = Buffer.from(encodeStateVector(doc));
+
+    // CONCERNS:
+    //   i. Because we save the real user's last seen action time as `updatedAt`,
+    //      it's possible to simply compare the `updatedAt` to determine if the snapshot is older than the one we are going to save.
+    //
+    //  ii. Prisma doesn't support `upsert` with additional `where` condition along side unique constraint.
+    //      In our case, we need to manually check the `updatedAt` to avoid overriding the newer snapshot.
+    //      where: { id_workspaceId: {}, updatedAt: { lt: updatedAt } }
+    //                                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    //
+    // iii. Only set the seq number when creating the snapshot.
+    //      For updating scenario, the seq number will be updated when updates pushed to db.
+    try {
+      const result: { updatedAt: Date }[] = await this.db.$queryRaw`
+        INSERT INTO "snapshots" ("workspace_id", "guid", "blob", "state", "seq", "created_at", "updated_at")
+        VALUES (${workspaceId}, ${guid}, ${blob}, ${state}, ${seq}, DEFAULT, ${updatedAt})
+        ON CONFLICT ("workspace_id", "guid")
+        DO UPDATE SET "blob" = ${blob}, "state" = ${state}, "updated_at" = ${updatedAt}, "seq" = ${seq}
+        WHERE "snapshots"."workspace_id" = ${workspaceId} AND "snapshots"."guid" = ${guid} AND "snapshots"."updated_at" <= ${updatedAt}
+        RETURNING "snapshots"."workspace_id" as "workspaceId", "snapshots"."guid" as "id", "snapshots"."updated_at" as "updatedAt"
+      `;
+
+      // const result = await this.db.snapshot.upsert({
+      //   select: {
+      //     updatedAt: true,
+      //     seq: true,
+      //   },
+      //   where: {
+      //     id_workspaceId: {
+      //       workspaceId,
+      //       id: guid,
+      //     },
+      //     ⬇️ NOT SUPPORTED BY PRISMA YET
+      //     updatedAt: {
+      //       lt: updatedAt,
+      //     },
+      //   },
+      //   update: {
+      //     blob,
+      //     state,
+      //     updatedAt,
+      //   },
+      //   create: {
+      //     workspaceId,
+      //     id: guid,
+      //     blob,
+      //     state,
+      //     updatedAt,
+      //     seq,
+      //   },
+      // });
+
+      // if the condition `snapshot.updatedAt > updatedAt` is true, by which means the snapshot has already been updated by other process,
+      // the updates has been applied to current `doc` must have been seen by the other process as well.
+      // The `updatedSnapshot` will be `undefined` in this case.
+      const updatedSnapshot = result.at(0);
+
+      if (!updatedSnapshot) {
+        return undefined;
       }
 
-      const state = Buffer.from(encodeStateVector(doc));
-
-      return await this.db.$transaction(async db => {
-        const snapshot = await db.snapshot.findUnique({
-          where: {
-            id_workspaceId: {
-              id: guid,
-              workspaceId,
-            },
-          },
-        });
-
-        // update
-        if (snapshot) {
-          // only update if state is newer
-          if (isStateNewer(snapshot.state ?? Buffer.from([0]), state)) {
-            await db.snapshot.update({
-              select: {
-                seq: true,
-              },
-              where: {
-                id_workspaceId: {
-                  workspaceId,
-                  id: guid,
-                },
-              },
-              data: {
-                blob,
-                state,
-                updatedAt,
-              },
-            });
-
-            return true;
-          } else {
-            return false;
-          }
-        } else {
-          // create
-          await db.snapshot.create({
-            select: {
-              seq: true,
-            },
-            data: {
-              id: guid,
-              workspaceId,
-              blob,
-              state,
-              seq: initialSeq,
-              createdAt: updatedAt,
-              updatedAt,
-            },
-          });
-
-          return true;
-        }
-      });
-    });
+      return true;
+    } catch (e) {
+      this.logger.error('Failed to upsert snapshot', e);
+      return false;
+    }
   }
 
   private async _get(
@@ -542,7 +527,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
 
     if (updates.length) {
       return {
-        doc: await this.squash(updates, snapshot),
+        doc: await this.squash(snapshot, updates),
       };
     }
 
@@ -553,17 +538,17 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
    * Squash updates into a single update and save it as snapshot,
    * and delete the updates records at the same time.
    */
-  private async squash(updates: Update[], snapshot: Snapshot | null) {
+  @CallTimer('doc', 'squash')
+  private async squash(snapshot: Snapshot | null, updates: Update[]) {
     if (!updates.length) {
       throw new Error('No updates to squash');
     }
-    const first = updates[0];
-    const last = updates[updates.length - 1];
 
-    const { id, workspaceId } = first;
+    const last = updates[updates.length - 1];
+    const { id, workspaceId } = last;
 
     const doc = await this.applyUpdates(
-      first.id,
+      id,
       snapshot ? snapshot.blob : Buffer.from([0, 0]),
       ...updates.map(u => u.blob)
     );
@@ -594,19 +579,24 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
       );
     }
 
-    // always delete updates
-    // the upsert will return false if the state is not newer, so we don't need to worry about it
-    const { count } = await this.db.update.deleteMany({
-      where: {
-        id,
-        workspaceId,
-        seq: {
-          in: updates.map(u => u.seq),
+    // we will keep the updates only if the upsert failed on unknown reason
+    // `done === undefined` means the updates is outdated(have already been merged by other process), safe to be deleted
+    // `done === true` means the upsert is successful, safe to be deleted
+    if (done !== false) {
+      // always delete updates
+      // the upsert will return false if the state is not newer, so we don't need to worry about it
+      const { count } = await this.db.update.deleteMany({
+        where: {
+          id,
+          workspaceId,
+          seq: {
+            in: updates.map(u => u.seq),
+          },
         },
-      },
-    });
+      });
 
-    await this.updateCachedUpdatesCount(workspaceId, id, -count);
+      await this.updateCachedUpdatesCount(workspaceId, id, -count);
+    }
 
     return doc;
   }
@@ -663,26 +653,44 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     count: number
   ) {
     const result = await this.cache.mapIncrease(
-      `doc:manager:updates`,
+      UPDATES_QUEUE_CACHE_KEY,
       `${workspaceId}::${guid}`,
       count
     );
 
     if (result <= 0) {
       await this.cache.mapDelete(
-        `doc:manager:updates`,
+        UPDATES_QUEUE_CACHE_KEY,
         `${workspaceId}::${guid}`
       );
     }
   }
 
   private async getAutoSquashCandidateFromCache() {
-    const key = await this.cache.mapRandomKey('doc:manager:updates');
+    const key = await this.cache.mapRandomKey(UPDATES_QUEUE_CACHE_KEY);
 
     if (key) {
-      const count = await this.cache.mapGet<number>('doc:manager:updates', key);
-      if (typeof count === 'number' && count > 0) {
+      const cachedCount = await this.cache.mapIncrease(
+        UPDATES_QUEUE_CACHE_KEY,
+        key,
+        0
+      );
+
+      if (cachedCount > 0) {
         const [workspaceId, id] = key.split('::');
+        const count = await this.db.update.count({
+          where: {
+            workspaceId,
+            id,
+          },
+        });
+
+        // FIXME(@forehalo): somehow the update count in cache is not accurate
+        if (count === 0) {
+          await this.cache.mapDelete(UPDATES_QUEUE_CACHE_KEY, key);
+
+          return null;
+        }
         return { id, workspaceId };
       }
     }
@@ -690,22 +698,38 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     return null;
   }
 
-  private async doWithLock<T>(lock: string, job: () => Promise<T>) {
+  private async doWithLock<T>(
+    lockScope: string,
+    lockResource: string,
+    job: () => Promise<T>
+  ) {
+    const lock = `lock:${lockScope}:${lockResource}`;
     const acquired = await this.cache.setnx(lock, 1, {
       ttl: 60 * 1000,
     });
+    metrics.doc.counter('lock').add(1, { scope: lockScope });
 
     if (!acquired) {
+      metrics.doc.counter('lock_failed').add(1, { scope: lockScope });
       return;
     }
+    metrics.doc.counter('lock_required').add(1, { scope: lockScope });
 
     try {
       return await job();
     } finally {
-      await this.cache.delete(lock).catch(e => {
-        // safe, the lock will be expired when ttl ends
-        this.logger.error(`Failed to release lock ${lock}`, e);
-      });
+      await this.cache
+        .delete(lock)
+        .then(() => {
+          metrics.doc.counter('lock_released').add(1, { scope: lockScope });
+        })
+        .catch(e => {
+          metrics.doc
+            .counter('lock_release_failed')
+            .add(1, { scope: lockScope });
+          // safe, the lock will be expired when ttl ends
+          this.logger.error(`Failed to release lock ${lock}`, e);
+        });
     }
   }
 
@@ -715,19 +739,16 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     job: () => Promise<T>
   ) {
     return this.doWithLock(
-      `doc:manager:updates-lock:${workspaceId}::${guid}`,
+      'doc:manager:updates',
+      `${workspaceId}::${guid}`,
       job
     );
   }
 
-  async lockSnapshotForUpsert<T>(
-    workspaceId: string,
-    guid: string,
-    job: () => Promise<T>
-  ) {
-    return this.doWithLock(
-      `doc:manager:snapshot-lock:${workspaceId}::${guid}`,
-      job
-    );
+  @Cron(CronExpression.EVERY_MINUTE)
+  async reportUpdatesQueueCount() {
+    metrics.doc
+      .gauge('updates_queue_count')
+      .record(await this.db.update.count());
   }
 }

packages/backend/server/src/core/features/feature.ts

@@ -1,4 +1,5 @@
-import { PrismaService } from '../../prisma';
+import { PrismaClient } from '@prisma/client';
+
 import { Feature, FeatureSchema, FeatureType } from './types';
 
 class FeatureConfig {
@@ -42,9 +43,22 @@ export class EarlyAccessFeatureConfig extends FeatureConfig {
   }
 }
 
+export class UnlimitedWorkspaceFeatureConfig extends FeatureConfig {
+  override config!: Feature & { feature: FeatureType.UnlimitedWorkspace };
+
+  constructor(data: any) {
+    super(data);
+
+    if (this.config.feature !== FeatureType.UnlimitedWorkspace) {
+      throw new Error('Invalid feature config: type is not UnlimitedWorkspace');
+    }
+  }
+}
+
 const FeatureConfigMap = {
   [FeatureType.Copilot]: CopilotFeatureConfig,
   [FeatureType.EarlyAccess]: EarlyAccessFeatureConfig,
+  [FeatureType.UnlimitedWorkspace]: UnlimitedWorkspaceFeatureConfig,
 };
 
 export type FeatureConfigType<F extends FeatureType> = InstanceType<
@@ -53,7 +67,7 @@ export type FeatureConfigType<F extends FeatureType> = InstanceType<
 
 const FeatureCache = new Map<number, FeatureConfigType<FeatureType>>();
 
-export async function getFeature(prisma: PrismaService, featureId: number) {
+export async function getFeature(prisma: PrismaClient, featureId: number) {
   const cachedQuota = FeatureCache.get(featureId);
 
   if (cachedQuota) {

packages/backend/server/src/core/features/index.ts

@@ -1,6 +1,5 @@
 import { Module } from '@nestjs/common';
 
-import { PrismaService } from '../../prisma';
 import { FeatureManagementService } from './management';
 import { FeatureService } from './service';
 
@@ -18,4 +17,4 @@ export class FeatureModule {}
 
 export { type CommonFeature, commonFeatureSchema } from './types';
 export { FeatureKind, Features, FeatureType } from './types';
-export { FeatureManagementService, FeatureService, PrismaService };
+export { FeatureManagementService, FeatureService };

packages/backend/server/src/core/features/management.ts

@@ -1,7 +1,6 @@
 import { Injectable, Logger } from '@nestjs/common';
 
-import { Config } from '../../config';
-import { PrismaService } from '../../prisma';
+import { Config, PrismaService } from '../../fundamentals';
 import { FeatureService } from './service';
 import { FeatureType } from './types';
 
@@ -48,22 +47,26 @@ export class FeatureManagementService {
     return this.feature.listFeatureUsers(FeatureType.EarlyAccess);
   }
 
+  async isEarlyAccessUser(email: string) {
+    const user = await this.prisma.user.findFirst({
+      where: {
+        email,
+      },
+    });
+    if (user) {
+      const canEarlyAccess = await this.feature
+        .hasUserFeature(user.id, FeatureType.EarlyAccess)
+        .catch(() => false);
+
+      return canEarlyAccess;
+    }
+    return false;
+  }
+
   /// check early access by email
   async canEarlyAccess(email: string) {
     if (this.config.featureFlags.earlyAccessPreview && !this.isStaff(email)) {
-      const user = await this.prisma.user.findFirst({
-        where: {
-          email,
-        },
-      });
-      if (user) {
-        const canEarlyAccess = await this.feature
-          .hasUserFeature(user.id, FeatureType.EarlyAccess)
-          .catch(() => false);
-
-        return canEarlyAccess;
-      }
-      return false;
+      return this.isEarlyAccessUser(email);
     } else {
       return true;
     }

packages/backend/server/src/core/features/service.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@nestjs/common';
 
-import { PrismaService } from '../../prisma';
+import { PrismaService } from '../../fundamentals';
 import { UserType } from '../users/types';
 import { WorkspaceType } from '../workspaces/types';
 import { FeatureConfigType, getFeature } from './feature';

packages/backend/server/src/core/features/types/common.ts

@@ -3,6 +3,7 @@ import { registerEnumType } from '@nestjs/graphql';
 export enum FeatureType {
   Copilot = 'copilot',
   EarlyAccess = 'early_access',
+  UnlimitedWorkspace = 'unlimited_workspace',
 }
 
 registerEnumType(FeatureType, {

packages/backend/server/src/core/features/types/early-access.ts

@@ -4,5 +4,8 @@ import { FeatureType } from './common';
 
 export const featureEarlyAccess = z.object({
   feature: z.literal(FeatureType.EarlyAccess),
-  configs: z.object({}),
+  configs: z.object({
+    // field polyfill, make it optional in the future
+    whitelist: z.string().array(),
+  }),
 });

packages/backend/server/src/core/features/types/index.ts

@@ -3,6 +3,7 @@ import { z } from 'zod';
 import { FeatureType } from './common';
 import { featureCopilot } from './copilot';
 import { featureEarlyAccess } from './early-access';
+import { featureUnlimitedWorkspace } from './unlimited-workspace';
 
 /// ======== common schema ========
 
@@ -41,6 +42,14 @@ export const Features: Feature[] = [
     feature: FeatureType.EarlyAccess,
     type: FeatureKind.Feature,
     version: 2,
+    configs: {
+      whitelist: [],
+    },
+  },
+  {
+    feature: FeatureType.UnlimitedWorkspace,
+    type: FeatureKind.Feature,
+    version: 1,
     configs: {},
   },
 ];
@@ -51,7 +60,13 @@ export const FeatureSchema = commonFeatureSchema
   .extend({
     type: z.literal(FeatureKind.Feature),
   })
-  .and(z.discriminatedUnion('feature', [featureCopilot, featureEarlyAccess]));
+  .and(
+    z.discriminatedUnion('feature', [
+      featureCopilot,
+      featureEarlyAccess,
+      featureUnlimitedWorkspace,
+    ])
+  );
 
 export type Feature = z.infer<typeof FeatureSchema>;
 

packages/backend/server/src/core/features/types/unlimited-workspace.ts

@@ -0,0 +1,8 @@
+import { z } from 'zod';
+
+import { FeatureType } from './common';
+
+export const featureUnlimitedWorkspace = z.object({
+  feature: z.literal(FeatureType.UnlimitedWorkspace),
+  configs: z.object({}),
+});

packages/backend/server/src/core/quota/index.ts

@@ -1,5 +1,6 @@
 import { Module } from '@nestjs/common';
 
+import { FeatureModule } from '../features';
 import { StorageModule } from '../storage';
 import { PermissionService } from '../workspaces/permission';
 import { QuotaService } from './service';
@@ -12,12 +13,12 @@ import { QuotaManagementService } from './storage';
  * - quota statistics
  */
 @Module({
-  imports: [StorageModule],
+  imports: [FeatureModule, StorageModule],
   providers: [PermissionService, QuotaService, QuotaManagementService],
   exports: [QuotaService, QuotaManagementService],
 })
 export class QuotaModule {}
 
 export { QuotaManagementService, QuotaService };
-export { Quota_FreePlanV1, Quota_ProPlanV1, Quotas } from './schema';
-export { QuotaType } from './types';
+export { Quota_FreePlanV1_1, Quota_ProPlanV1, Quotas } from './schema';
+export { QuotaQueryType, QuotaType } from './types';

packages/backend/server/src/core/quota/quota.ts

@@ -1,4 +1,4 @@
-import { PrismaService } from '../../prisma';
+import { PrismaService } from '../../fundamentals';
 import { formatDate, formatSize, Quota, QuotaSchema } from './types';
 
 const QuotaCache = new Map<number, QuotaConfig>();
@@ -44,6 +44,10 @@ export class QuotaConfig {
     }
   }
 
+  get version() {
+    return this.config.version;
+  }
+
   /// feature name of quota
   get name() {
     return this.config.feature;
@@ -53,6 +57,12 @@ export class QuotaConfig {
     return this.config.configs.blobLimit;
   }
 
+  get businessBlobLimit() {
+    return (
+      this.config.configs.businessBlobLimit || this.config.configs.blobLimit
+    );
+  }
+
   get storageQuota() {
     return this.config.configs.storageQuota;
   }

packages/backend/server/src/core/quota/schema.ts

@@ -0,0 +1,106 @@
+import { FeatureKind } from '../features';
+import { OneDay, OneGB, OneMB } from './constant';
+import { Quota, QuotaType } from './types';
+
+export const Quotas: Quota[] = [
+  {
+    feature: QuotaType.FreePlanV1,
+    type: FeatureKind.Quota,
+    version: 1,
+    configs: {
+      // quota name
+      name: 'Free',
+      // single blob limit 10MB
+      blobLimit: 10 * OneMB,
+      // total blob limit 10GB
+      storageQuota: 10 * OneGB,
+      // history period of validity 7 days
+      historyPeriod: 7 * OneDay,
+      // member limit 3
+      memberLimit: 3,
+    },
+  },
+  {
+    feature: QuotaType.ProPlanV1,
+    type: FeatureKind.Quota,
+    version: 1,
+    configs: {
+      // quota name
+      name: 'Pro',
+      // single blob limit 100MB
+      blobLimit: 100 * OneMB,
+      // total blob limit 100GB
+      storageQuota: 100 * OneGB,
+      // history period of validity 30 days
+      historyPeriod: 30 * OneDay,
+      // member limit 10
+      memberLimit: 10,
+    },
+  },
+  {
+    feature: QuotaType.RestrictedPlanV1,
+    type: FeatureKind.Quota,
+    version: 1,
+    configs: {
+      // quota name
+      name: 'Restricted',
+      // single blob limit 10MB
+      blobLimit: OneMB,
+      // total blob limit 1GB
+      storageQuota: 10 * OneMB,
+      // history period of validity 30 days
+      historyPeriod: 30 * OneDay,
+      // member limit 10
+      memberLimit: 10,
+    },
+  },
+  {
+    feature: QuotaType.FreePlanV1,
+    type: FeatureKind.Quota,
+    version: 2,
+    configs: {
+      // quota name
+      name: 'Free',
+      // single blob limit 10MB
+      blobLimit: 100 * OneMB,
+      // total blob limit 10GB
+      storageQuota: 10 * OneGB,
+      // history period of validity 7 days
+      historyPeriod: 7 * OneDay,
+      // member limit 3
+      memberLimit: 3,
+    },
+  },
+  {
+    feature: QuotaType.FreePlanV1,
+    type: FeatureKind.Quota,
+    version: 3,
+    configs: {
+      // quota name
+      name: 'Free',
+      // single blob limit 10MB
+      blobLimit: 10 * OneMB,
+      // server limit will larger then client to handle a edge case:
+      // when a user downgrades from pro to free, he can still continue
+      // to upload previously added files that exceed the free limit
+      // NOTE: this is a product decision, may change in future
+      businessBlobLimit: 100 * OneMB,
+      // total blob limit 10GB
+      storageQuota: 10 * OneGB,
+      // history period of validity 7 days
+      historyPeriod: 7 * OneDay,
+      // member limit 3
+      memberLimit: 3,
+    },
+  },
+];
+
+export const Quota_FreePlanV1_1 = {
+  feature: Quotas[4].feature,
+  version: Quotas[4].version,
+};
+
+export const Quota_ProPlanV1 = {
+  feature: Quotas[1].feature,
+  version: Quotas[1].version,
+};

packages/backend/server/src/core/quota/service.ts

@@ -1,10 +1,12 @@
 import { Injectable } from '@nestjs/common';
 
-import { PrismaService } from '../../prisma';
+import { type EventPayload, OnEvent, PrismaService } from '../../fundamentals';
 import { FeatureKind } from '../features';
 import { QuotaConfig } from './quota';
 import { QuotaType } from './types';
 
+type Transaction = Parameters<Parameters<PrismaService['$transaction']>[0]>[0];
+
 @Injectable()
 export class QuotaService {
   constructor(private readonly prisma: PrismaService) {}
@@ -83,6 +85,13 @@ export class QuotaService {
     expiredAt?: Date
   ) {
     await this.prisma.$transaction(async tx => {
+      const hasSameActivatedQuota = await this.hasQuota(userId, quota, tx);
+
+      if (hasSameActivatedQuota) {
+        // don't need to switch
+        return;
+      }
+
       const latestPlanVersion = await tx.features.aggregate({
         where: {
           feature: quota,
@@ -130,8 +139,10 @@ export class QuotaService {
     });
   }
 
-  async hasQuota(userId: string, quota: QuotaType) {
-    return this.prisma.userFeatures
+  async hasQuota(userId: string, quota: QuotaType, transaction?: Transaction) {
+    const executor = transaction ?? this.prisma;
+
+    return executor.userFeatures
       .count({
         where: {
           userId,
@@ -144,4 +155,26 @@ export class QuotaService {
       })
       .then(count => count > 0);
   }
+
+  @OnEvent('user.subscription.activated')
+  async onSubscriptionUpdated({
+    userId,
+  }: EventPayload<'user.subscription.activated'>) {
+    await this.switchUserQuota(
+      userId,
+      QuotaType.ProPlanV1,
+      'subscription activated'
+    );
+  }
+
+  @OnEvent('user.subscription.canceled')
+  async onSubscriptionCanceled(
+    userId: EventPayload<'user.subscription.canceled'>
+  ) {
+    await this.switchUserQuota(
+      userId,
+      QuotaType.FreePlanV1,
+      'subscription canceled'
+    );
+  }
 }

packages/backend/server/src/core/quota/storage.ts

@@ -0,0 +1,112 @@
+import { Injectable, NotFoundException } from '@nestjs/common';
+
+import { FeatureService, FeatureType } from '../features';
+import { WorkspaceBlobStorage } from '../storage';
+import { PermissionService } from '../workspaces/permission';
+import { OneGB } from './constant';
+import { QuotaService } from './service';
+import { formatSize, QuotaQueryType } from './types';
+
+type QuotaBusinessType = QuotaQueryType & { businessBlobLimit: number };
+
+@Injectable()
+export class QuotaManagementService {
+  constructor(
+    private readonly feature: FeatureService,
+    private readonly quota: QuotaService,
+    private readonly permissions: PermissionService,
+    private readonly storage: WorkspaceBlobStorage
+  ) {}
+
+  async getUserQuota(userId: string) {
+    const quota = await this.quota.getUserQuota(userId);
+
+    return {
+      name: quota.feature.name,
+      reason: quota.reason,
+      createAt: quota.createdAt,
+      expiredAt: quota.expiredAt,
+      blobLimit: quota.feature.blobLimit,
+      businessBlobLimit: quota.feature.businessBlobLimit,
+      storageQuota: quota.feature.storageQuota,
+      historyPeriod: quota.feature.historyPeriod,
+      memberLimit: quota.feature.memberLimit,
+    };
+  }
+
+  // TODO: lazy calc, need to be optimized with cache
+  async getUserUsage(userId: string) {
+    const workspaces = await this.permissions.getOwnedWorkspaces(userId);
+
+    const sizes = await Promise.all(
+      workspaces.map(workspace => this.storage.totalSize(workspace))
+    );
+
+    return sizes.reduce((total, size) => total + size, 0);
+  }
+
+  // get workspace's owner quota and total size of used
+  // quota was apply to owner's account
+  async getWorkspaceUsage(workspaceId: string): Promise<QuotaBusinessType> {
+    const { user: owner } =
+      await this.permissions.getWorkspaceOwner(workspaceId);
+    if (!owner) throw new NotFoundException('Workspace owner not found');
+    const {
+      feature: {
+        name,
+        blobLimit,
+        businessBlobLimit,
+        historyPeriod,
+        memberLimit,
+        storageQuota,
+        humanReadable,
+      },
+    } = await this.quota.getUserQuota(owner.id);
+    // get all workspaces size of owner used
+    const usedSize = await this.getUserUsage(owner.id);
+
+    const quota = {
+      name,
+      blobLimit,
+      businessBlobLimit,
+      historyPeriod,
+      memberLimit,
+      storageQuota,
+      humanReadable,
+      usedSize,
+    };
+
+    // relax restrictions if workspace has unlimited feature
+    // todo(@darkskygit): need a mechanism to allow feature as a middleware to edit quota
+    const unlimited = await this.feature.hasWorkspaceFeature(
+      workspaceId,
+      FeatureType.UnlimitedWorkspace
+    );
+    if (unlimited) {
+      return this.mergeUnlimitedQuota(quota);
+    }
+
+    return quota;
+  }
+
+  private mergeUnlimitedQuota(orig: QuotaBusinessType) {
+    return {
+      ...orig,
+      storageQuota: 1000 * OneGB,
+      memberLimit: 1000,
+      humanReadable: {
+        ...orig.humanReadable,
+        name: 'Unlimited',
+        storageQuota: formatSize(1000 * OneGB),
+        memberLimit: '1000',
+      },
+    };
+  }
+
+  async checkBlobQuota(workspaceId: string, size: number) {
+    const { storageQuota, usedSize } =
+      await this.getWorkspaceUsage(workspaceId);
+
+    return storageQuota - (size + usedSize);
+  }
+}

packages/backend/server/src/core/quota/types.ts

@@ -0,0 +1,110 @@
+import { Field, ObjectType } from '@nestjs/graphql';
+import { SafeIntResolver } from 'graphql-scalars';
+import { z } from 'zod';
+
+import { commonFeatureSchema, FeatureKind } from '../features';
+import { ByteUnit, OneDay, OneKB } from './constant';
+
+/// ======== quota define ========
+
+/**
+ * naming rule:
+ * we append Vx to the end of the feature name to indicate the version of the feature
+ * x is a number, start from 1, this number will be change only at the time we change the schema of config
+ * for example, we change the value of `blobLimit` from 10MB to 100MB, then we will only change `version` field from 1 to 2
+ * but if we remove the `blobLimit` field or rename it, then we will change the Vx to Vx+1
+ */
+export enum QuotaType {
+  FreePlanV1 = 'free_plan_v1',
+  ProPlanV1 = 'pro_plan_v1',
+  // only for test, smaller quota
+  RestrictedPlanV1 = 'restricted_plan_v1',
+}
+
+const quotaPlan = z.object({
+  feature: z.enum([
+    QuotaType.FreePlanV1,
+    QuotaType.ProPlanV1,
+    QuotaType.RestrictedPlanV1,
+  ]),
+  configs: z.object({
+    name: z.string(),
+    blobLimit: z.number().positive().int(),
+    storageQuota: z.number().positive().int(),
+    historyPeriod: z.number().positive().int(),
+    memberLimit: z.number().positive().int(),
+    businessBlobLimit: z.number().positive().int().nullish(),
+  }),
+});
+
+/// ======== schema infer ========
+
+export const QuotaSchema = commonFeatureSchema
+  .extend({
+    type: z.literal(FeatureKind.Quota),
+  })
+  .and(z.discriminatedUnion('feature', [quotaPlan]));
+
+export type Quota = z.infer<typeof QuotaSchema>;
+
+/// ======== query types ========
+
+@ObjectType()
+export class HumanReadableQuotaType {
+  @Field(() => String)
+  name!: string;
+
+  @Field(() => String)
+  blobLimit!: string;
+
+  @Field(() => String)
+  storageQuota!: string;
+
+  @Field(() => String)
+  historyPeriod!: string;
+
+  @Field(() => String)
+  memberLimit!: string;
+}
+
+@ObjectType()
+export class QuotaQueryType {
+  @Field(() => String)
+  name!: string;
+
+  @Field(() => SafeIntResolver)
+  blobLimit!: number;
+
+  @Field(() => SafeIntResolver)
+  historyPeriod!: number;
+
+  @Field(() => SafeIntResolver)
+  memberLimit!: number;
+
+  @Field(() => SafeIntResolver)
+  storageQuota!: number;
+
+  @Field(() => HumanReadableQuotaType)
+  humanReadable!: HumanReadableQuotaType;
+
+  @Field(() => SafeIntResolver)
+  usedSize!: number;
+}
+
+/// ======== utils ========
+
+export function formatSize(bytes: number, decimals: number = 2): string {
+  if (bytes === 0) return '0 B';
+
+  const dm = decimals < 0 ? 0 : decimals;
+
+  const i = Math.floor(Math.log(bytes) / Math.log(OneKB));
+
+  return (
+    parseFloat((bytes / Math.pow(OneKB, i)).toFixed(dm)) + ' ' + ByteUnit[i]
+  );
+}
+
+export function formatDate(ms: number): string {
+  return `${(ms / OneDay).toFixed(0)} days`;
+}

packages/backend/server/src/core/storage/wrappers/avatar.ts

@@ -1,18 +1,17 @@
 import { Injectable } from '@nestjs/common';
 
-import { AFFiNEStorageConfig, Config } from '../../../config';
-import { type EventPayload, OnEvent } from '../../../event';
-import {
+import type {
   BlobInputType,
-  createStorageProvider,
+  EventPayload,
   PutObjectMetadata,
   StorageProvider,
-} from '../providers';
+} from '../../../fundamentals';
+import { Config, createStorageProvider, OnEvent } from '../../../fundamentals';
 
 @Injectable()
 export class AvatarStorage {
   public readonly provider: StorageProvider;
-  private readonly storageConfig: AFFiNEStorageConfig['storages']['avatar'];
+  private readonly storageConfig: Config['storage']['storages']['avatar'];
 
   constructor(private readonly config: Config) {
     this.provider = createStorageProvider(this.config.storage, 'avatar');

packages/backend/server/src/core/storage/wrappers/blob.ts

@@ -1,27 +1,21 @@
-import { Readable } from 'node:stream';
+import { Injectable } from '@nestjs/common';
 
-import type { Storage } from '@affine/storage';
-import { Injectable, OnModuleInit } from '@nestjs/common';
-
-import { Config } from '../../../config';
-import { EventEmitter, type EventPayload, OnEvent } from '../../../event';
-import { OctoBaseStorageModule } from '../../../storage';
-import {
+import type {
   BlobInputType,
-  createStorageProvider,
+  EventPayload,
   StorageProvider,
-} from '../providers';
-import { toBuffer } from '../providers/utils';
+} from '../../../fundamentals';
+import {
+  Config,
+  createStorageProvider,
+  EventEmitter,
+  OnEvent,
+} from '../../../fundamentals';
 
 @Injectable()
-export class WorkspaceBlobStorage implements OnModuleInit {
+export class WorkspaceBlobStorage {
   public readonly provider: StorageProvider;
 
-  /**
-   * @deprecated for backwards compatibility, need to be removed in next stable release
-   */
-  private octobase: Storage | null = null;
-
   constructor(
     private readonly event: EventEmitter,
     private readonly config: Config
@@ -29,42 +23,12 @@ export class WorkspaceBlobStorage implements OnModuleInit {
     this.provider = createStorageProvider(this.config.storage, 'blob');
   }
 
-  async onModuleInit() {
-    if (!this.config.node.test) {
-      this.octobase = await OctoBaseStorageModule.Storage.connect(
-        this.config.db.url
-      );
-    }
-  }
-
   async put(workspaceId: string, key: string, blob: BlobInputType) {
-    const buf = await toBuffer(blob);
-    await this.provider.put(`${workspaceId}/${key}`, buf);
-    if (this.octobase) {
-      await this.octobase.uploadBlob(workspaceId, buf);
-    }
+    await this.provider.put(`${workspaceId}/${key}`, blob);
   }
 
   async get(workspaceId: string, key: string) {
-    const result = await this.provider.get(`${workspaceId}/${key}`);
-    if (!result.body && this.octobase) {
-      const blob = await this.octobase.getBlob(workspaceId, key);
-
-      if (!blob) {
-        return result;
-      }
-
-      return {
-        body: Readable.from(blob.data),
-        metadata: {
-          contentType: blob.contentType,
-          contentLength: blob.size,
-          lastModified: new Date(blob.lastModified),
-        },
-      };
-    }
-
-    return result;
+    return this.provider.get(`${workspaceId}/${key}`);
   }
 
   async list(workspaceId: string) {

packages/backend/server/src/core/sync/events/events.gateway.ts

@@ -11,12 +11,11 @@ import {
 import { Server, Socket } from 'socket.io';
 import { encodeStateAsUpdate, encodeStateVector } from 'yjs';
 
-import { metrics } from '../../../metrics';
-import { CallTimer } from '../../../metrics/utils';
-import { DocID } from '../../../utils/doc';
+import { CallTimer, metrics } from '../../../fundamentals';
 import { Auth, CurrentUser } from '../../auth';
 import { DocManager } from '../../doc';
 import { UserType } from '../../users';
+import { DocID } from '../../utils/doc';
 import { PermissionService } from '../../workspaces/permission';
 import { Permission } from '../../workspaces/types';
 import {

packages/backend/server/src/core/users/management.ts

@@ -5,7 +5,7 @@ import {
 } from '@nestjs/common';
 import { Args, Context, Int, Mutation, Query, Resolver } from '@nestjs/graphql';
 
-import { CloudThrottlerGuard, Throttle } from '../../throttler';
+import { CloudThrottlerGuard, Throttle } from '../../fundamentals';
 import { Auth, CurrentUser } from '../auth/guard';
 import { AuthService } from '../auth/service';
 import { FeatureManagementService } from '../features';

packages/backend/server/src/core/users/resolver.ts

@@ -11,10 +11,13 @@ import type { User } from '@prisma/client';
 import { GraphQLError } from 'graphql';
 import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';
 
-import { EventEmitter } from '../../event';
-import { PrismaService } from '../../prisma/service';
-import { CloudThrottlerGuard, Throttle } from '../../throttler';
-import type { FileUpload } from '../../types';
+import {
+  CloudThrottlerGuard,
+  EventEmitter,
+  type FileUpload,
+  PrismaService,
+  Throttle,
+} from '../../fundamentals';
 import { Auth, CurrentUser, Public, Publicable } from '../auth/guard';
 import { FeatureManagementService } from '../features';
 import { QuotaService } from '../quota';
@@ -109,6 +112,9 @@ export class UserResolver {
     const user = await this.users.findUserByEmail(email);
     if (currentUser) return user;
 
+    // return empty response when user not exists
+    if (!user) return null;
+
     // only return limited info when not logged in
     return {
       email: user?.email,

packages/backend/server/src/core/users/types.ts

@@ -1,5 +1,6 @@
-import { createUnionType, Field, Float, ID, ObjectType } from '@nestjs/graphql';
+import { createUnionType, Field, ID, ObjectType } from '@nestjs/graphql';
 import type { User } from '@prisma/client';
+import { SafeIntResolver } from 'graphql-scalars';
 
 @ObjectType('UserQuotaHumanReadable')
 export class UserQuotaHumanReadableType {
@@ -24,13 +25,13 @@ export class UserQuotaType {
   @Field({ name: 'name' })
   name!: string;
 
-  @Field(() => Float, { name: 'blobLimit' })
+  @Field(() => SafeIntResolver, { name: 'blobLimit' })
   blobLimit!: number;
 
-  @Field(() => Float, { name: 'storageQuota' })
+  @Field(() => SafeIntResolver, { name: 'storageQuota' })
   storageQuota!: number;
 
-  @Field(() => Float, { name: 'historyPeriod' })
+  @Field(() => SafeIntResolver, { name: 'historyPeriod' })
   historyPeriod!: number;
 
   @Field({ name: 'memberLimit' })

packages/backend/server/src/core/users/users.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@nestjs/common';
 
-import { PrismaService } from '../../prisma';
+import { PrismaService } from '../../fundamentals';
 
 @Injectable()
 export class UsersService {

packages/backend/server/src/core/workspaces/controller.ts

@@ -9,13 +9,12 @@ import {
 } from '@nestjs/common';
 import type { Response } from 'express';
 
-import { CallTimer } from '../../metrics';
-import { PrismaService } from '../../prisma';
-import { DocID } from '../../utils/doc';
+import { CallTimer, PrismaService } from '../../fundamentals';
 import { Auth, CurrentUser, Publicable } from '../auth';
 import { DocHistoryManager, DocManager } from '../doc';
 import { WorkspaceBlobStorage } from '../storage';
 import { UserType } from '../users';
+import { DocID } from '../utils/doc';
 import { PermissionService, PublicPageMode } from './permission';
 import { Permission } from './types';
 
@@ -57,7 +56,7 @@ export class WorkspacesController {
       this.logger.warn(`Blob ${workspaceId}/${name} has no metadata`);
     }
 
-    res.setHeader('cache-control', 'public, max-age=31536000, immutable');
+    res.setHeader('cache-control', 'public, max-age=2592000, immutable');
     body.pipe(res);
   }
 
@@ -107,6 +106,7 @@ export class WorkspacesController {
     }
 
     res.setHeader('content-type', 'application/octet-stream');
+    res.setHeader('cache-control', 'no-cache');
     res.send(update);
   }
 
@@ -143,6 +143,7 @@ export class WorkspacesController {
 
     if (history) {
       res.setHeader('content-type', 'application/octet-stream');
+      res.setHeader('cache-control', 'public, max-age=2592000, immutable');
       res.send(history.blob);
     } else {
       throw new NotFoundException('Doc history not found');

packages/backend/server/src/core/workspaces/management.ts

@@ -9,7 +9,7 @@ import {
   Resolver,
 } from '@nestjs/graphql';
 
-import { CloudThrottlerGuard, Throttle } from '../../throttler';
+import { CloudThrottlerGuard, Throttle } from '../../fundamentals';
 import { Auth, CurrentUser } from '../auth';
 import { FeatureManagementService, FeatureType } from '../features';
 import { UserType } from '../users';
@@ -119,7 +119,8 @@ export class WorkspaceManagementResolver {
   async availableFeatures(
     @CurrentUser() user: UserType
   ): Promise<FeatureType[]> {
-    if (await this.feature.canEarlyAccess(user.email)) {
+    const isEarlyAccessUser = await this.feature.isEarlyAccessUser(user.email);
+    if (isEarlyAccessUser) {
       return [FeatureType.Copilot];
     } else {
       return [];

packages/backend/server/src/core/workspaces/permission.ts

@@ -1,7 +1,7 @@
 import { ForbiddenException, Injectable } from '@nestjs/common';
 import { Prisma } from '@prisma/client';
 
-import { PrismaService } from '../../prisma';
+import { PrismaService } from '../../fundamentals';
 import { Permission } from './types';
 
 export enum PublicPageMode {

packages/backend/server/src/core/workspaces/resolvers/blob.ts

@@ -1,7 +1,6 @@
-import { ForbiddenException, Logger, UseGuards } from '@nestjs/common';
+import { HttpStatus, Logger, UseGuards } from '@nestjs/common';
 import {
   Args,
-  Float,
   Int,
   Mutation,
   Parent,
@@ -9,12 +8,18 @@ import {
   ResolveField,
   Resolver,
 } from '@nestjs/graphql';
+import { GraphQLError } from 'graphql';
+import { SafeIntResolver } from 'graphql-scalars';
 import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';
 
-import { MakeCache, PreventCache } from '../../../cache';
-import { CloudThrottlerGuard } from '../../../throttler';
-import type { FileUpload } from '../../../types';
+import {
+  CloudThrottlerGuard,
+  type FileUpload,
+  MakeCache,
+  PreventCache,
+} from '../../../fundamentals';
 import { Auth, CurrentUser } from '../../auth';
+import { FeatureManagementService, FeatureType } from '../../features';
 import { QuotaManagementService } from '../../quota';
 import { WorkspaceBlobStorage } from '../../storage';
 import { UserType } from '../../users';
@@ -28,10 +33,26 @@ export class WorkspaceBlobResolver {
   logger = new Logger(WorkspaceBlobResolver.name);
   constructor(
     private readonly permissions: PermissionService,
+    private readonly feature: FeatureManagementService,
     private readonly quota: QuotaManagementService,
     private readonly storage: WorkspaceBlobStorage
   ) {}
 
+  @ResolveField(() => [String], {
+    description: 'List blobs of workspace',
+    complexity: 2,
+  })
+  async blobs(
+    @CurrentUser() user: UserType,
+    @Parent() workspace: WorkspaceType
+  ) {
+    await this.permissions.checkWorkspace(workspace.id, user.id);
+
+    return this.storage
+      .list(workspace.id)
+      .then(list => list.map(item => item.key));
+  }
+
   @ResolveField(() => Int, {
     description: 'Blobs size of workspace',
     complexity: 2,
@@ -79,7 +100,7 @@ export class WorkspaceBlobResolver {
   async checkBlobSize(
     @CurrentUser() user: UserType,
     @Args('workspaceId') workspaceId: string,
-    @Args('size', { type: () => Float }) blobSize: number
+    @Args('size', { type: () => SafeIntResolver }) blobSize: number
   ) {
     const canWrite = await this.permissions.tryCheckWorkspace(
       workspaceId,
@@ -107,15 +128,33 @@ export class WorkspaceBlobResolver {
       Permission.Write
     );
 
-    const { quota, size } = await this.quota.getWorkspaceUsage(workspaceId);
+    const { storageQuota, usedSize, businessBlobLimit } =
+      await this.quota.getWorkspaceUsage(workspaceId);
+
+    const unlimited = await this.feature.hasWorkspaceFeature(
+      workspaceId,
+      FeatureType.UnlimitedWorkspace
+    );
 
     const checkExceeded = (recvSize: number) => {
-      if (!quota) {
-        throw new ForbiddenException('cannot find user quota');
+      if (!storageQuota) {
+        throw new GraphQLError('cannot find user quota', {
+          extensions: {
+            status: HttpStatus[HttpStatus.FORBIDDEN],
+            code: HttpStatus.FORBIDDEN,
+          },
+        });
       }
-      if (size + recvSize > quota) {
+      const total = usedSize + recvSize;
+      // only skip total storage check if workspace has unlimited feature
+      if (total > storageQuota && !unlimited) {
         this.logger.log(
-          `storage size limit exceeded: ${size + recvSize} > ${quota}`
+          `storage size limit exceeded: ${total} > ${storageQuota}`
+        );
+        return true;
+      } else if (recvSize > businessBlobLimit) {
+        this.logger.log(
+          `blob size limit exceeded: ${recvSize} > ${businessBlobLimit}`
         );
         return true;
       } else {
@@ -124,7 +163,12 @@ export class WorkspaceBlobResolver {
     };
 
     if (checkExceeded(0)) {
-      throw new ForbiddenException('storage size limit exceeded');
+      throw new GraphQLError('storage or blob size limit exceeded', {
+        extensions: {
+          status: HttpStatus[HttpStatus.PAYLOAD_TOO_LARGE],
+          code: HttpStatus.PAYLOAD_TOO_LARGE,
+        },
+      });
     }
     const buffer = await new Promise<Buffer>((resolve, reject) => {
       const stream = blob.createReadStream();
@@ -135,7 +179,14 @@ export class WorkspaceBlobResolver {
         // check size after receive each chunk to avoid unnecessary memory usage
         const bufferSize = chunks.reduce((acc, cur) => acc + cur.length, 0);
         if (checkExceeded(bufferSize)) {
-          reject(new ForbiddenException('storage size limit exceeded'));
+          reject(
+            new GraphQLError('storage or blob size limit exceeded', {
+              extensions: {
+                status: HttpStatus[HttpStatus.PAYLOAD_TOO_LARGE],
+                code: HttpStatus.PAYLOAD_TOO_LARGE,
+              },
+            })
+          );
         }
       });
       stream.on('error', reject);
@@ -143,17 +194,20 @@ export class WorkspaceBlobResolver {
         const buffer = Buffer.concat(chunks);
 
         if (checkExceeded(buffer.length)) {
-          reject(new ForbiddenException('storage size limit exceeded'));
+          reject(
+            new GraphQLError('storage limit exceeded', {
+              extensions: {
+                status: HttpStatus[HttpStatus.PAYLOAD_TOO_LARGE],
+                code: HttpStatus.PAYLOAD_TOO_LARGE,
+              },
+            })
+          );
         } else {
           resolve(buffer);
         }
       });
     });
 
-    if (!(await this.quota.checkBlobQuota(workspaceId, buffer.length))) {
-      throw new ForbiddenException('blob size limit exceeded');
-    }
-
     await this.storage.put(workspaceId, blob.filename, buffer);
     return blob.filename;
   }

packages/backend/server/src/core/workspaces/resolvers/history.ts

@@ -12,11 +12,11 @@ import {
 } from '@nestjs/graphql';
 import type { SnapshotHistory } from '@prisma/client';
 
-import { CloudThrottlerGuard } from '../../../throttler';
-import { DocID } from '../../../utils/doc';
+import { CloudThrottlerGuard } from '../../../fundamentals';
 import { Auth, CurrentUser } from '../../auth';
-import { DocHistoryManager } from '../../doc/history';
+import { DocHistoryManager } from '../../doc';
 import { UserType } from '../../users';
+import { DocID } from '../../utils/doc';
 import { PermissionService } from '../permission';
 import { Permission, WorkspaceType } from '../types';
 

packages/backend/server/src/core/workspaces/resolvers/page.ts

@@ -11,11 +11,10 @@ import {
 } from '@nestjs/graphql';
 import type { WorkspacePage as PrismaWorkspacePage } from '@prisma/client';
 
-import { PrismaService } from '../../../prisma';
-import { CloudThrottlerGuard } from '../../../throttler';
-import { DocID } from '../../../utils/doc';
+import { CloudThrottlerGuard, PrismaService } from '../../../fundamentals';
 import { Auth, CurrentUser } from '../../auth';
 import { UserType } from '../../users';
+import { DocID } from '../../utils/doc';
 import { PermissionService, PublicPageMode } from '../permission';
 import { Permission, WorkspaceType } from '../types';