chore: skip sync when offline (#5786)

Replace internal CMDK command filtering/sorting logic.
The new implementation includes the following for command scoring:
- categories weights
- highlighted fragments
- original command score value

The new logic should be much cleaner and remove some hacks in the original implementation.

Not sure if this is optimal yet. Could be changed later.

fix https://github.com/toeverything/AFFiNE/issues/5699

.eslintignore

@@ -11,6 +11,4 @@ e2e-dist-*
 static
 web-static
 public
-packages/common/sdk/src/*.d.ts
-packages/common/sdk/src/*.js
 packages/frontend/i18n/src/i18n-generated.ts

.eslintrc.js

@@ -61,18 +61,15 @@ const allPackages = [
   'packages/frontend/core',
   'packages/frontend/electron',
   'packages/frontend/graphql',
-  'packages/frontend/hooks',
   'packages/frontend/i18n',
   'packages/frontend/native',
   'packages/frontend/templates',
-  'packages/frontend/workspace',
+  'packages/frontend/workspace-impl',
   'packages/common/debug',
   'packages/common/env',
   'packages/common/infra',
-  'packages/common/sdk',
   'packages/common/theme',
   'packages/common/y-indexeddb',
-  'packages/plugins/copilot',
   'tools/cli',
   'tests/storybook',
 ];
@@ -220,6 +217,7 @@ const config = {
     'unicorn/no-useless-promise-resolve-reject': 'error',
     'unicorn/no-new-array': 'error',
     'unicorn/new-for-builtins': 'error',
+    'unicorn/prefer-node-protocol': 'error',
     'sonarjs/no-all-duplicated-branches': 'error',
     'sonarjs/no-element-overwrite': 'error',
     'sonarjs/no-empty-collection': 'error',

.github/CLA.md

@@ -33,32 +33,6 @@ You accept and agree to the following terms and conditions for your past, presen
 
 9. This Agreement will be governed by the laws of Republic of Singapore without reference to conflict of laws principles.
 
-## List of Contributors
+## How To Sign
 
-The below-signed are contributors to a code repository that is part of the project named "AFFiNE". Each below-signed contributor has read, understand and agrees to the terms above in the section within this document entitled "AFFiNE Contributor License Agreement" as of the date beside their real name (or entity name) and GitHub account name.
-
----
-
-<!--
-Example:
-
-- Dark Sky, @darkskygit, 2022/07/22
--->
-
-- Dark Sky, @darkskygit, 2022/07/22
-- Lin Onetwo, @linonetwo, 2022/02/14
-- zqran, @zqran, 2023/02/17
-- Alessio Gravili, @AlessioGr, 2023/03/04
-- Victor Nanka, @victornanka, 2023/03/09
-- Aditya Sharma, @adityash1, 2023/03/21
-- Fangdun Tsai, @fundon, 2023/03/21
-- Zhilin Liu, @lzlme, 2023/04/09
-- Skye Sun, @skyesun, 2023/04/14
-- Jordy Delgado, @Jdelgad8, 2023/04/17
-- Howard Do, @howarddo2208, 2023/04/20
-- 三咲智子 Kevin Deng, @sxzz, 2023/04/21
-- Moeyua, @moeyua, 2023/04/22
-- Shishu, @shishudesu, 2023/05/19
-- Kushagra Singh, @kush002, 2023/06/28
-- Sarvesh Kumar, @sarvesh521 2023/08/25
-- 微扰理论 Qinghao Huang, @wfnuser 2023/09/29
+Visit https://cla-assistant.io/toeverything/AFFiNE and sign it.

.github/actions/build-rust/action.yml

@@ -37,7 +37,7 @@ runs:
         echo "TARGET_CC=clang" >> "$GITHUB_ENV"
 
     - name: Cache cargo
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: |
           ~/.cargo/registry/index/

.github/actions/deploy/deploy.mjs

@@ -13,7 +13,6 @@ const {
   R2_ACCOUNT_ID,
   R2_ACCESS_KEY_ID,
   R2_SECRET_ACCESS_KEY,
-  R2_BUCKET,
   ENABLE_CAPTCHA,
   CAPTCHA_TURNSTILE_SECRET,
   OAUTH_EMAIL_SENDER,
@@ -66,8 +65,16 @@ const createHelmCommand = ({ isDryRun }) => {
         ]
       : [];
   const webReplicaCount = isProduction ? 3 : isBeta ? 2 : 2;
-  const graphqlReplicaCount = isProduction ? 10 : isBeta ? 5 : 2;
-  const syncReplicaCount = isProduction ? 10 : isBeta ? 5 : 2;
+  const graphqlReplicaCount = isProduction
+    ? Number(process.env.PRODUCTION_GRAPHQL_REPLICA) || 3
+    : isBeta
+      ? Number(process.env.isBeta_GRAPHQL_REPLICA) || 2
+      : 2;
+  const syncReplicaCount = isProduction
+    ? Number(process.env.PRODUCTION_SYNC_REPLICA) || 3
+    : isBeta
+      ? Number(process.env.BETA_SYNC_REPLICA) || 2
+      : 2;
   const namespace = isProduction
     ? 'production'
     : isBeta
@@ -96,7 +103,6 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set-string graphql.app.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`,
     `--set-string graphql.app.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`,
     `--set-string graphql.app.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`,
-    `--set-string graphql.app.objectStorage.r2.bucket="${R2_BUCKET}"`,
     `--set-string graphql.app.oauth.email.sender="${OAUTH_EMAIL_SENDER}"`,
     `--set-string graphql.app.oauth.email.login="${OAUTH_EMAIL_LOGIN}"`,
     `--set-string graphql.app.oauth.email.password="${OAUTH_EMAIL_PASSWORD}"`,
@@ -107,6 +113,7 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set-string graphql.app.payment.stripe.webhookKey="${STRIPE_WEBHOOK_KEY}"`,
     `--set        graphql.app.experimental.enableJwstCodec=true`,
     `--set        graphql.app.features.earlyAccessPreview=false`,
+    `--set        graphql.app.features.syncClientVersionCheck=true`,
     `--set        sync.replicaCount=${syncReplicaCount}`,
     `--set-string sync.image.tag="${imageTag}"`,
     ...serviceAnnotations,

.github/actions/download-core/action.yml

@@ -9,7 +9,7 @@ runs:
   using: 'composite'
   steps:
     - name: Download tar.gz
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: core
         path: .

.github/actions/setup-node/action.yml

@@ -21,14 +21,6 @@ inputs:
     description: 'set nmMode to hardlinks-local in .yarnrc.yml'
     required: false
     default: 'true'
-  build-infra:
-    description: 'Build infra'
-    required: false
-    default: 'true'
-  build-plugins:
-    description: 'Build plugins'
-    required: false
-    default: 'true'
   nmHoistingLimits:
     description: 'Set nmHoistingLimits in .yarnrc.yml'
     required: false
@@ -71,7 +63,7 @@ runs:
       run: node -e "const p = $(yarn config cacheFolder --json).effective; console.log('yarn_global_cache=' + p)" >> $GITHUB_OUTPUT
 
     - name: Cache non-full yarn cache on Linux
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       if: ${{ inputs.full-cache != 'true' && runner.os == 'Linux' }}
       with:
         path: |
@@ -83,7 +75,7 @@ runs:
     # and the decompression performance on Windows is very terrible
     # so we reduce the number of cached files on non-Linux systems by remove node_modules from cache path.
     - name: Cache non-full yarn cache on non-Linux
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       if: ${{ inputs.full-cache != 'true' && runner.os != 'Linux' }}
       with:
         path: |
@@ -91,7 +83,7 @@ runs:
         key: node_modules-cache-${{ github.job }}-${{ runner.os }}
 
     - name: Cache full yarn cache on Linux
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       if: ${{ inputs.full-cache == 'true' && runner.os == 'Linux' }}
       with:
         path: |
@@ -100,7 +92,7 @@ runs:
         key: node_modules-cache-full-${{ runner.os }}
 
     - name: Cache full yarn cache on non-Linux
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       if: ${{ inputs.full-cache == 'true' && runner.os != 'Linux' }}
       with:
         path: |
@@ -142,7 +134,7 @@ runs:
       # Note: Playwright's cache directory is hard coded because that's what it
       # says to do in the docs. There doesn't appear to be a command that prints
       # it out for us.
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       id: playwright-cache
       if: ${{ inputs.playwright-install == 'true' }}
       with:
@@ -175,7 +167,7 @@ runs:
       run: |
         echo "version=$(yarn why --json electron | grep -h 'workspace:.' | jq --raw-output '.children[].locator' | sed -e 's/@playwright\/test@.*://' | head -n 1)" >> $GITHUB_OUTPUT
 
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       id: electron-cache
       if: ${{ inputs.electron-install == 'true' }}
       with:
@@ -190,13 +182,3 @@ runs:
       run: node ./node_modules/electron/install.js
       env:
         electron_config_cache: ./node_modules/.cache/electron
-
-    - name: Build Infra
-      shell: bash
-      if: inputs.build-infra == 'true'
-      run: yarn run build:infra
-
-    - name: Build Plugins
-      if: inputs.build-plugins == 'true'
-      shell: bash
-      run: yarn run build:plugins

.github/deployment/front/Dockerfile

@@ -1,4 +1,4 @@
-FROM openresty/openresty:1.21.4.1-0-buster
+FROM openresty/openresty:1.25.3.1-0-buster
 WORKDIR /app
 COPY ./packages/frontend/core/dist ./dist
 COPY ./.github/deployment/front/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf

.github/deployment/node/Dockerfile

@@ -1,6 +1,7 @@
 FROM node:18-bookworm-slim
 
 COPY ./packages/backend/server /app
+COPY ./packages/frontend/core/dist /app/static
 WORKDIR /app
 
 RUN apt-get update && \

.github/deployment/self-host/compose.yaml

@@ -0,0 +1,59 @@
+services:
+  affine:
+    image: ghcr.io/toeverything/affine-graphql:beta
+    container_name: affine_selfhosted
+    command:
+      ['sh', '-c', 'node ./scripts/self-host-predeploy && node ./dist/index.js']
+    ports:
+      - '3010:3010'
+      - '5555:5555'
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+    volumes:
+      # custom configurations
+      - ~/.affine/self-host/config:/root/.affine/config
+      # blob storage
+      - ~/.affine/self-host/storage:/root/.affine/storage
+    logging:
+      driver: 'json-file'
+      options:
+        max-size: '1000m'
+    restart: unless-stopped
+    environment:
+      - NODE_OPTIONS=--es-module-specifier-resolution node
+      - AFFINE_CONFIG_PATH=/root/.affine/config
+      - REDIS_SERVER_HOST=redis
+      - DATABASE_URL=postgres://affine:affine@postgres:5432/affine
+      - NODE_ENV=production
+      - AFFINE_ADMIN_EMAIL=${AFFINE_ADMIN_EMAIL}
+      - AFFINE_ADMIN_PASSWORD=${AFFINE_ADMIN_PASSWORD}
+  redis:
+    image: redis
+    container_name: affine_redis
+    restart: unless-stopped
+    volumes:
+      - ~/.affine/self-host/redis:/data
+    healthcheck:
+      test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  postgres:
+    image: postgres
+    container_name: affine_postgres
+    restart: unless-stopped
+    volumes:
+      - ~/.affine/self-host/postgres:/var/lib/postgresql/data
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U affine']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    environment:
+      POSTGRES_USER: affine
+      POSTGRES_PASSWORD: affine
+      POSTGRES_DB: affine
+      PGDATA: /var/lib/postgresql/data/pgdata

.github/helm/affine-cloud/.gitignore

@@ -1 +0,0 @@
-charts/

.github/helm/affine-cloud/.helmignore

@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

.github/helm/affine-cloud/Chart.lock

@@ -1,6 +0,0 @@
-dependencies:
-- name: postgresql
-  repository: https://charts.bitnami.com/bitnami
-  version: 13.2.23
-digest: sha256:5b64538509bd067bb0f67bf082847a2c5d66dc37d0b9d7948a40405d9c446400
-generated: "2023-12-05T03:04:57.997927753Z"

.github/helm/affine-cloud/Chart.yaml

@@ -1,12 +0,0 @@
-apiVersion: v2
-name: affine-cloud
-description: A Helm chart for AFFiNE Cloud
-
-type: application
-version: 0.6.1
-appVersion: '0.6.1'
-
-dependencies:
-  - name: postgresql
-    version: 13.2.23
-    repository: https://charts.bitnami.com/bitnami

.github/helm/affine-cloud/readme.md

@@ -1,30 +0,0 @@
-# Helm Chart Configuration
-
-The following table lists the configurable parameters of this Helm chart and their default values.
-
-## AFFiNE Cloud Server parameters
-
-| Parameter                      | Description                                        | Default            |
-| ------------------------------ | -------------------------------------------------- | ------------------ |
-| `affineCloud.tag`              | The Docker tag of the AffineCloud image to be used | `'nightly-latest'` |
-| `affineCloud.resources.cpu`    | The CPU resources allocated for AffineCloud        | `'250m'`           |
-| `affineCloud.resources.memory` | The memory resources allocated for AffineCloud     | `'0.5Gi'`          |
-| `affineCloud.signKey`          | The key used to sign the JWT tokens                | `'c2VjcmV0'`       |
-| `affineCloud.service.type`     | The type of the Kubernetes service                 | `'ClusterIP'`      |
-| `affineCloud.service.port`     | The port of the Kubernetes service                 | `'http'`           |
-| `affineCloud.mail.account`     | The email account used to send emails              | `''`               |
-| `affineCloud.mail.password`    | The password of the email account                  | `''`               |
-
-## PostgreSQL parameters
-
-| Parameter                                    | Description                                                                           | Default      |
-| -------------------------------------------- | ------------------------------------------------------------------------------------- | ------------ |
-| `postgresql.auth.username`                   | Username for the PostgreSQL database                                                  | `'affine'`   |
-| `postgresql.auth.password`                   | Password for the PostgreSQL database. Please change this for production environments. | `'password'` |
-| `postgresql.auth.database`                   | The name of the default database that will be created on image startup                | `'affine'`   |
-| `postgresql.primary.resources.limits.cpu`    | The CPU resources allocated for the PostgreSQL primary node                           | `'500m'`     |
-| `postgresql.primary.resources.limits.memory` | The memory resources allocated for the PostgreSQL primary node                        | `'0.5Gi'`    |
-
-For more postgres parameters, please refer to: https://artifacthub.io/packages/helm/bitnami/postgresql
-
-Please note that for the `postgresql.auth.password`, you should provide your own password for production environments. The default value is provided only for demonstration purposes.

.github/helm/affine-cloud/templates/_helper.tpl

@@ -1,51 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "affine-cloud.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "affine-cloud.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "affine-cloud.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "affine-cloud.labels" -}}
-helm.sh/chart: {{ include "affine-cloud.chart" . }}
-{{ include "affine-cloud.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "affine-cloud.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "affine-cloud.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}

.github/helm/affine-cloud/templates/deployment.yaml

@@ -1,51 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: "{{ include "affine-cloud.fullname" . }}"
-  labels:
-    {{- include "affine-cloud.labels" . | nindent 4 }}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      {{- include "affine-cloud.selectorLabels" . | nindent 6 }}
-  strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 2
-  template:
-    metadata:
-      labels:
-        {{- include "affine-cloud.selectorLabels" . | nindent 8 }}
-    spec:
-      restartPolicy: Always
-      containers:
-      - name: affine-cloud
-        image: "ghcr.io/toeverything/cloud-self-hosted:{{ .Values.affineCloud.tag | default .Chart.AppVersion }}"
-        env:
-        - name: PG_USER
-          value: "{{ .Values.postgresql.auth.username }}"
-        - name: PG_PASS
-          value: "{{ .Values.postgresql.auth.password }}"
-        - name: PG_DATABASE
-          value: "{{ .Values.postgresql.auth.database }}"
-        - name: PG_HOST
-          value: "{{ .Values.postgresql.fullnameOverride | default (printf "%s-postgresql" .Release.Name) }}"
-        - name: DATABASE_URL
-          value: "{{ .Values.affineCloud.databaseUrl | default "postgresql://$(PG_USER):$(PG_PASS)@$(PG_HOST)/$(PG_DATABASE)" }}"
-        envFrom:
-        - secretRef:
-            name: affine-cloud-secret
-        ports:
-        - containerPort: 3000
-        livenessProbe:
-          httpGet:
-            path: /api/healthz
-            port: 3000
-          failureThreshold: 1
-          initialDelaySeconds: 10
-          periodSeconds: 10
-        resources:
-          limits:
-            cpu: "{{ .Values.affineCloud.resources.cpu }}"
-            memory: "{{ .Values.affineCloud.resources.memory }}"

.github/helm/affine-cloud/templates/secret.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: affine-cloud-secret
-type: Opaque
-data:
-  SIGN_KEY: "{{ .Values.affineCloud.signKey }}"
-  MAIL_ACCOUNT: "{{ .Values.affineCloud.mail.account }}"
-  MAIL_PASSWORD: "{{ .Values.affineCloud.mail.password }}"

.github/helm/affine-cloud/templates/services.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: "{{ include "affine-cloud.fullname" . }}"
-  labels:
-    {{- include "affine-cloud.labels" . | nindent 4 }}
-spec:
-  type: "{{ .Values.affineCloud.service.type }}"
-  ports:
-    - name: http
-      protocol: TCP
-      port: {{ .Values.affineCloud.service.port }}
-      targetPort: 3000
-  selector:
-    {{- include "affine-cloud.selectorLabels" . | nindent 4 }}

.github/helm/affine-cloud/values.yaml

@@ -1,30 +0,0 @@
-affineCloud:
-  tag: 'canary-5e0d5e0cc65ea46f326fdde12658bfac59b38c9f-0949'
-  # databaseUrl: 'postgresql://affine:password@affine-cloud-postgresql:5432/affine'
-  signKey: TUFtdFdzQTJhdGJuem01TA==
-  mail:
-    account: ''
-    password: ''
-  service:
-    type: ClusterIP
-    port: 80
-  resources:
-    cpu: '250m'
-    memory: 0.5Gi
-postgresql:
-  fullnameOverride: tcp-postgresql
-  auth:
-    # only for demo, please modify it at prod env
-    username: affine
-    password: password
-    database: affine
-  primary:
-    initdb:
-      scripts:
-        01-init.sql: |
-          CREATE DATABASE affine_binary;
-          GRANT ALL PRIVILEGES ON DATABASE affine_binary TO affine;
-    resources:
-      limits:
-        cpu: '500m'
-        memory: 0.5Gi

.github/helm/affine/Chart.yaml

@@ -3,4 +3,4 @@ name: affine
 description: AFFiNE cloud chart
 type: application
 version: 0.0.0
-appVersion: "0.11.0"
+appVersion: "0.12.0"

.github/helm/affine/charts/graphql/Chart.yaml

@@ -3,7 +3,7 @@ name: graphql
 description: AFFiNE GraphQL server
 type: application
 version: 0.0.0
-appVersion: "0.11.0"
+appVersion: "0.12.0"
 dependencies:
   - name: gcloud-sql-proxy
     version: 0.0.0

.github/helm/affine/charts/graphql/templates/deployment.yaml

@@ -39,6 +39,8 @@ spec:
             value: "--max-old-space-size=4096"
           - name: NO_COLOR
             value: "1"
+          - name: DEPLOYMENT_TYPE
+            value: "affine"
           - name: SERVER_FLAVOR
             value: "graphql"
           - name: AFFINE_ENV
@@ -73,12 +75,16 @@ spec:
             value: "{{ .Values.app.path }}"
           - name: AFFINE_SERVER_HOST
             value: "{{ .Values.app.host }}"
+          - name: AFFINE_SERVER_HTTPS
+            value: "{{ .Values.app.https }}"
           - name: ENABLE_R2_OBJECT_STORAGE
             value: "{{ .Values.app.objectStorage.r2.enabled }}"
           - name: ENABLE_CAPTCHA
             value: "{{ .Values.app.captcha.enabled }}"
           - name: FEATURES_EARLY_ACCESS_PREVIEW
             value: "{{ .Values.app.features.earlyAccessPreview }}"
+          - name: FEATURES_SYNC_CLIENT_VERSION_CHECK
+            value: "{{ .Values.app.features.syncClientVersionCheck }}"
           - name: OAUTH_EMAIL_SENDER
             valueFrom:
               secretKeyRef:
@@ -136,11 +142,6 @@ spec:
               secretKeyRef:
                 name: "{{ .Values.app.objectStorage.r2.secretName }}"
                 key: secretAccessKey
-          - name: R2_OBJECT_STORAGE_BUCKET
-            valueFrom:
-              secretKeyRef:
-                name: "{{ .Values.app.objectStorage.r2.secretName }}"
-                key: bucket
           {{ end }}
           {{ if .Values.app.captcha.enabled }}
           - name: CAPTCHA_TURNSTILE_SECRET
@@ -150,6 +151,8 @@ spec:
                 key: turnstileSecret
           {{ end }}
           {{ if .Values.app.oauth.google.enabled }}
+          - name: OAUTH_GOOGLE_ENABLED
+            value: "true"
           - name: OAUTH_GOOGLE_CLIENT_ID
             valueFrom:
               secretKeyRef:

.github/helm/affine/charts/graphql/templates/migration.yaml

@@ -35,6 +35,23 @@ spec:
           - name: DATABASE_URL
             value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.gcloud.cloudSqlInternal }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
           {{ end }}
+          {{ if .Values.app.objectStorage.r2.enabled }}
+          - name: R2_OBJECT_STORAGE_ACCOUNT_ID
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                key: accountId
+          - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                key: accessKeyId
+          - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                key: secretAccessKey
+          {{ end }}
         resources:
           requests:
             cpu: '100m'

.github/helm/affine/charts/graphql/templates/r2-secret.yaml

@@ -8,5 +8,4 @@ data:
   accountId: {{ .Values.app.objectStorage.r2.accountId | b64enc }}
   accessKeyId: {{ .Values.app.objectStorage.r2.accessKeyId | b64enc }}
   secretAccessKey: {{ .Values.app.objectStorage.r2.secretAccessKey | b64enc }}
-  bucket: {{ .Values.app.objectStorage.r2.bucket | b64enc }}
 {{- end }}

.github/helm/affine/charts/graphql/values.yaml

@@ -16,6 +16,7 @@ app:
   path: ''
   # AFFINE_SERVER_HOST
   host: '0.0.0.0'
+  https: true
   doc:
     mergeInterval: "3000"
   jwt:
@@ -34,7 +35,6 @@ app:
       accountId: ''
       accessKeyId: ''
       secretAccessKey: ''
-      bucket: ''
   oauth:
     email:
       secretName: 'oauth-email'
@@ -60,6 +60,7 @@ app:
       webhookKey: ''
   features:
     earlyAccessPreview: false
+    syncClientVersionCheck: false
 
 serviceAccount:
   create: true

.github/helm/affine/charts/sync/Chart.yaml

@@ -3,7 +3,7 @@ name: sync
 description: AFFiNE Sync Server
 type: application
 version: 0.0.0
-appVersion: "0.11.0"
+appVersion: "0.12.0"
 dependencies:
   - name: gcloud-sql-proxy
     version: 0.0.0

.github/helm/affine/charts/sync/templates/deployment.yaml

@@ -36,6 +36,8 @@ spec:
             value: "{{ .Values.env }}"
           - name: NO_COLOR
             value: "1"
+          - name: DEPLOYMENT_TYPE
+            value: "affine"
           - name: SERVER_FLAVOR
             value: "sync"
           - name: NEXTAUTH_URL

.github/helm/deployment_guide.md

@@ -1,60 +0,0 @@
-# Cluster Deployment Guide
-
-This document provides a step-by-step guide for developers on how to deploy services in a Kubernetes cluster. The following content assumes that the reader already has a basic understanding of Kubernetes concepts and operations.
-
-### 1. Configure Service Mesh (Optional)
-
-In the Kubernetes cluster, we optionally use Service Mesh (like Istio and Anthos Service Mesh) to manage the network interactions of microservices. If Service Mesh is already deployed on your cluster or do not need to use the service network, you can skip this step. In this step, we assume that you are using Google Kubernetes Engine (GKE) and have already installed Anthos Service Mesh on your cluster, if you wish to use another Ingress Controller, please refer to the relevant documentation.
-
-To configure your kubectl context to interact with your Kubernetes cluster using the gcloud tool, you need to execute the following commands:
-
-```sh
-export CLUSTER_NAME=your_cluster_name
-export REGION=your_cluster_region
-export PROJECT=your_project_id
-gcloud container clusters get-credentials $CLUSTER_NAME --region $REGION --project $PROJECT
-```
-
-In this command, you should replace `CLUSTER_NAME`, `REGION` and `PROJECT` with the actual name, region and project id of your Kubernetes cluster. This command retrieves the access credentials for your Kubernetes cluster and automatically configures kubectl to use these credentials.
-
-Now, to inject Service Mesh for a specific Namespace, first, set the environment variable `NAMESPACE` that should correspond to your target Kubernetes Namespace. In this example, we use `prod` as the target Namespace:
-
-```sh
-export NAMESPACE=prod
-```
-
-Then, we label the Namespace which will enable Istio to automatically inject the sidecar container for all new Pods under this Namespace:
-
-```sh
-kubectl label namespace $NAMESPACE istio-injection- istio.io/rev=asm-managed --overwrite
-```
-
-Finally, we trigger the Kubernetes Deployment restart mechanism to allow existing Pods to also obtain sidecar container injection:
-
-```sh
-kubectl rollout restart deployment -n $NAMESPACE
-```
-
-### 2. Deploying the Application
-
-Next, we will deploy our application in the Kubernetes cluster through Helm. First, set relevant environment variables:
-
-```sh
-export NAMESPACE=prod
-export RELEASE=affine-cloud-prod
-export PATH=.github/helm/affine-cloud
-```
-
-- `NAMESPACE` should be consistent with the first step, indicating your target Kubernetes Namespace.
-- `RELEASE` is the name of your Helm release.
-- `PATH` is the location of your Helm chart in your file system.
-
-Finally, use the `helm upgrade --install` command to deploy or upgrade your application:
-
-```sh
-helm upgrade --namespace $NAMESPACE --create-namespace --install $RELEASE $PATH
-```
-
-This command creates (if it doesn't already exist) and deploys your Helm chart in the specified Namespace. If the release already exists, it will be upgraded.
-
-The above are the complete steps for deploying an application in a Kubernetes cluster. Make sure all prerequisites are met before deploying, and also ensure that you have the correct permissions for operations in Kubernetes.

.github/labeler.yml

@@ -19,35 +19,20 @@ mod:dev:
           - 'tools/cli/**/*'
           - 'packages/common/debug/**/*'
 
-mod:plugin:
-  - changed-files:
-      - any-glob-to-any-file:
-          - 'packages/plugins/**/*'
-
-plugin:copilot:
-  - changed-files:
-      - any-glob-to-any-file:
-          - 'packages/plugins/copilot/**/*'
-
 mod:infra:
   - changed-files:
       - any-glob-to-any-file:
           - 'packages/common/infra/**/*'
 
-mod:sdk:
-  - changed-files:
-      - any-glob-to-any-file:
-          - 'packages/common/sdk/**/*'
-
 mod:plugin-cli:
   - changed-files:
       - any-glob-to-any-file:
           - 'tools/plugin-cli/**/*'
 
-mod:workspace:
+mod:workspace-impl:
   - changed-files:
       - any-glob-to-any-file:
-          - 'packages/frontend/workspace/**/*'
+          - 'packages/frontend/workspace-impl/**/*'
 
 mod:i18n:
   - changed-files:
@@ -59,11 +44,6 @@ mod:env:
       - any-glob-to-any-file:
           - 'packages/common/env/**/*'
 
-mod:hooks:
-  - changed-files:
-      - any-glob-to-any-file:
-          - 'packages/frontend/hooks/**/*'
-
 mod:component:
   - changed-files:
       - any-glob-to-any-file:

.github/renovate.json

@@ -1,27 +1,35 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:base",
-    "group:allNonMajor",
-    ":preserveSemverRanges",
-    ":disablePeerDependencies"
-  ],
+  "extends": ["config:recommended", ":disablePeerDependencies"],
   "labels": ["dependencies"],
+  "ignorePaths": [
+    "**/node_modules/**",
+    "**/bower_components/**",
+    "**/vendor/**",
+    "**/examples/**",
+    "**/__tests__/**",
+    "**/test/**",
+    "**/__fixtures__/**"
+  ],
   "packageRules": [
     {
       "matchPackageNames": ["napi", "napi-build", "napi-derive"],
+      "rangeStrategy": "replace",
       "groupName": "napi-rs"
     },
     {
       "matchPackagePatterns": ["^eslint", "^@typescript-eslint"],
+      "rangeStrategy": "replace",
       "groupName": "linter"
     },
     {
       "matchPackagePatterns": ["^@nestjs"],
+      "rangeStrategy": "replace",
       "groupName": "nestjs"
     },
     {
       "matchPackagePatterns": ["^@opentelemetry"],
+      "rangeStrategy": "replace",
       "groupName": "opentelemetry"
     },
     {
@@ -30,22 +38,44 @@
         "@prisma/instrumentation",
         "prisma"
       ],
+      "rangeStrategy": "replace",
       "groupName": "prisma"
     },
     {
       "matchPackagePatterns": ["^@electron-forge"],
+      "rangeStrategy": "replace",
       "groupName": "electron-forge"
     },
     {
+      "matchPackageNames": ["oxlint"],
+      "rangeStrategy": "replace",
+      "groupName": "oxlint"
+    },
+    {
+      "groupName": "blocksuite-canary",
       "matchPackagePatterns": ["^@blocksuite"],
       "excludePackageNames": ["@blocksuite/icons"],
-      "followTag": "nightly"
+      "rangeStrategy": "replace",
+      "followTag": "canary"
+    },
+    {
+      "groupName": "all non-major dependencies",
+      "groupSlug": "all-minor-patch",
+      "matchPackagePatterns": ["*"],
+      "excludePackagePatterns": ["^@blocksuite/", "oxlint"],
+      "matchUpdateTypes": ["minor", "patch"]
+    },
+    {
+      "matchPackagePatterns": ["*"],
+      "rangeStrategy": "replace",
+      "excludePackagePatterns": ["^@blocksuite/"]
     }
   ],
   "commitMessagePrefix": "chore: ",
   "commitMessageAction": "bump up",
   "commitMessageTopic": "{{depName}} version",
   "ignoreDeps": [],
+  "postUpdateOptions": ["yarnDedupeHighest"],
   "lockFileMaintenance": {
     "enabled": true,
     "extends": ["schedule:weekly"]

.github/workflows/build-test.yml

@@ -19,6 +19,7 @@ env:
   MACOSX_DEPLOYMENT_TARGET: '10.13'
   NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
   PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/node_modules/.cache/ms-playwright
+  DEPLOYMENT_TYPE: affine
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -95,6 +96,8 @@ jobs:
         run: |
           git checkout .yarnrc.yml
           yarn lint:prettier
+      - name: Yarn Dedupe
+        run: yarn dedupe --check
       - name: Run Type Check
         run: yarn typecheck
 
@@ -108,44 +111,6 @@ jobs:
           yarn set version $(node -e "console.log(require('./package.json').packageManager.split('@')[1])")
           git diff --exit-code
 
-  e2e-plugin-test:
-    name: E2E Plugin Test
-    runs-on: ubuntu-latest
-    env:
-      DISTRIBUTION: browser
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-        with:
-          playwright-install: true
-          electron-install: false
-          full-cache: true
-      - name: Run playwright tests
-        run: yarn e2e --forbid-only
-        working-directory: tests/affine-plugin
-        env:
-          COVERAGE: true
-      - name: Collect code coverage report
-        run: yarn exec nyc report -t .nyc_output --report-dir .coverage --reporter=lcov
-
-      - name: Upload e2e test coverage results
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./.coverage/lcov.info
-          flags: e2e-plugin-test
-          name: affine
-          fail_ci_if_error: false
-
-      - name: Upload test results
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results-e2e-plugin
-          path: ./test-results
-          if-no-files-found: ignore
-
   e2e-test:
     name: E2E Test
     runs-on: ubuntu-latest
@@ -169,7 +134,7 @@ jobs:
 
       - name: Upload test results
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: test-results-e2e-${{ matrix.shard }}
           path: ./test-results
@@ -194,7 +159,7 @@ jobs:
 
       - name: Upload test results
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: test-results-e2e-migration
           path: ./tests/affine-migration/test-results
@@ -216,7 +181,7 @@ jobs:
           full-cache: true
 
       - name: Download affine.linux-x64-gnu.node
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: affine.linux-x64-gnu.node
           path: ./packages/frontend/native
@@ -225,7 +190,7 @@ jobs:
         run: yarn nx test:coverage @affine/monorepo
 
       - name: Upload unit test coverage results
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./.coverage/store/lcov.info
@@ -244,8 +209,8 @@ jobs:
         spec:
           - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu }
           - { os: windows-latest, target: x86_64-pc-windows-msvc }
-          - { os: macos-latest, target: x86_64-apple-darwin }
-          - { os: macos-latest, target: aarch64-apple-darwin }
+          - { os: macos-14, target: x86_64-apple-darwin }
+          - { os: macos-14, target: aarch64-apple-darwin }
 
     steps:
       - uses: actions/checkout@v4
@@ -254,8 +219,6 @@ jobs:
         with:
           extra-flags: workspaces focus @affine/native
           electron-install: false
-          build-infra: false
-          build-plugins: false
       - name: Setup filename
         id: filename
         shell: bash
@@ -269,7 +232,7 @@ jobs:
           package: '@affine/native'
           nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
       - name: Upload ${{ steps.filename.outputs.filename }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.filename.outputs.filename }}
           path: ./packages/frontend/native/${{ steps.filename.outputs.filename }}
@@ -287,8 +250,6 @@ jobs:
         with:
           extra-flags: workspaces focus @affine/storage
           electron-install: false
-          build-infra: false
-          build-plugins: false
       - name: Build Rust
         uses: ./.github/actions/build-rust
         with:
@@ -296,7 +257,7 @@ jobs:
           package: '@affine/storage'
           nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
       - name: Upload storage.node
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: storage.node
           path: ./packages/backend/storage/storage.node
@@ -312,7 +273,6 @@ jobs:
         uses: ./.github/actions/setup-node
         with:
           electron-install: false
-          build-plugins: false
           full-cache: true
       - name: Build Core
         # always skip cache because its fast, and cache configuration is always changing
@@ -320,7 +280,7 @@ jobs:
       - name: zip core
         run: tar -czf dist.tar.gz --directory=packages/frontend/core/dist .
       - name: Upload core artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: core
           path: dist.tar.gz
@@ -331,6 +291,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-storage
     env:
+      NODE_ENV: test
       DISTRIBUTION: browser
     services:
       postgres:
@@ -358,6 +319,12 @@ jobs:
           electron-install: false
           full-cache: true
 
+      - name: Download storage.node
+        uses: actions/download-artifact@v4
+        with:
+          name: storage.node
+          path: ./packages/backend/server
+
       - name: Initialize database
         run: |
           psql -h localhost -U postgres -c "CREATE DATABASE affine;"
@@ -380,12 +347,6 @@ jobs:
         env:
           DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
 
-      - name: Download storage.node
-        uses: actions/download-artifact@v3
-        with:
-          name: storage.node
-          path: ./packages/backend/server
-
       - name: Run server tests
         run: yarn workspace @affine/server test:coverage
         env:
@@ -393,7 +354,7 @@ jobs:
           DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
 
       - name: Upload server test coverage results
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./packages/backend/server/.coverage/lcov.info
@@ -450,6 +411,18 @@ jobs:
           playwright-install: true
           hard-link-nm: false
 
+      - name: Download storage.node
+        uses: actions/download-artifact@v4
+        with:
+          name: storage.node
+          path: ./packages/backend/server
+
+      - name: Download affine.linux-x64-gnu.node
+        uses: actions/download-artifact@v4
+        with:
+          name: affine.linux-x64-gnu.node
+          path: ./packages/frontend/native
+
       - name: Initialize database
         run: |
           psql -h localhost -U postgres -c "CREATE DATABASE affine;"
@@ -469,17 +442,6 @@ jobs:
         run: |
           yarn workspace @affine/server data-migration run
           yarn workspace @affine/server exec node --loader ts-node/esm/transpile-only ./scripts/init-db.ts
-      - name: Download storage.node
-        uses: actions/download-artifact@v3
-        with:
-          name: storage.node
-          path: ./packages/backend/server
-
-      - name: Download affine.linux-x64-gnu.node
-        uses: actions/download-artifact@v3
-        with:
-          name: affine.linux-x64-gnu.node
-          path: ./packages/frontend/native
 
       - name: ${{ matrix.tests.name }}
         run: |
@@ -490,7 +452,7 @@ jobs:
 
       - name: Upload test results
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: test-results-e2e-server
           path: ./tests/affine-cloud/test-results
@@ -501,22 +463,21 @@ jobs:
     runs-on: ${{ matrix.spec.os }}
     strategy:
       fail-fast: false
-      # all combinations: macos-latest x64, macos-latest arm64, windows-latest x64, ubuntu-latest x64
       matrix:
         spec:
           - {
-              os: macos-latest,
+              os: macos-14,
               platform: macos,
               arch: x64,
               target: x86_64-apple-darwin,
-              test: true,
+              test: false,
             }
           - {
-              os: macos-latest,
+              os: macos-14,
               platform: macos,
               arch: arm64,
               target: aarch64-apple-darwin,
-              test: false,
+              test: true,
             }
           - {
               os: ubuntu-latest,
@@ -554,7 +515,7 @@ jobs:
           echo "filename=affine.$PLATFORM_ARCH_ABI.node" >> "$GITHUB_OUTPUT"
 
       - name: Download ${{ steps.filename.outputs.filename }}
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ steps.filename.outputs.filename }}
           path: ./packages/frontend/native
@@ -562,8 +523,7 @@ jobs:
       - name: Run unit tests
         if: ${{ matrix.spec.test }}
         shell: bash
-        run: yarn vitest
-        working-directory: packages/frontend/electron
+        run: yarn workspace @affine/electron vitest
 
       - name: Download core artifact
         uses: ./.github/actions/download-core
@@ -574,7 +534,7 @@ jobs:
         run: yarn workspace @affine/electron build
 
       - name: Run desktop tests
-        if: ${{ matrix.spec.test && matrix.spec.os == 'ubuntu-latest' }}
+        if: ${{ matrix.spec.os == 'ubuntu-latest' }}
         run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine-test/affine-desktop e2e
 
       - name: Run desktop tests
@@ -582,7 +542,7 @@ jobs:
         run: yarn workspace @affine-test/affine-desktop e2e
 
       - name: Make bundle
-        if: ${{ matrix.spec.os == 'macos-latest' && matrix.spec.arch == 'arm64' }}
+        if: ${{ matrix.spec.os == 'macos-14' && matrix.spec.arch == 'arm64' }}
         env:
           SKIP_BUNDLE: true
           SKIP_WEB_BUILD: true
@@ -590,14 +550,33 @@ jobs:
         run: yarn workspace @affine/electron package --platform=darwin --arch=arm64
 
       - name: Output check
-        if: ${{ matrix.spec.os == 'macos-latest' && matrix.spec.arch == 'arm64' }}
+        if: ${{ matrix.spec.os == 'macos-14' && matrix.spec.arch == 'arm64' }}
         run: |
           yarn workspace @affine/electron exec node --loader ts-node/esm/transpile-only ./scripts/macos-arm64-output-check.ts
 
       - name: Upload test results
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: test-results-e2e-${{ matrix.spec.os }}-${{ matrix.spec.arch }}
           path: ./test-results
           if-no-files-found: ignore
+
+  test-done:
+    needs:
+      - analyze
+      - lint
+      - check-yarn-binary
+      - e2e-test
+      - e2e-migration-test
+      - unit-test
+      - server-test
+      - server-e2e-test
+      - desktop-test
+    if: always()
+    runs-on: ubuntu-latest
+    name: 3, 2, 1 Launch
+    steps:
+      - run: exit 1
+        # Thank you, next https://github.com/vercel/next.js/blob/canary/.github/workflows/build_and_test.yml#L379
+        if: ${{ always() && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}

.github/workflows/deploy.yml

@@ -29,10 +29,11 @@ jobs:
         uses: ./.github/actions/setup-node
         with:
           electron-install: false
+          extra-flags: workspaces focus @affine/server
       - name: Build Server
         run: yarn workspace @affine/server build
       - name: Upload server dist
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: server-dist
           path: ./packages/backend/server/dist
@@ -48,8 +49,6 @@ jobs:
         uses: ./.github/actions/setup-version
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
-      - name: Build Plugins
-        run: yarn run build:plugins
       - name: Build Core
         run: yarn nx build @affine/core --skip-nx-cache
         env:
@@ -64,17 +63,18 @@ jobs:
           SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+          PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
       - name: Upload core artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: core
           path: ./packages/frontend/core/dist
           if-no-files-found: error
 
-  build-storage:
-    name: Build Storage
+  build-core-selfhost:
+    name: Build @affine/core
     runs-on: ubuntu-latest
-
+    environment: ${{ github.event.inputs.flavor }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Version
@@ -82,22 +82,33 @@ jobs:
         uses: ./.github/actions/setup-version
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
-      - name: Build Rust
-        uses: ./.github/actions/build-rust
+      - name: Build Core
+        run: yarn nx build @affine/core --skip-nx-cache
+        env:
+          BUILD_TYPE: ${{ github.event.inputs.flavor }}
+          SHOULD_REPORT_TRACE: false
+          PUBLIC_PATH: '/'
+      - name: Download selfhost fonts
+        run: node ./scripts/download-blocksuite-fonts.mjs
+      - name: Upload core artifact
+        uses: actions/upload-artifact@v4
         with:
-          target: 'x86_64-unknown-linux-gnu'
-          package: '@affine/storage'
-          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-      - name: Upload storage.node
-        uses: actions/upload-artifact@v3
-        with:
-          name: storage.node
-          path: ./packages/backend/storage/storage.node
+          name: selfhost-core
+          path: ./packages/frontend/core/dist
           if-no-files-found: error
 
-  build-storage-arm64:
-    name: Build Storage arm64
+  build-storage:
+    name: Build Storage - ${{ matrix.targets.name }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        targets:
+          - name: x86_64-unknown-linux-gnu
+            file: storage.node
+          - name: aarch64-unknown-linux-gnu
+            file: storage.arm64.node
+          - name: armv7-unknown-linux-gnueabihf
+            file: storage.armv7.node
 
     steps:
       - uses: actions/checkout@v4
@@ -106,16 +117,19 @@ jobs:
         uses: ./.github/actions/setup-version
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
+        with:
+          electron-install: false
+          extra-flags: workspaces focus @affine/storage
       - name: Build Rust
         uses: ./.github/actions/build-rust
         with:
-          target: 'aarch64-unknown-linux-gnu'
+          target: ${{ matrix.targets.name }}
           package: '@affine/storage'
           nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-      - name: Upload storage.node
-        uses: actions/upload-artifact@v3
+      - name: Upload ${{ matrix.targets.file }}
+        uses: actions/upload-artifact@v4
         with:
-          name: storage.arm64.node
+          name: ${{ matrix.targets.file }}
           path: ./packages/backend/storage/storage.node
           if-no-files-found: error
 
@@ -125,32 +139,39 @@ jobs:
     needs:
       - build-server
       - build-core
+      - build-core-selfhost
       - build-storage
-      - build-storage-arm64
     steps:
       - uses: actions/checkout@v4
       - name: Download core artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: core
           path: ./packages/frontend/core/dist
       - name: Download server dist
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: server-dist
           path: ./packages/backend/server/dist
       - name: Download storage.node
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: storage.node
           path: ./packages/backend/server
       - name: Download storage.node arm64
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: storage.arm64.node
           path: ./packages/backend/storage
-      - name: move storage.arm64.node
-        run: mv ./packages/backend/storage/storage.node ./packages/backend/server/storage.arm64.node
+      - name: Download storage.node arm64
+        uses: actions/download-artifact@v4
+        with:
+          name: storage.armv7.node
+          path: .
+      - name: move storage files
+        run: |
+          mv ./packages/backend/storage/storage.node ./packages/backend/server/storage.arm64.node
+          mv storage.node ./packages/backend/server/storage.armv7.node
       - name: Setup env
         run: |
           echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
@@ -192,9 +213,19 @@ jobs:
           registry-url: https://npm.pkg.github.com
           scope: '@toeverything'
 
+      - name: Remove core dist
+        run: rm -rf ./packages/frontend/core/dist
+
+      - name: Download selfhost core artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: selfhost-core
+          path: ./packages/frontend/core/dist
+
       - name: Install Node.js dependencies
         run: |
-          yarn config set --json supportedArchitectures.cpu '["x64", "arm64"]'
+          yarn config set --json supportedArchitectures.cpu '["x64", "arm64", "arm"]'
+          yarn config set --json supportedArchitectures.libc '["glibc"]'
           yarn workspaces focus @affine/server --production
 
       - name: Generate Prisma client
@@ -206,7 +237,7 @@ jobs:
           context: .
           push: true
           pull: true
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
           provenance: true
           file: .github/deployment/node/Dockerfile
           tags: ghcr.io/toeverything/affine-graphql:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-graphql:${{env.RELEASE_FLAVOR}}
@@ -242,7 +273,6 @@ jobs:
           R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-          R2_BUCKET: ${{ secrets.R2_BUCKET }}
           ENABLE_CAPTCHA: true
           CAPTCHA_TURNSTILE_SECRET: ${{ secrets.CAPTCHA_TURNSTILE_SECRET }}
           OAUTH_EMAIL_SENDER: ${{ secrets.OAUTH_EMAIL_SENDER }}

.github/workflows/pr-auto-assign.yml

@@ -9,4 +9,4 @@ jobs:
   add-reviews:
     runs-on: ubuntu-latest
     steps:
-      - uses: kentaro-m/auto-assign-action@v1.2.5
+      - uses: kentaro-m/auto-assign-action@v2.0.0

.github/workflows/publish-storybook.yml

@@ -32,8 +32,6 @@ jobs:
         uses: ./.github/actions/setup-node
         with:
           electron-install: false
-      - name: Build Plugins
-        run: yarn run build:plugins
       - uses: chromaui/action-next@v1
         with:
           workingDir: tests/storybook
@@ -44,7 +42,7 @@ jobs:
         env:
           CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
           NODE_OPTIONS: ${{ env.NODE_OPTIONS }}
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: chromatic-build-artifacts-${{ github.run_id }}

.github/workflows/publish-ui-storybook.yml

@@ -0,0 +1,51 @@
+name: Publish UI Storybook
+
+env:
+  NODE_OPTIONS: --max-old-space-size=4096
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - canary
+  pull_request:
+    branches:
+      - canary
+    paths-ignore:
+      - README.md
+      - .github/**
+      - packages/backend/server
+      - packages/frontend/electron
+      - '!.github/workflows/publish-storybook.yml'
+
+jobs:
+  publish-ui-storybook:
+    name: Publish UI Storybook
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
+          # This is required to fetch all commits for chromatic
+          fetch-depth: 0
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          electron-install: false
+      - uses: chromaui/action-next@v1
+        with:
+          workingDir: packages/frontend/component
+          buildScriptName: build:storybook
+          exitOnceUploaded: true
+          onlyChanged: false
+          diagnostics: true
+        env:
+          CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_UI_PROJECT_TOKEN }}
+          NODE_OPTIONS: ${{ env.NODE_OPTIONS }}
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: chromatic-build-artifacts-${{ github.run_id }}
+          path: |
+            chromatic-diagnostics.json
+            **/build-storybook.log

.github/workflows/release-desktop.yml

@@ -61,22 +61,20 @@ jobs:
           SKIP_NX_CACHE: 'true'
 
       - name: Upload core artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: core
           path: packages/frontend/electron/resources/web-static
 
   make-distribution:
     strategy:
-      # all combinations: macos-latest x64, macos-latest arm64, ubuntu-latest x64
-      # For windows, we need a separate approach
       matrix:
         spec:
-          - runner: macos-latest
+          - runner: macos-14
             platform: darwin
             arch: x64
             target: x86_64-apple-darwin
-          - runner: macos-latest
+          - runner: macos-14
             platform: darwin
             arch: arm64
             target: aarch64-apple-darwin
@@ -102,7 +100,6 @@ jobs:
         with:
           extra-flags: workspaces focus @affine/electron @affine/monorepo
           hard-link-nm: false
-          build-plugins: false
           nmHoistingLimits: workspaces
           enableScripts: false
       - name: Build AFFiNE native
@@ -111,7 +108,7 @@ jobs:
           target: ${{ matrix.spec.target }}
           package: '@affine/native'
           nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: core
           path: packages/frontend/electron/resources/web-static
@@ -147,15 +144,13 @@ jobs:
           mv packages/frontend/electron/out/*/make/AppImage/x64/*.AppImage ./builds/affine-${{ env.BUILD_TYPE }}-linux-x64.AppImage
 
       - name: Upload Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: affine-${{ matrix.spec.platform }}-${{ matrix.spec.arch }}-builds
           path: builds
 
   package-distribution-windows:
     strategy:
-      # all combinations: macos-latest x64, macos-latest arm64, ubuntu-latest x64
-      # For windows, we need a separate approach
       matrix:
         spec:
           - runner: windows-latest
@@ -179,7 +174,6 @@ jobs:
         with:
           extra-flags: workspaces focus @affine/electron @affine/monorepo
           hard-link-nm: false
-          build-plugins: false
           nmHoistingLimits: workspaces
       - name: Build AFFiNE native
         uses: ./.github/actions/build-rust
@@ -187,14 +181,11 @@ jobs:
           target: ${{ matrix.spec.target }}
           package: '@affine/native'
           nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: core
           path: packages/frontend/electron/resources/web-static
 
-      - name: Build Plugins
-        run: yarn run build:plugins
-
       - name: Build Desktop Layers
         run: yarn workspace @affine/electron build
 
@@ -216,7 +207,7 @@ jobs:
         run: Compress-Archive -CompressionLevel Fastest -Path packages/frontend/electron/out/* -DestinationPath archive.zip
 
       - name: Save packaged artifacts for signing
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: packaged-${{ matrix.spec.platform }}-${{ matrix.spec.arch }}
           path: |
@@ -233,8 +224,6 @@ jobs:
   make-windows-installer:
     needs: sign-packaged-artifacts-windows
     strategy:
-      # all combinations: macos-latest x64, macos-latest arm64, ubuntu-latest x64
-      # For windows, we need a separate approach
       matrix:
         spec:
           - runner: windows-latest
@@ -250,7 +239,7 @@ jobs:
         timeout-minutes: 10
         uses: ./.github/actions/setup-node
       - name: Download and overwrite packaged artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: signed-packaged-${{ matrix.spec.platform }}-${{ matrix.spec.arch }}
           path: .
@@ -271,7 +260,7 @@ jobs:
           echo $FILES_TO_BE_SIGNED
 
       - name: Save installer for signing
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: installer-${{ matrix.spec.platform }}-${{ matrix.spec.arch }}
           path: archive.zip
@@ -286,8 +275,6 @@ jobs:
   finalize-installer-windows:
     needs: sign-installer-artifacts-windows
     strategy:
-      # all combinations: macos-latest x64, macos-latest arm64, ubuntu-latest x64
-      # For windows, we need a separate approach
       matrix:
         spec:
           - runner: windows-latest
@@ -297,7 +284,7 @@ jobs:
     runs-on: ${{ matrix.spec.runner }}
     steps:
       - name: Download and overwrite installer artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: signed-installer-${{ matrix.spec.platform }}-${{ matrix.spec.arch }}
           path: .
@@ -312,7 +299,7 @@ jobs:
           mv packages/frontend/electron/out/*/make/squirrel.windows/x64/*.msi ./builds/affine-${{ env.BUILD_TYPE }}-windows-x64.msi
 
       - name: Upload Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: affine-${{ matrix.spec.platform }}-${{ matrix.spec.arch }}-builds
           path: builds
@@ -323,29 +310,29 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: core
           path: web-static
       - name: Zip web-static
         run: zip -r web-static.zip web-static
       - name: Download Artifacts (macos-x64)
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: affine-darwin-x64-builds
           path: ./
       - name: Download Artifacts (macos-arm64)
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: affine-darwin-arm64-builds
           path: ./
       - name: Download Artifacts (windows-x64)
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: affine-win32-x64-builds
           path: ./
       - name: Download Artifacts (linux-x64)
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: affine-linux-x64-builds
           path: ./
@@ -358,6 +345,7 @@ jobs:
         env:
           RELEASE_VERSION: ${{ needs.before-make.outputs.RELEASE_VERSION }}
       - name: Create Release Draft
+        if: ${{ github.ref_type == 'tag' }}
         uses: softprops/action-gh-release@v1
         with:
           name: ${{ needs.before-make.outputs.RELEASE_VERSION }}
@@ -372,3 +360,25 @@ jobs:
             ./*.AppImage
             ./*.apk
             ./*.yml
+      - name: Create Nightly Release Draft
+        if: ${{ github.ref_type == 'branch' }}
+        uses: softprops/action-gh-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}
+        with:
+          # Temporarily, treat release from branch as nightly release, artifact saved to AFFiNE-Releases.
+          # Need to improve internal build and nightly release logic.
+          repository: 'toeverything/AFFiNE-Releases'
+          name: ${{ needs.before-make.outputs.RELEASE_VERSION }}
+          tag_name: ${{ needs.before-make.outputs.RELEASE_VERSION }}
+          body: ''
+          draft: false
+          prerelease: true
+          files: |
+            ./VERSION
+            ./*.zip
+            ./*.dmg
+            ./*.exe
+            ./*.AppImage
+            ./*.apk
+            ./*.yml

.github/workflows/windows-signer.yml

@@ -14,7 +14,7 @@ jobs:
     env:
       ARCHIVE_DIR: ${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.artifact-name }}
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: ${{ inputs.artifact-name }}
           path: ${{ env.ARCHIVE_DIR }}
@@ -36,7 +36,7 @@ jobs:
           cd ${{ env.ARCHIVE_DIR }}
           7za a signed.zip .\out\*
       - name: upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: signed-${{ inputs.artifact-name }}
           path: ${{ env.ARCHIVE_DIR }}/signed.zip

.github/workflows/workers.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Publish
-        uses: cloudflare/wrangler-action@v3.3.2
+        uses: cloudflare/wrangler-action@v3.4.1
         with:
           apiToken: ${{ secrets.CF_API_TOKEN }}
           accountId: ${{ secrets.CF_ACCOUNT_ID }}

.husky/pre-commit

@@ -1,4 +1 @@
-#!/usr/bin/env sh
-. "$(dirname -- "$0")/_/husky.sh"
-
 yarn lint-staged && yarn lint:ox

.prettierignore

@@ -16,6 +16,8 @@ packages/frontend/i18n/src/i18n-generated.ts
 packages/frontend/graphql/src/graphql/index.ts
 tests/affine-legacy/**/static
 .yarnrc.yml
+packages/frontend/templates/templates.gen.ts
+packages/frontend/templates/onboarding
 
 # auto-generated by NAPI-RS
 # fixme(@joooye34): need script to check and generate ignore list here

.yarnrc.yml

@@ -12,4 +12,4 @@ npmPublishAccess: public
 
 npmPublishRegistry: "https://registry.npmjs.org"
 
-yarnPath: .yarn/releases/yarn-4.0.2.cjs
+yarnPath: .yarn/releases/yarn-4.1.0.cjs

README.md

@@ -61,7 +61,7 @@
 
 ## Join our community
 
-Before we tell you how to get started with AFFiNE, we'd like to shamelessly plug our awesome user and developer communities across [official social platforms](https://community.affine.pro/c/start-here/)! Once you’re familiar with using the software, maybe you will share your wisdom with others and even consider joining the [AFFiNE Ambassador program](https://community.affine.pro/c/start-here/affine-ambassador) to help spread AFFiNE to the world.
+Before we tell you how to get started with AFFiNE, we'd like to shamelessly plug our awesome user and developer communities across [official discord server](https://discord.gg/tSNqN4S4)! Once you’re familiar with using the software, maybe you will share your wisdom with others and even consider joining the [AFFiNE Ambassador program](https://community.affine.pro/c/start-here/affine-ambassador) to help spread AFFiNE to the world.
 
 ## Getting started & staying tuned with us.
 
@@ -113,21 +113,6 @@ If you have questions, you are welcome to contact us. One of the best places to
 | [@toeverything/y-indexeddb](packages/common/y-indexeddb) | IndexedDB database adapter for Yjs | [![](https://img.shields.io/npm/dm/@toeverything/y-indexeddb?style=flat-square&color=eee)](https://www.npmjs.com/package/@toeverything/y-indexeddb) |
 | [@toeverything/theme](packages/common/theme)             | AFFiNE theme                       | [![](https://img.shields.io/npm/dm/@toeverything/theme?style=flat-square&color=eee)](https://www.npmjs.com/package/@toeverything/theme)             |
 
-## Plugins
-
-> Plugins are a way to extend the functionality of AFFiNE. You can use plugins to add new blocks, new features, and even new ways to edit content.
->
-> (Currently, the plugin system is under heavy development. You will see the plugin system in the canary release.)
-
-- [@affine/sdk](./packages/common/sdk) - SDK for developing plugins
-- [@affine/plugin-cli](./tools/plugin-cli) - CLI for developing plugins
-
-| Official Plugin                                                  | Description                               | Status |
-| ---------------------------------------------------------------- | ----------------------------------------- | ------ |
-| [@affine/copilot-plugin](./packages/plugins/copilot)             | AI Copilot that help you document writing | 🚧     |
-| [@affine/image-preview-plugin](./packages/plugins/image-preview) | Component for previewing an image         | ✅     |
-| [@affine/outline](./packages/plugins/outline)                    | Outline for your document                 | ✅     |
-
 ## Upstreams
 
 We would also like to give thanks to open-source projects that make AFFiNE possible:
@@ -153,24 +138,6 @@ We would like to express our gratitude to all the individuals who have already c
   <img alt="contributors" src="https://opencollective.com/affine/contributors.svg?width=890&button=false" />
 </a>
 
-## Data Compatibility
-
-Data compatibility is a very important issue for us. We will try our best to ensure that the data is compatible with the previous version.
-
-If you encounter any problems when upgrading the version, please feel free to [contact us](mailto:developer@toeverything.info).
-
-| AFFiNE Version  | Export/Import workspace | Data auto migration |
-| --------------- | ----------------------- | ------------------- |
-| <= 0.5.4        | ❌️                     | ❌                  |
-| 0.6.x           | ✅️                     | ✅                  |
-| 0.7.x           | ✅️                     | ✅                  |
-| 0.8.x (current) | ✅                      | ✅                  |
-| 0.9.x (next)    | 🚧                      | 🚧                  |
-
-- ❌️: Not compatible
-- ✅: Compatible
-- 🚧: Work in progress
-
 ## Self-Host
 
 > We know that the self-host version has been out of date for a long time.
@@ -227,7 +194,7 @@ See [LICENSE] for details.
 [jobs available]: ./docs/jobs.md
 [latest packages]: https://github.com/toeverything/AFFiNE/pkgs/container/affine-self-hosted
 [contributor license agreement]: https://github.com/toeverything/affine/edit/canary/.github/CLA.md
-[rust-version-icon]: https://img.shields.io/badge/Rust-1.74.1-dea584
+[rust-version-icon]: https://img.shields.io/badge/Rust-1.75.0-dea584
 [stars-icon]: https://img.shields.io/github/stars/toeverything/AFFiNE.svg?style=flat&logo=github&colorB=red&label=stars
 [codecov]: https://codecov.io/gh/toeverything/affine/branch/canary/graphs/badge.svg?branch=canary
 [node-version-icon]: https://img.shields.io/badge/node-%3E=18.16.1-success

docs/BUILDING.md

@@ -57,6 +57,29 @@ corepack prepare yarn@stable --activate
 yarn install
 ```
 
+### Clone repository
+
+#### Linux & MacOS
+
+```sh
+git clone https://github.com/toeverything/AFFiNE
+```
+
+#### Windows
+
+In our codebase, we use symbolic links. Due to the security design of Windows, the creation of symbolic links requires administrator privileges. This is part of the security policy settings of Windows, and more information can be found at [Security Policy Settings for Creating Symbolic Links](https://learn.microsoft.com/en-us/windows/security/threat-protection/security-policy-settings/create-symbolic-links).
+
+For detailed guidance on enabling this feature, please refer to the official documentation: [Enable Developer Mode on Windows](https://learn.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development).
+
+Once Developer Mode is enabled, execute the following command with administrator privileges:
+
+```sh
+# Enable symbolic links
+git config --global core.symlinks true
+# Clone the repository, also need to be run with administrator privileges
+git clone https://github.com/toeverything/AFFiNE
+```
+
 ### Build Native Dependencies
 
 Run the following script. It will build the native module at [`/packages/frontend/native`](/packages/frontend/native) and build Node.js binding using [NAPI.rs](https://napi.rs/).
@@ -67,18 +90,6 @@ Note: use `strip` from system instead of `binutils` if you are running MacOS. [s
 yarn workspace @affine/native build
 ```
 
-### Build Infra
-
-```sh
-yarn run build:infra
-```
-
-### Build Plugins
-
-```sh
-yarn run build:plugins
-```
-
 ### Build Server Dependencies
 
 ```sh
@@ -102,7 +113,7 @@ yarn test
 ### E2E Test
 
 ```shell
-# there are `affine-local`, `affine-migration`, `affine-local`, `affine-plugin`, `affine-prototype` e2e tests,
+# there are `affine-local`, `affine-migration`, `affine-local`, `affine-prototype` e2e tests,
 #   which are run under different situations.
 cd tests/affine-local
 yarn e2e

docs/contributing/tutorial.md

@@ -17,7 +17,6 @@ The codebase is organized as follows:
 - `packages/` contains all code running in production.
   - `backend/` contains backend code, more information from <https://github.com/toeverything/OctoBase>.
   - `frontend/` contains frontend code, including the web app, the electron app and business libraries.
-  - `plugins/` contains all build-in plugins.
   - `common` contains the isomorphic code or basic libraries without business.
 - `tools/` contains tools to help developing or CI, not used in production.
 - `tests/` contains testings across different libraries, including e2e testings and integration testings.

docs/reference/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@affine/docs",
+  "type": "module",
+  "private": true,
+  "scripts": {
+    "build": "typedoc --options ../../typedoc.json",
+    "dev": "nodemon --exec 'typedoc --options ../../typedoc.json' & serve dist/"
+  },
+  "devDependencies": {
+    "nodemon": "^3.0.1",
+    "serve": "^14.2.1",
+    "typedoc": "^0.25.4"
+  },
+  "nodemonConfig": {
+    "watch": [
+      "./readme.md",
+      "../../packages/*/*/src/*.ts",
+      "../../**/typedoc{.base,}.json"
+    ],
+    "ext": "ts,md,json"
+  },
+  "version": "0.12.0"
+}

docs/reference/readme.md

@@ -0,0 +1,7 @@
+Welcome to AFFiNE development reference.
+
+This document is intended for developers who want to contribute to AFFiNE. It contains information about the architecture of AFFiNE, how to build it, and how to contribute to it.
+
+### The Infrastructure of AFFiNE
+
+see {@link @toeverything/infra!}

nx.json

@@ -1,12 +1,13 @@
 {
   "$schema": "./node_modules/nx/schemas/nx-schema.json",
   "npmScope": "toeverything",
+  "nxCloudAccessToken": "MzUwNTU4YWItZGFhYi00YjE2LWIxODAtODk4NmIwYjMwYzZkfHJlYWQ=",
   "tasksRunnerOptions": {
     "default": {
       "runner": "nx-cloud",
       "options": {
         "cacheableOperations": ["build", "test", "e2e", "lint"],
-        "accessToken": "YmQ2NTg1ODktZTk5Mi00YzhiLTk2ZmUtNWQzMDg0NDBkOWM3fHJlYWQtb25seQ=="
+        "runtimeCacheInputs": ["node -v"]
       }
     }
   },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@affine/monorepo",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "private": true,
   "author": "toeverything",
   "license": "MIT",
@@ -8,10 +8,9 @@
     ".",
     "packages/*/*",
     "tools/*",
-    "!tools/@types",
+    "docs/reference",
     "tools/@types/*",
     "tests/*",
-    "!tests/affine-legacy",
     "tests/affine-legacy/*"
   ],
   "engines": {
@@ -23,8 +22,6 @@
     "build": "yarn nx build @affine/core",
     "build:electron": "yarn nx build @affine/electron",
     "build:storage": "yarn nx run-many -t build -p @affine/storage",
-    "build:infra": "yarn nx run-many -t build --projects=tag:infra",
-    "build:plugins": "yarn nx run-many -t build --projects=tag:plugin",
     "build:storybook": "yarn nx build @affine/storybook",
     "start:web-static": "yarn workspace @affine/core static-server",
     "start:storybook": "yarn exec serve tests/storybook/storybook-static -l 6006",
@@ -33,14 +30,15 @@
     "lint:eslint:fix": "yarn lint:eslint --fix",
     "lint:prettier": "prettier --ignore-unknown --cache --check .",
     "lint:prettier:fix": "prettier --ignore-unknown --cache --write .",
-    "lint:ox": "oxlint --deny-warnings -D correctness -D nursery -D prefer-array-some -D no-useless-promise-resolve-reject -A no-undef -A consistent-type-exports -A default -A named -A ban-ts-comment",
+    "lint:ox": "oxlint --import-plugin --deny-warnings -D correctness -D nursery -D prefer-array-some -D no-useless-promise-resolve-reject -D perf -A no-undef -A consistent-type-exports -A default -A named -A ban-ts-comment -A export",
     "lint": "yarn lint:eslint && yarn lint:prettier",
     "lint:fix": "yarn lint:eslint:fix && yarn lint:prettier:fix",
     "test": "vitest --run",
     "test:ui": "vitest --ui",
     "test:coverage": "vitest run --coverage",
     "typecheck": "tsc -b tsconfig.json --diagnostics",
-    "postinstall": "node ./scripts/check-version.mjs && yarn i18n-codegen gen && yarn husky install"
+    "postinstall": "node ./scripts/check-version.mjs && yarn i18n-codegen gen && yarn husky install",
+    "prepare": "husky"
   },
   "lint-staged": {
     "*": "prettier --write --ignore-unknown --cache",
@@ -58,16 +56,14 @@
   "devDependencies": {
     "@affine-test/kit": "workspace:*",
     "@affine/cli": "workspace:*",
-    "@affine/plugin-cli": "workspace:*",
     "@commitlint/cli": "^18.4.3",
     "@commitlint/config-conventional": "^18.4.3",
     "@faker-js/faker": "^8.3.1",
     "@istanbuljs/schema": "^0.1.3",
     "@magic-works/i18n-codegen": "^0.5.0",
-    "@nx/vite": "17.1.3",
-    "@perfsee/sdk": "^1.9.0",
-    "@playwright/test": "^1.40.0",
-    "@taplo/cli": "^0.5.2",
+    "@nx/vite": "17.3.1",
+    "@playwright/test": "^1.41.0",
+    "@taplo/cli": "^0.7.0",
     "@testing-library/react": "^14.1.2",
     "@toeverything/infra": "workspace:*",
     "@types/affine__env": "workspace:*",
@@ -75,32 +71,31 @@
     "@types/node": "^20.9.3",
     "@typescript-eslint/eslint-plugin": "^6.13.1",
     "@typescript-eslint/parser": "^6.13.1",
-    "@vanilla-extract/vite-plugin": "^3.9.2",
+    "@vanilla-extract/vite-plugin": "^4.0.0",
     "@vanilla-extract/webpack-plugin": "^2.3.1",
     "@vitejs/plugin-react-swc": "^3.5.0",
-    "@vitest/coverage-istanbul": "1.0.4",
-    "@vitest/ui": "1.0.4",
-    "electron": "^27.1.0",
+    "@vitest/coverage-istanbul": "1.2.2",
+    "@vitest/ui": "1.2.2",
+    "electron": "^28.2.1",
     "eslint": "^8.54.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-i": "^2.29.0",
     "eslint-plugin-react": "^7.33.2",
     "eslint-plugin-react-hooks": "^4.6.0",
-    "eslint-plugin-simple-import-sort": "^10.0.0",
+    "eslint-plugin-simple-import-sort": "^12.0.0",
     "eslint-plugin-sonarjs": "^0.23.0",
-    "eslint-plugin-unicorn": "^50.0.0",
+    "eslint-plugin-unicorn": "^51.0.0",
     "eslint-plugin-unused-imports": "^3.0.0",
     "eslint-plugin-vue": "^9.18.1",
-    "fake-indexeddb": "5.0.1",
-    "happy-dom": "^12.10.3",
-    "husky": "^8.0.3",
+    "fake-indexeddb": "5.0.2",
+    "happy-dom": "^13.0.0",
+    "husky": "^9.0.6",
     "lint-staged": "^15.1.0",
     "msw": "^2.0.8",
     "nanoid": "^5.0.3",
-    "nx": "^17.1.3",
-    "nx-cloud": "^16.5.2",
+    "nx": "^17.2.8",
     "nyc": "^15.1.0",
-    "oxlint": "0.0.21",
+    "oxlint": "0.0.22",
     "prettier": "^3.1.0",
     "semver": "^7.5.4",
     "serve": "^14.2.1",
@@ -111,11 +106,11 @@
     "vite-plugin-istanbul": "^5.0.0",
     "vite-plugin-static-copy": "^1.0.0",
     "vite-tsconfig-paths": "^4.2.1",
-    "vitest": "1.0.4",
+    "vitest": "1.2.2",
     "vitest-fetch-mock": "^0.2.2",
     "vitest-mock-extended": "^1.3.1"
   },
-  "packageManager": "yarn@4.0.2",
+  "packageManager": "yarn@4.1.0",
   "resolutions": {
     "vite": "^5.0.6",
     "array-buffer-byte-length": "npm:@nolyfill/array-buffer-byte-length@latest",
@@ -176,6 +171,7 @@
     "next-auth@^4.24.5": "patch:next-auth@npm%3A4.24.5#~/.yarn/patches/next-auth-npm-4.24.5-8428e11927.patch",
     "@reforged/maker-appimage/@electron-forge/maker-base": "7.2.0",
     "macos-alias": "npm:macos-alias-building@latest",
-    "fs-xattr": "npm:@napi-rs/xattr@latest"
+    "fs-xattr": "npm:@napi-rs/xattr@latest",
+    "@radix-ui/react-dialog": "npm:@radix-ui/react-dialog@latest"
   }
 }

packages/backend/server/.env.example

@@ -1,8 +1,4 @@
-DATABASE_URL="postgresql://affine@localhost:5432/affine"
-NEXTAUTH_URL="http://localhost:8080"
-OAUTH_EMAIL_SENDER="noreply@toeverything.info"
-OAUTH_EMAIL_LOGIN=""
-OAUTH_EMAIL_PASSWORD=""
-ENABLE_LOCAL_EMAIL="true"
-STRIPE_API_KEY=
-STRIPE_WEBHOOK_KEY=
+# AFFINE_SERVER_PORT=3010
+# AFFINE_SERVER_HOST=app.affine.pro
+# AFFINE_SERVER_HTTPS=true
+# DATABASE_URL="postgres://affine@localhost:5432/affine"

packages/backend/server/migrations/20240103092238_add_workspace_features/migration.sql

@@ -0,0 +1,18 @@
+-- CreateTable
+CREATE TABLE "workspace_features" (
+    "id" SERIAL NOT NULL,
+    "workspace_id" VARCHAR(36) NOT NULL,
+    "feature_id" INTEGER NOT NULL,
+    "reason" VARCHAR NOT NULL,
+    "created_at" TIMESTAMPTZ(6) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "expired_at" TIMESTAMPTZ(6),
+    "activated" BOOLEAN NOT NULL DEFAULT false,
+
+    CONSTRAINT "workspace_features_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "workspace_features" ADD CONSTRAINT "workspace_features_feature_id_fkey" FOREIGN KEY ("feature_id") REFERENCES "features"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "workspace_features" ADD CONSTRAINT "workspace_features_workspace_id_fkey" FOREIGN KEY ("workspace_id") REFERENCES "workspaces"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/backend/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/server",
   "private": true,
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Affine Node.js server",
   "type": "module",
   "bin": {
@@ -9,20 +9,21 @@
   },
   "scripts": {
     "build": "tsc",
-    "start": "node --loader ts-node/esm.mjs --es-module-specifier-resolution node ./src/index.ts",
+    "start": "node --loader ts-node/esm/transpile-only.mjs --es-module-specifier-resolution node ./src/index.ts",
     "dev": "nodemon ./src/index.ts",
     "test": "ava --concurrency 1 --serial",
     "test:coverage": "c8 ava --concurrency 1 --serial",
     "postinstall": "prisma generate",
-    "data-migration": "node --loader ts-node/esm.mjs --es-module-specifier-resolution node ./src/data/app.ts",
-    "predeploy": "yarn prisma migrate deploy && node --es-module-specifier-resolution node ./dist/data/app.js run"
+    "data-migration": "node --loader ts-node/esm/transpile-only.mjs --es-module-specifier-resolution node ./src/data/index.ts",
+    "predeploy": "yarn prisma migrate deploy && node --es-module-specifier-resolution node ./dist/data/index.js run"
   },
   "dependencies": {
     "@apollo/server": "^4.9.5",
     "@auth/prisma-adapter": "^1.0.7",
-    "@aws-sdk/client-s3": "^3.454.0",
+    "@aws-sdk/client-s3": "^3.499.0",
     "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.17.0",
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.1.0",
+    "@google-cloud/opentelemetry-resource-util": "^2.1.0",
     "@keyv/redis": "^2.8.0",
     "@nestjs/apollo": "^12.0.11",
     "@nestjs/common": "^10.2.10",
@@ -32,35 +33,39 @@
     "@nestjs/platform-express": "^10.2.10",
     "@nestjs/platform-socket.io": "^10.2.10",
     "@nestjs/schedule": "^4.0.0",
+    "@nestjs/serve-static": "^4.0.0",
     "@nestjs/throttler": "^5.0.1",
     "@nestjs/websockets": "^10.2.10",
     "@node-rs/argon2": "^1.5.2",
     "@node-rs/crc32": "^1.7.2",
-    "@node-rs/jsonwebtoken": "^0.2.3",
+    "@node-rs/jsonwebtoken": "^0.4.0",
     "@opentelemetry/api": "^1.7.0",
-    "@opentelemetry/core": "^1.18.1",
-    "@opentelemetry/exporter-prometheus": "^0.45.1",
-    "@opentelemetry/exporter-zipkin": "^1.18.1",
-    "@opentelemetry/host-metrics": "^0.34.0",
-    "@opentelemetry/instrumentation": "^0.45.1",
-    "@opentelemetry/instrumentation-graphql": "^0.36.0",
-    "@opentelemetry/instrumentation-http": "^0.45.1",
-    "@opentelemetry/instrumentation-ioredis": "^0.36.0",
-    "@opentelemetry/instrumentation-nestjs-core": "^0.33.3",
-    "@opentelemetry/instrumentation-socket.io": "^0.34.3",
-    "@opentelemetry/resources": "^1.18.1",
-    "@opentelemetry/sdk-metrics": "^1.18.1",
-    "@opentelemetry/sdk-node": "^0.45.1",
-    "@opentelemetry/sdk-trace-node": "^1.18.1",
-    "@prisma/client": "^5.6.0",
-    "@prisma/instrumentation": "^5.6.0",
+    "@opentelemetry/core": "^1.20.0",
+    "@opentelemetry/exporter-prometheus": "^0.48.0",
+    "@opentelemetry/exporter-zipkin": "^1.20.0",
+    "@opentelemetry/host-metrics": "^0.35.0",
+    "@opentelemetry/instrumentation": "^0.48.0",
+    "@opentelemetry/instrumentation-graphql": "^0.37.0",
+    "@opentelemetry/instrumentation-http": "^0.48.0",
+    "@opentelemetry/instrumentation-ioredis": "^0.37.0",
+    "@opentelemetry/instrumentation-nestjs-core": "^0.34.0",
+    "@opentelemetry/instrumentation-socket.io": "^0.36.0",
+    "@opentelemetry/resources": "^1.20.0",
+    "@opentelemetry/sdk-metrics": "^1.20.0",
+    "@opentelemetry/sdk-node": "^0.48.0",
+    "@opentelemetry/sdk-trace-node": "^1.20.0",
+    "@opentelemetry/semantic-conventions": "^1.20.0",
+    "@prisma/client": "^5.7.1",
+    "@prisma/instrumentation": "^5.7.1",
     "@socket.io/redis-adapter": "^8.2.1",
     "cookie-parser": "^1.4.6",
     "dotenv": "^16.3.1",
+    "dotenv-cli": "^7.3.0",
     "express": "^4.18.2",
-    "file-type": "^18.7.0",
+    "file-type": "^19.0.0",
     "get-stream": "^8.0.1",
     "graphql": "^16.8.1",
+    "graphql-scalars": "^1.22.4",
     "graphql-type-json": "^0.3.2",
     "graphql-upload": "^16.0.2",
     "ioredis": "^5.3.2",
@@ -74,9 +79,9 @@
     "on-headers": "^1.0.2",
     "parse-duration": "^1.1.0",
     "pretty-time": "^1.1.0",
-    "prisma": "^5.6.0",
+    "prisma": "^5.7.1",
     "prom-client": "^15.0.0",
-    "reflect-metadata": "^0.1.13",
+    "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1",
     "semver": "^7.5.4",
     "socket.io": "^4.7.2",
@@ -101,10 +106,10 @@
     "@types/on-headers": "^1.0.3",
     "@types/pretty-time": "^1.1.5",
     "@types/sinon": "^17.0.2",
-    "@types/supertest": "^2.0.16",
+    "@types/supertest": "^6.0.0",
     "@types/ws": "^8.5.10",
     "ava": "^6.0.0",
-    "c8": "^8.0.1",
+    "c8": "^9.0.0",
     "nodemon": "^3.0.1",
     "sinon": "^17.0.1",
     "supertest": "^6.3.3",
@@ -112,6 +117,7 @@
     "typescript": "^5.3.2"
   },
   "ava": {
+    "timeout": "1m",
     "extensions": {
       "ts": "module"
     },
@@ -156,7 +162,6 @@
     "env": {
       "TS_NODE_TRANSPILE_ONLY": true,
       "TS_NODE_PROJECT": "./tsconfig.json",
-      "NODE_ENV": "development",
       "DEBUG": "affine:*",
       "FORCE_COLOR": true,
       "DEBUG_COLORS": true

packages/backend/server/schema.prisma

@@ -1,7 +1,7 @@
 generator client {
   provider        = "prisma-client-js"
   binaryTargets   = ["native", "debian-openssl-3.0.x", "linux-arm64-openssl-3.0.x"]
-  previewFeatures = ["metrics", "tracing"]
+  previewFeatures = ["metrics", "tracing", "relationJoins", "nativeDistinct"]
 }
 
 datasource db {
@@ -40,6 +40,7 @@ model Workspace {
   pages           WorkspacePage[]
   permissions     WorkspaceUserPermission[]
   pagePermissions WorkspacePageUserPermission[]
+  features        WorkspaceFeatures[]
 
   @@map("workspaces")
 }
@@ -135,12 +136,39 @@ model UserFeatures {
   // - if we switch the user to another plan, we will set the old plan to deactivated, but dont delete it
   activated Boolean   @default(false)
 
-  user    User     @relation(fields: [userId], references: [id], onDelete: Cascade)
   feature Features @relation(fields: [featureId], references: [id], onDelete: Cascade)
+  user    User     @relation(fields: [userId], references: [id], onDelete: Cascade)
 
   @@map("user_features")
 }
 
+// feature gates is a way to enable/disable features for a workspace
+// for example:
+// - copilet is a feature that allow some users in a workspace to access the copilet feature
+model WorkspaceFeatures {
+  id          Int    @id @default(autoincrement())
+  workspaceId String @map("workspace_id") @db.VarChar(36)
+  featureId   Int    @map("feature_id") @db.Integer
+
+  // we will record the reason why the feature is enabled/disabled
+  // for example:
+  // - copilet_v1: "owner buy the copilet feature package"
+  reason    String    @db.VarChar
+  // record the feature enabled time
+  createdAt DateTime  @default(now()) @map("created_at") @db.Timestamptz(6)
+  // record the quota expired time, pay plan is a subscription, so it will expired
+  expiredAt DateTime? @map("expired_at") @db.Timestamptz(6)
+  // whether the feature is activated
+  // for example:
+  // - if owner unsubscribe a feature package, we will set the feature to deactivated, but dont delete it
+  activated Boolean   @default(false)
+
+  feature   Features  @relation(fields: [featureId], references: [id], onDelete: Cascade)
+  workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)
+
+  @@map("workspace_features")
+}
+
 model Features {
   id        Int      @id @default(autoincrement())
   feature   String   @db.VarChar
@@ -151,7 +179,8 @@ model Features {
   configs   Json     @db.Json
   createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
 
-  UserFeatureGates UserFeatures[]
+  UserFeatureGates  UserFeatures[]
+  WorkspaceFeatures WorkspaceFeatures[]
 
   @@unique([feature, version])
   @@map("features")
@@ -196,6 +225,7 @@ model VerificationToken {
   @@map("verificationtokens")
 }
 
+// deprecated, use [ObjectStorage]
 model Blob {
   id          Int       @id @default(autoincrement()) @db.Integer
   hash        String    @db.VarChar
@@ -210,6 +240,7 @@ model Blob {
   @@map("blobs")
 }
 
+// deprecated, use [ObjectStorage]
 model OptimizedBlob {
   id          Int       @id @default(autoincrement()) @db.Integer
   hash        String    @db.VarChar
@@ -234,7 +265,9 @@ model Snapshot {
   seq         Int      @default(0) @db.Integer
   state       Bytes?   @db.ByteA
   createdAt   DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
-  updatedAt   DateTime @updatedAt @map("updated_at") @db.Timestamptz(6)
+  // the `updated_at` field will not record the time of record changed,
+  // but the created time of last seen update that has been merged into snapshot.
+  updatedAt   DateTime @map("updated_at") @db.Timestamptz(6)
 
   @@id([id, workspaceId])
   @@map("snapshots")

packages/backend/server/scripts/self-host-predeploy.js

@@ -0,0 +1,54 @@
+import { execSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const SELF_HOST_CONFIG_DIR = '/root/.affine/config';
+/**
+ * @type {Array<{ from: string; to?: string, modifier?: (content: string): string }>}
+ */
+const configFiles = [
+  { from: './.env.example', to: '.env' },
+  { from: './dist/config/affine.js', modifier: configCleaner },
+  { from: './dist/config/affine.env.js', modifier: configCleaner },
+];
+
+function configCleaner(content) {
+  return content.replace(
+    /(^\/\/#.*$)|(^\/\/\s+TODO.*$)|("use\sstrict";?)|(^.*eslint-disable.*$)/gm,
+    ''
+  );
+}
+
+function prepare() {
+  fs.mkdirSync(SELF_HOST_CONFIG_DIR, { recursive: true });
+
+  for (const { from, to, modifier } of configFiles) {
+    const targetFileName = to ?? path.parse(from).base;
+    const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, targetFileName);
+    if (!fs.existsSync(targetFilePath)) {
+      console.log(`creating config file [${targetFilePath}].`);
+      if (modifier) {
+        const content = fs.readFileSync(from, 'utf-8');
+        fs.writeFileSync(targetFilePath, modifier(content), 'utf-8');
+      } else {
+        fs.cpSync(from, targetFilePath, {
+          force: false,
+        });
+      }
+    }
+  }
+}
+
+function runPredeployScript() {
+  console.log('running predeploy script.');
+  execSync('yarn predeploy', {
+    env: {
+      ...process.env,
+      NODE_OPTIONS:
+        (process.env.NODE_OPTIONS ?? '') + ' --import ./dist/prelude.js',
+    },
+  });
+}
+
+prepare();
+runPredeployScript();

packages/backend/server/src/app.controller.ts

@@ -1,13 +1,18 @@
 import { Controller, Get } from '@nestjs/common';
 
+import { Config } from './fundamentals/config';
+
 @Controller('/')
 export class AppController {
+  constructor(private readonly config: Config) {}
+
   @Get()
   info() {
-    const version = AFFiNE.version;
     return {
-      compatibility: version,
-      message: `AFFiNE ${version} Server`,
+      compatibility: this.config.version,
+      message: `AFFiNE ${this.config.version} Server`,
+      type: this.config.type,
+      flavor: this.config.flavor,
     };
   }
 }

packages/backend/server/src/app.module.ts

@@ -0,0 +1,171 @@
+import { join } from 'node:path';
+
+import { Logger, Module } from '@nestjs/common';
+import { APP_INTERCEPTOR } from '@nestjs/core';
+import { ScheduleModule } from '@nestjs/schedule';
+import { ServeStaticModule } from '@nestjs/serve-static';
+import { get } from 'lodash-es';
+
+import { AppController } from './app.controller';
+import { AuthModule } from './core/auth';
+import { ADD_ENABLED_FEATURES, ServerConfigModule } from './core/config';
+import { DocModule } from './core/doc';
+import { FeatureModule } from './core/features';
+import { QuotaModule } from './core/quota';
+import { StorageModule } from './core/storage';
+import { SyncModule } from './core/sync';
+import { UsersModule } from './core/users';
+import { WorkspaceModule } from './core/workspaces';
+import { getOptionalModuleMetadata } from './fundamentals';
+import { CacheInterceptor, CacheModule } from './fundamentals/cache';
+import {
+  type AvailablePlugins,
+  Config,
+  ConfigModule,
+} from './fundamentals/config';
+import { EventModule } from './fundamentals/event';
+import { GqlModule } from './fundamentals/graphql';
+import { MailModule } from './fundamentals/mailer';
+import { MetricsModule } from './fundamentals/metrics';
+import { PrismaModule } from './fundamentals/prisma';
+import { SessionModule } from './fundamentals/session';
+import { StorageProviderModule } from './fundamentals/storage';
+import { RateLimiterModule } from './fundamentals/throttler';
+import { WebSocketModule } from './fundamentals/websocket';
+import { pluginsMap } from './plugins';
+
+export const FunctionalityModules = [
+  ConfigModule.forRoot(),
+  ScheduleModule.forRoot(),
+  EventModule,
+  CacheModule,
+  PrismaModule,
+  MetricsModule,
+  RateLimiterModule,
+  SessionModule,
+  MailModule,
+  StorageProviderModule,
+];
+
+export class AppModuleBuilder {
+  private readonly modules: AFFiNEModule[] = [];
+  constructor(private readonly config: Config) {}
+
+  use(...modules: AFFiNEModule[]): this {
+    modules.forEach(m => {
+      const requirements = getOptionalModuleMetadata(m, 'requires');
+      // if condition not set or condition met, include the module
+      if (requirements?.length) {
+        const nonMetRequirements = requirements.filter(c => {
+          const value = get(this.config, c);
+          return (
+            value === undefined ||
+            value === null ||
+            (typeof value === 'string' && value.trim().length === 0)
+          );
+        });
+
+        if (nonMetRequirements.length) {
+          const name = 'module' in m ? m.module.name : m.name;
+          new Logger(name).warn(
+            `${name} is not enabled because of the required configuration is not satisfied.`,
+            'Unsatisfied configuration:',
+            ...nonMetRequirements.map(config => `  AFFiNE.${config}`)
+          );
+          return;
+        }
+      }
+
+      const predicator = getOptionalModuleMetadata(m, 'if');
+      if (predicator && !predicator(this.config)) {
+        return;
+      }
+
+      const contribution = getOptionalModuleMetadata(m, 'contributesTo');
+      if (contribution) {
+        ADD_ENABLED_FEATURES(contribution);
+      }
+      this.modules.push(m);
+    });
+
+    return this;
+  }
+
+  useIf(
+    predicator: (config: Config) => boolean,
+    ...modules: AFFiNEModule[]
+  ): this {
+    if (predicator(this.config)) {
+      this.use(...modules);
+    }
+
+    return this;
+  }
+
+  compile() {
+    @Module({
+      providers: [
+        {
+          provide: APP_INTERCEPTOR,
+          useClass: CacheInterceptor,
+        },
+      ],
+      imports: this.modules,
+      controllers: this.config.isSelfhosted ? [] : [AppController],
+    })
+    class AppModule {}
+
+    return AppModule;
+  }
+}
+
+function buildAppModule() {
+  const factor = new AppModuleBuilder(AFFiNE);
+
+  factor
+    // common fundamental modules
+    .use(...FunctionalityModules)
+    // auth
+    .use(AuthModule)
+
+    // business modules
+    .use(DocModule)
+
+    // sync server only
+    .useIf(config => config.flavor.sync, SyncModule)
+
+    // graphql server only
+    .useIf(
+      config => config.flavor.graphql,
+      ServerConfigModule,
+      WebSocketModule,
+      GqlModule,
+      StorageModule,
+      UsersModule,
+      WorkspaceModule,
+      FeatureModule,
+      QuotaModule
+    )
+
+    // self hosted server only
+    .useIf(
+      config => config.isSelfhosted,
+      ServeStaticModule.forRoot({
+        rootPath: join('/app', 'static'),
+      })
+    );
+
+  // plugin modules
+  AFFiNE.plugins.enabled.forEach(name => {
+    const plugin = pluginsMap.get(name as AvailablePlugins);
+    if (!plugin) {
+      throw new Error(`Unknown plugin ${name}`);
+    }
+
+    factor.use(plugin);
+  });
+
+  return factor.compile();
+}
+
+export const AppModule = buildAppModule();

packages/backend/server/src/app.ts

@@ -1,36 +1,47 @@
-import { Module } from '@nestjs/common';
-import { APP_INTERCEPTOR } from '@nestjs/core';
+import { Type } from '@nestjs/common';
+import { NestFactory } from '@nestjs/core';
+import type { NestExpressApplication } from '@nestjs/platform-express';
+import cookieParser from 'cookie-parser';
+import graphqlUploadExpress from 'graphql-upload/graphqlUploadExpress.mjs';
 
-import { AppController } from './app.controller';
-import { CacheInterceptor, CacheModule } from './cache';
-import { ConfigModule } from './config';
-import { EventModule } from './event';
-import { BusinessModules } from './modules';
-import { AuthModule } from './modules/auth';
-import { PrismaModule } from './prisma';
-import { SessionModule } from './session';
-import { StorageModule } from './storage';
-import { RateLimiterModule } from './throttler';
+import { GlobalExceptionFilter } from './fundamentals';
+import { SocketIoAdapter, SocketIoAdapterImpl } from './fundamentals/websocket';
+import { serverTimingAndCache } from './middleware/timing';
 
-const BasicModules = [
-  PrismaModule,
-  ConfigModule.forRoot(),
-  CacheModule,
-  EventModule,
-  StorageModule.forRoot(),
-  SessionModule,
-  RateLimiterModule,
-  AuthModule,
-];
+export async function createApp() {
+  const { AppModule } = await import('./app.module');
 
-@Module({
-  providers: [
-    {
-      provide: APP_INTERCEPTOR,
-      useClass: CacheInterceptor,
-    },
-  ],
-  imports: [...BasicModules, ...BusinessModules],
-  controllers: [AppController],
-})
-export class AppModule {}
+  const app = await NestFactory.create<NestExpressApplication>(AppModule, {
+    cors: true,
+    rawBody: true,
+    bodyParser: true,
+    logger: AFFiNE.affine.stable ? ['log'] : ['verbose'],
+  });
+
+  app.use(serverTimingAndCache);
+
+  app.use(
+    graphqlUploadExpress({
+      // TODO: dynamic limit by quota
+      maxFileSize: 100 * 1024 * 1024,
+      maxFiles: 5,
+    })
+  );
+
+  app.useGlobalFilters(new GlobalExceptionFilter(app.getHttpAdapter()));
+  app.use(cookieParser());
+
+  if (AFFiNE.flavor.sync) {
+    const SocketIoAdapter = app.get<Type<SocketIoAdapter>>(
+      SocketIoAdapterImpl,
+      {
+        strict: false,
+      }
+    );
+
+    const adapter = new SocketIoAdapter(app);
+    app.useWebSocketAdapter(adapter);
+  }
+
+  return app;
+}

packages/backend/server/src/cache/index.ts

@@ -1,26 +0,0 @@
-import { FactoryProvider, Global, Module } from '@nestjs/common';
-import { Redis } from 'ioredis';
-
-import { Config } from '../config';
-import { LocalCache } from './cache';
-import { RedisCache } from './redis';
-
-const CacheProvider: FactoryProvider = {
-  provide: LocalCache,
-  useFactory: (config: Config) => {
-    return config.redis.enabled
-      ? new RedisCache(new Redis(config.redis))
-      : new LocalCache();
-  },
-  inject: [Config],
-};
-
-@Global()
-@Module({
-  providers: [CacheProvider],
-  exports: [CacheProvider],
-})
-export class CacheModule {}
-export { LocalCache as Cache };
-
-export { CacheInterceptor, MakeCache, PreventCache } from './interceptor';

packages/backend/server/src/config/affine.env.ts

@@ -0,0 +1,41 @@
+// Convenient way to map environment variables to config values.
+AFFiNE.ENV_MAP = {
+  AFFINE_SERVER_PORT: ['port', 'int'],
+  AFFINE_SERVER_HOST: 'host',
+  AFFINE_SERVER_SUB_PATH: 'path',
+  AFFINE_SERVER_HTTPS: ['https', 'boolean'],
+  DATABASE_URL: 'db.url',
+  ENABLE_CAPTCHA: ['auth.captcha.enable', 'boolean'],
+  CAPTCHA_TURNSTILE_SECRET: ['auth.captcha.turnstile.secret', 'string'],
+  OAUTH_GOOGLE_ENABLED: ['auth.oauthProviders.google.enabled', 'boolean'],
+  OAUTH_GOOGLE_CLIENT_ID: 'auth.oauthProviders.google.clientId',
+  OAUTH_GOOGLE_CLIENT_SECRET: 'auth.oauthProviders.google.clientSecret',
+  OAUTH_GITHUB_ENABLED: ['auth.oauthProviders.github.enabled', 'boolean'],
+  OAUTH_GITHUB_CLIENT_ID: 'auth.oauthProviders.github.clientId',
+  OAUTH_GITHUB_CLIENT_SECRET: 'auth.oauthProviders.github.clientSecret',
+  OAUTH_EMAIL_LOGIN: 'auth.email.login',
+  OAUTH_EMAIL_SENDER: 'auth.email.sender',
+  OAUTH_EMAIL_SERVER: 'auth.email.server',
+  OAUTH_EMAIL_PORT: ['auth.email.port', 'int'],
+  OAUTH_EMAIL_PASSWORD: 'auth.email.password',
+  THROTTLE_TTL: ['rateLimiter.ttl', 'int'],
+  THROTTLE_LIMIT: ['rateLimiter.limit', 'int'],
+  REDIS_SERVER_HOST: 'plugins.redis.host',
+  REDIS_SERVER_PORT: ['plugins.redis.port', 'int'],
+  REDIS_SERVER_USER: 'plugins.redis.username',
+  REDIS_SERVER_PASSWORD: 'plugins.redis.password',
+  REDIS_SERVER_DATABASE: ['plugins.redis.db', 'int'],
+  DOC_MERGE_INTERVAL: ['doc.manager.updatePollInterval', 'int'],
+  DOC_MERGE_USE_JWST_CODEC: [
+    'doc.manager.experimentalMergeWithYOcto',
+    'boolean',
+  ],
+  ENABLE_LOCAL_EMAIL: ['auth.localEmail', 'boolean'],
+  STRIPE_API_KEY: 'plugins.payment.stripe.keys.APIKey',
+  STRIPE_WEBHOOK_KEY: 'plugins.payment.stripe.keys.webhookKey',
+  FEATURES_EARLY_ACCESS_PREVIEW: ['featureFlags.earlyAccessPreview', 'boolean'],
+  FEATURES_SYNC_CLIENT_VERSION_CHECK: [
+    'featureFlags.syncClientVersionCheck',
+    'boolean',
+  ],
+};

packages/backend/server/src/config/affine.self.ts

@@ -0,0 +1,46 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+// Custom configurations for AFFiNE Cloud
+// ====================================================================================
+// Q: WHY THIS FILE EXISTS?
+// A: AFFiNE deployment environment may have a lot of custom environment variables,
+//    which are not suitable to be put in the `affine.ts` file.
+//    For example, AFFiNE Cloud Clusters are deployed on Google Cloud Platform.
+//    We need to enable the `gcloud` plugin to make sure the nodes working well,
+//    but the default selfhost version may not require it.
+//    So it's not a good idea to put such logic in the common `affine.ts` file.
+//
+//    ```
+//    if (AFFiNE.deploy) {
+//      AFFiNE.plugins.use('gcloud');
+//    }
+//    ```
+// ====================================================================================
+const env = process.env;
+
+AFFiNE.metrics.enabled = !AFFiNE.node.test;
+
+if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {
+  AFFiNE.plugins.use('cloudflare-r2', {
+    accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID,
+    credentials: {
+      accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!,
+      secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!,
+    },
+  });
+  AFFiNE.storage.storages.avatar.provider = 'cloudflare-r2';
+  AFFiNE.storage.storages.avatar.bucket = 'account-avatar';
+  AFFiNE.storage.storages.avatar.publicLinkFactory = key =>
+    `https://avatar.affineassets.com/${key}`;
+
+  AFFiNE.storage.storages.blob.provider = 'cloudflare-r2';
+  AFFiNE.storage.storages.blob.bucket = `workspace-blobs-${
+    AFFiNE.affine.canary ? 'canary' : 'prod'
+  }`;
+}
+
+AFFiNE.plugins.use('redis');
+AFFiNE.plugins.use('payment');
+
+if (AFFiNE.deploy) {
+  AFFiNE.plugins.use('gcloud');
+}

packages/backend/server/src/config/affine.ts

@@ -0,0 +1,117 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+//
+// ###############################################################
+// ##                AFFiNE Configuration System                ##
+// ###############################################################
+// Here is the file of all AFFiNE configurations that will affect runtime behavior.
+// Override any configuration here and it will be merged when starting the server.
+// Any changes in this file won't take effect before server restarted.
+//
+//
+// > Configurations merge order
+//   1. load environment variables (`.env` if provided, and from system)
+//   2. load `src/fundamentals/config/default.ts` for all default settings
+//   3. apply `./affine.ts` patches (this file)
+//   4. apply `./affine.env.ts` patches
+//
+//
+// ###############################################################
+// ##                       General settings                    ##
+// ###############################################################
+//
+// /* The unique identity of the server */
+// AFFiNE.serverId = 'some-randome-uuid';
+//
+// /* The name of AFFiNE Server, may show on the UI */
+// AFFiNE.serverName = 'Your Cool AFFiNE Selfhosted Cloud';
+//
+// /* Whether the server is deployed behind a HTTPS proxied environment */
+AFFiNE.https = false;
+// /* Domain of your server that your server will be available at */
+AFFiNE.host = 'localhost';
+// /* The local port of your server that will listen on */
+AFFiNE.port = 3010;
+// /* The sub path of your server */
+// /* For example, if you set `AFFiNE.path = '/affine'`, then the server will be available at `${domain}/affine` */
+// AFFiNE.path = '/affine';
+//
+//
+// ###############################################################
+// ##                       Database settings                   ##
+// ###############################################################
+//
+// /* The URL of the database where most of AFFiNE server data will be stored in */
+// AFFiNE.db.url = 'postgres://user:passsword@localhost:5432/affine';
+//
+//
+// ###############################################################
+// ##                   Server Function settings                ##
+// ###############################################################
+//
+// /* Whether enable metrics and tracing while running the server */
+// /* The metrics will be available at `http://localhost:9464/metrics` with [Prometheus] format exported */
+// AFFiNE.metrics.enabled = true;
+//
+// /* GraphQL configurations that control the behavior of the Apollo Server behind */
+// /* @see https://www.apollographql.com/docs/apollo-server/api/apollo-server */
+// AFFiNE.graphql = {
+//   /* Path to mount GraphQL API */
+//   path: '/graphql',
+//   buildSchemaOptions: {
+//     numberScalarMode: 'integer',
+//   },
+//   /* Whether allow client to query the schema introspection */
+//   introspection: true,
+//   /* Whether enable GraphQL Playground UI */
+//   playground: true,
+// }
+//
+// /* Doc Store & Collaberation */
+// /* How long the buffer time of creating a new history snapshot when doc get updated */
+// AFFiNE.doc.history.interval = 1000 * 60 * 10; // 10 minutes
+//
+// /* Use `y-octo` to merge updates at the same time when merging using Yjs */
+// AFFiNE.doc.manager.experimentalMergeWithYOcto = true;
+//
+// /* How often the manager will start a new turn of merging pending updates into doc snapshot */
+// AFFiNE.doc.manager.updatePollInterval = 1000 * 3;
+//
+//
+// ###############################################################
+// ##                        Plugins settings                   ##
+// ###############################################################
+//
+// /* Redis Plugin */
+// /* Provide caching and session storing backed by Redis. */
+// /* Useful when you deploy AFFiNE server in a cluster. */
+AFFiNE.plugins.use('redis', {
+  /* override options */
+});
+//
+//
+// /* Payment Plugin */
+AFFiNE.plugins.use('payment', {
+  stripe: { keys: {}, apiVersion: '2023-10-16' },
+});
+//
+//
+// /* Cloudflare R2 Plugin */
+// /* Enable if you choose to store workspace blobs or user avatars in Cloudflare R2 Storage Service */
+// AFFiNE.plugins.use('cloudflare-r2', {
+//   accountId: '',
+//   credentials: {
+//     accessKeyId: '',
+//     secretAccessKey: '',
+//   },
+// });
+//
+// /* AWS S3 Plugin */
+// /* Enable if you choose to store workspace blobs or user avatars in AWS S3 Storage Service */
+// AFFiNE.plugins.use('aws-s3', {
+//  credentials: {
+//    accessKeyId: '',
+//    secretAccessKey: '',
+// })
+// /* Update the provider of storages */
+// AFFiNE.storage.storages.blob.provider = 'r2';
+// AFFiNE.storage.storages.avatar.provider = 'r2';

packages/backend/server/src/config/default.ts

@@ -1,231 +0,0 @@
-/// <reference types="../global.d.ts" />
-
-import { createPrivateKey, createPublicKey } from 'node:crypto';
-import { homedir } from 'node:os';
-import { join } from 'node:path';
-
-import parse from 'parse-duration';
-
-import pkg from '../../package.json' assert { type: 'json' };
-import type { AFFiNEConfig, ServerFlavor } from './def';
-import { applyEnvToConfig } from './env';
-
-export const SERVER_FLAVOR = (process.env.SERVER_FLAVOR ??
-  'allinone') as ServerFlavor;
-
-// Don't use this in production
-export const examplePrivateKey = `-----BEGIN EC PRIVATE KEY-----
-MHcCAQEEIEtyAJLIULkphVhqXqxk4Nr8Ggty3XLwUJWBxzAWCWTMoAoGCCqGSM49
-AwEHoUQDQgAEF3U/0wIeJ3jRKXeFKqQyBKlr9F7xaAUScRrAuSP33rajm3cdfihI
-3JvMxVNsS2lE8PSGQrvDrJZaDo0L+Lq9Gg==
------END EC PRIVATE KEY-----`;
-
-const jwtKeyPair = (function () {
-  const AUTH_PRIVATE_KEY = process.env.AUTH_PRIVATE_KEY ?? examplePrivateKey;
-  const privateKey = createPrivateKey({
-    key: Buffer.from(AUTH_PRIVATE_KEY),
-    format: 'pem',
-    type: 'sec1',
-  })
-    .export({
-      format: 'pem',
-      type: 'pkcs8',
-    })
-    .toString('utf8');
-  const publicKey = createPublicKey({
-    key: Buffer.from(AUTH_PRIVATE_KEY),
-    format: 'pem',
-    type: 'spki',
-  })
-    .export({
-      format: 'pem',
-      type: 'spki',
-    })
-    .toString('utf8');
-
-  return {
-    publicKey,
-    privateKey,
-  };
-})();
-
-export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
-  const defaultConfig = {
-    serverId: 'affine-nestjs-server',
-    version: pkg.version,
-    ENV_MAP: {
-      AFFINE_SERVER_PORT: ['port', 'int'],
-      AFFINE_SERVER_HOST: 'host',
-      AFFINE_SERVER_SUB_PATH: 'path',
-      AFFINE_ENV: 'affineEnv',
-      DATABASE_URL: 'db.url',
-      ENABLE_R2_OBJECT_STORAGE: ['objectStorage.r2.enabled', 'boolean'],
-      R2_OBJECT_STORAGE_ACCOUNT_ID: 'objectStorage.r2.accountId',
-      R2_OBJECT_STORAGE_ACCESS_KEY_ID: 'objectStorage.r2.accessKeyId',
-      R2_OBJECT_STORAGE_SECRET_ACCESS_KEY: 'objectStorage.r2.secretAccessKey',
-      R2_OBJECT_STORAGE_BUCKET: 'objectStorage.r2.bucket',
-      ENABLE_CAPTCHA: ['auth.captcha.enable', 'boolean'],
-      CAPTCHA_TURNSTILE_SECRET: ['auth.captcha.turnstile.secret', 'string'],
-      OAUTH_GOOGLE_ENABLED: ['auth.oauthProviders.google.enabled', 'boolean'],
-      OAUTH_GOOGLE_CLIENT_ID: 'auth.oauthProviders.google.clientId',
-      OAUTH_GOOGLE_CLIENT_SECRET: 'auth.oauthProviders.google.clientSecret',
-      OAUTH_GITHUB_ENABLED: ['auth.oauthProviders.github.enabled', 'boolean'],
-      OAUTH_GITHUB_CLIENT_ID: 'auth.oauthProviders.github.clientId',
-      OAUTH_GITHUB_CLIENT_SECRET: 'auth.oauthProviders.github.clientSecret',
-      OAUTH_EMAIL_LOGIN: 'auth.email.login',
-      OAUTH_EMAIL_SENDER: 'auth.email.sender',
-      OAUTH_EMAIL_SERVER: 'auth.email.server',
-      OAUTH_EMAIL_PORT: ['auth.email.port', 'int'],
-      OAUTH_EMAIL_PASSWORD: 'auth.email.password',
-      THROTTLE_TTL: ['rateLimiter.ttl', 'int'],
-      THROTTLE_LIMIT: ['rateLimiter.limit', 'int'],
-      REDIS_SERVER_ENABLED: ['redis.enabled', 'boolean'],
-      REDIS_SERVER_HOST: 'redis.host',
-      REDIS_SERVER_PORT: ['redis.port', 'int'],
-      REDIS_SERVER_USER: 'redis.username',
-      REDIS_SERVER_PASSWORD: 'redis.password',
-      REDIS_SERVER_DATABASE: ['redis.database', 'int'],
-      DOC_MERGE_INTERVAL: ['doc.manager.updatePollInterval', 'int'],
-      DOC_MERGE_USE_JWST_CODEC: [
-        'doc.manager.experimentalMergeWithJwstCodec',
-        'boolean',
-      ],
-      ENABLE_LOCAL_EMAIL: ['auth.localEmail', 'boolean'],
-      STRIPE_API_KEY: 'payment.stripe.keys.APIKey',
-      STRIPE_WEBHOOK_KEY: 'payment.stripe.keys.webhookKey',
-      FEATURES_EARLY_ACCESS_PREVIEW: [
-        'featureFlags.earlyAccessPreview',
-        'boolean',
-      ],
-    } satisfies AFFiNEConfig['ENV_MAP'],
-    affineEnv: 'dev',
-    get affine() {
-      const env = this.affineEnv;
-      return {
-        canary: env === 'dev',
-        beta: env === 'beta',
-        stable: env === 'production',
-      };
-    },
-    env: process.env.NODE_ENV ?? 'development',
-    get node() {
-      const env = this.env;
-      return {
-        prod: env === 'production',
-        dev: env === 'development',
-        test: env === 'test',
-      };
-    },
-    get deploy() {
-      return !this.node.dev && !this.node.test;
-    },
-    featureFlags: {
-      earlyAccessPreview: false,
-    },
-    get https() {
-      return !this.node.dev;
-    },
-    host: 'localhost',
-    port: 3010,
-    path: '',
-    db: {
-      url: '',
-    },
-    get origin() {
-      return this.node.dev
-        ? 'http://localhost:8080'
-        : `${this.https ? 'https' : 'http'}://${this.host}${
-            this.host === 'localhost' ? `:${this.port}` : ''
-          }`;
-    },
-    get baseUrl() {
-      return `${this.origin}${this.path}`;
-    },
-    graphql: {
-      buildSchemaOptions: {
-        numberScalarMode: 'integer',
-      },
-      introspection: true,
-      playground: true,
-    },
-    auth: {
-      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-      accessTokenExpiresIn: parse('1h')! / 1000,
-      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-      refreshTokenExpiresIn: parse('7d')! / 1000,
-      leeway: 60,
-      captcha: {
-        enable: false,
-        turnstile: {
-          secret: '1x0000000000000000000000000000000AA',
-        },
-        challenge: {
-          bits: 20,
-        },
-      },
-      privateKey: jwtKeyPair.privateKey,
-      publicKey: jwtKeyPair.publicKey,
-      enableSignup: true,
-      enableOauth: false,
-      get nextAuthSecret() {
-        return this.privateKey;
-      },
-      oauthProviders: {},
-      localEmail: false,
-      email: {
-        server: 'smtp.gmail.com',
-        port: 465,
-        login: '',
-        sender: '',
-        password: '',
-      },
-    },
-    objectStorage: {
-      r2: {
-        enabled: false,
-        bucket: '',
-        accountId: '',
-        accessKeyId: '',
-        secretAccessKey: '',
-      },
-      fs: {
-        path: join(homedir(), '.affine-storage'),
-      },
-    },
-    rateLimiter: {
-      ttl: 60,
-      limit: 60,
-    },
-    redis: {
-      enabled: false,
-      host: '127.0.0.1',
-      port: 6379,
-      username: '',
-      password: '',
-      database: 0,
-    },
-    doc: {
-      manager: {
-        enableUpdateAutoMerging: SERVER_FLAVOR !== 'sync',
-        updatePollInterval: 3000,
-        experimentalMergeWithJwstCodec: false,
-      },
-      history: {
-        interval: 1000 * 60 * 10 /* 10 mins */,
-      },
-    },
-    payment: {
-      stripe: {
-        keys: {
-          APIKey: '',
-          webhookKey: '',
-        },
-        apiVersion: '2023-10-16',
-      },
-    },
-  } satisfies AFFiNEConfig;
-
-  applyEnvToConfig(defaultConfig);
-
-  return defaultConfig;
-};

packages/backend/server/src/config/env.ts

@@ -1,17 +0,0 @@
-import { set } from 'lodash-es';
-
-import { type AFFiNEConfig, parseEnvValue } from './def';
-
-export function applyEnvToConfig(rawConfig: AFFiNEConfig) {
-  for (const env in rawConfig.ENV_MAP) {
-    const config = rawConfig.ENV_MAP[env];
-    const [path, value] =
-      typeof config === 'string'
-        ? [config, process.env[env]]
-        : [config[0], parseEnvValue(process.env[env], config[1])];
-
-    if (value !== undefined) {
-      set(rawConfig, path, value);
-    }
-  }
-}

packages/backend/server/src/constants.ts

@@ -1,3 +0,0 @@
-export const OPERATION_NAME = 'x-operation-name';
-
-export const REQUEST_ID = 'x-request-id';

packages/backend/server/src/core/auth/guard.ts

@@ -4,14 +4,17 @@ import {
   Inject,
   Injectable,
   SetMetadata,
+  UnauthorizedException,
   UseGuards,
 } from '@nestjs/common';
 import { Reflector } from '@nestjs/core';
 import type { NextAuthOptions } from 'next-auth';
 import { AuthHandler } from 'next-auth/core';
 
-import { PrismaService } from '../../prisma';
-import { getRequestResponseFromContext } from '../../utils/nestjs';
+import {
+  getRequestResponseFromContext,
+  PrismaService,
+} from '../../fundamentals';
 import { NextAuthOptionsProvide } from './next-auth-options';
 import { AuthService } from './service';
 
@@ -67,6 +70,10 @@ class AuthGuard implements CanActivate {
       'isPublic',
       context.getHandler()
     );
+
+    // FIXME(@forehalo): @Publicable() is duplicated with @CurrentUser() user?: User
+    //                                                                       ^ optional
+    //   we can prefetch user session in each request even before this `Guard`
     // api can be public, but if user is logged in, we can get user info
     const isPublicable = this.reflector.get<boolean>(
       'isPublicable',
@@ -92,7 +99,7 @@ class AuthGuard implements CanActivate {
 
       const { body = {}, cookies, status = 200 } = session;
       if (!body && !isPublicable) {
-        return false;
+        throw new UnauthorizedException('You are not signed in.');
       }
 
       // @ts-expect-error body is user here

packages/backend/server/src/core/auth/index.ts

@@ -1,7 +1,5 @@
 import { Global, Module } from '@nestjs/common';
 
-import { SessionModule } from '../../session';
-import { MAILER, MailService } from './mailer';
 import { NextAuthController } from './next-auth.controller';
 import { NextAuthOptionsProvider } from './next-auth-options';
 import { AuthResolver } from './resolver';
@@ -9,18 +7,12 @@ import { AuthService } from './service';
 
 @Global()
 @Module({
-  imports: [SessionModule],
-  providers: [
-    AuthService,
-    AuthResolver,
-    NextAuthOptionsProvider,
-    MAILER,
-    MailService,
-  ],
-  exports: [AuthService, NextAuthOptionsProvider, MailService],
+  providers: [AuthService, AuthResolver, NextAuthOptionsProvider],
+  exports: [AuthService, NextAuthOptionsProvider],
   controllers: [NextAuthController],
 })
 export class AuthModule {}
 
 export * from './guard';
 export { TokenType } from './resolver';
+export { AuthService };

packages/backend/server/src/core/auth/next-auth-options.ts

@@ -8,12 +8,14 @@ import Email from 'next-auth/providers/email';
 import Github from 'next-auth/providers/github';
 import Google from 'next-auth/providers/google';
 
-import { Config } from '../../config';
-import { PrismaService } from '../../prisma';
-import { SessionService } from '../../session';
+import {
+  Config,
+  MailService,
+  PrismaService,
+  SessionService,
+} from '../../fundamentals';
 import { FeatureType } from '../features';
-import { Quota_FreePlanV1 } from '../quota';
-import { MailService } from './mailer';
+import { Quota_FreePlanV1_1 } from '../quota';
 import {
   decode,
   encode,
@@ -50,7 +52,7 @@ export const NextAuthOptionsProvider: FactoryProvider<NextAuthOptions> = {
             activated: true,
             feature: {
               connect: {
-                feature_version: Quota_FreePlanV1,
+                feature_version: Quota_FreePlanV1_1,
               },
             },
           },
@@ -179,7 +181,7 @@ export const NextAuthOptionsProvider: FactoryProvider<NextAuthOptions> = {
       );
     }
 
-    if (config.auth.oauthProviders.google) {
+    if (config.auth.oauthProviders.google?.enabled) {
       nextAuthOptions.providers.push(
         // @ts-expect-error esm interop issue
         Google.default({
@@ -242,7 +244,10 @@ export const NextAuthOptionsProvider: FactoryProvider<NextAuthOptions> = {
             .count({
               where: {
                 user: {
-                  email,
+                  email: {
+                    equals: email,
+                    mode: 'insensitive',
+                  },
                 },
                 feature: {
                   feature: FeatureType.EarlyAccess,

packages/backend/server/src/core/auth/next-auth.controller.ts

@@ -22,11 +22,14 @@ import { nanoid } from 'nanoid';
 import type { AuthAction, CookieOption, NextAuthOptions } from 'next-auth';
 import { AuthHandler } from 'next-auth/core';
 
-import { Config } from '../../config';
-import { metrics } from '../../metrics';
-import { PrismaService } from '../../prisma/service';
-import { SessionService } from '../../session';
-import { AuthThrottlerGuard, Throttle } from '../../throttler';
+import {
+  AuthThrottlerGuard,
+  Config,
+  metrics,
+  PrismaService,
+  SessionService,
+  Throttle,
+} from '../../fundamentals';
 import { NextAuthOptionsProvide } from './next-auth-options';
 import { AuthService } from './service';
 
@@ -104,7 +107,10 @@ export class NextAuthController {
       if (email) {
         const user = await this.prisma.user.findFirst({
           where: {
-            email,
+            email: {
+              equals: email,
+              mode: 'insensitive',
+            },
           },
         });
         if (!user) {
@@ -186,15 +192,17 @@ export class NextAuthController {
     }
 
     let nextAuthTokenCookie: (CookieOption & { value: string }) | undefined;
-    const cookiePrefix = this.config.node.prod ? '__Secure-' : '';
-    const sessionCookieName = `${cookiePrefix}next-auth.session-token`;
+    const secureCookiePrefix = '__Secure-';
+    const sessionCookieName = `next-auth.session-token`;
     // next-auth credentials login only support JWT strategy
     // https://next-auth.js.org/configuration/providers/credentials
     // let's store the session token in the database
     if (
       credentialsSignIn &&
       (nextAuthTokenCookie = cookies?.find(
-        ({ name }) => name === sessionCookieName
+        ({ name }) =>
+          name === sessionCookieName ||
+          name === `${secureCookiePrefix}${sessionCookieName}`
       ))
     ) {
       const cookieExpires = new Date();

packages/backend/server/src/core/auth/resolver.ts

@@ -16,9 +16,12 @@ import {
 import type { Request } from 'express';
 import { nanoid } from 'nanoid';
 
-import { Config } from '../../config';
-import { SessionService } from '../../session';
-import { CloudThrottlerGuard, Throttle } from '../../throttler';
+import {
+  CloudThrottlerGuard,
+  Config,
+  SessionService,
+  Throttle,
+} from '../../fundamentals';
 import { UserType } from '../users';
 import { Auth, CurrentUser } from './guard';
 import { AuthService } from './service';
@@ -167,8 +170,13 @@ export class AuthResolver {
     @CurrentUser() user: UserType,
     @Args('token') token: string
   ) {
+    const key = await this.session.get(token);
+    if (!key) {
+      throw new ForbiddenException('Invalid token');
+    }
+
     // email has set token in `sendVerifyChangeEmail`
-    const [id, email] = (await this.session.get(token)).split(',');
+    const [id, email] = key.split(',');
     if (!id || id !== user.id || !email) {
       throw new ForbiddenException('Invalid token');
     }

packages/backend/server/src/core/auth/service.ts

@@ -11,11 +11,13 @@ import { Algorithm, sign, verify as jwtVerify } from '@node-rs/jsonwebtoken';
 import type { User } from '@prisma/client';
 import { nanoid } from 'nanoid';
 
-import { Config } from '../../config';
-import { PrismaService } from '../../prisma';
-import { verifyChallengeResponse } from '../../storage';
-import { Quota_FreePlanV1 } from '../quota';
-import { MailService } from './mailer';
+import {
+  Config,
+  MailService,
+  PrismaService,
+  verifyChallengeResponse,
+} from '../../fundamentals';
+import { Quota_FreePlanV1_1 } from '../quota';
 
 export type UserClaim = Pick<
   User,
@@ -51,7 +53,7 @@ export class AuthService {
         exp: now + this.config.auth.accessTokenExpiresIn,
         iss: this.config.serverId,
         sub: user.id,
-        aud: user.name,
+        aud: 'https://affine.pro',
         jti: randomUUID({
           disableEntropyCache: true,
         }),
@@ -80,7 +82,7 @@ export class AuthService {
         iat: now,
         iss: this.config.serverId,
         sub: user.id,
-        aud: user.name,
+        aud: 'https://affine.pro',
         jti: randomUUID({
           disableEntropyCache: true,
         }),
@@ -100,6 +102,7 @@ export class AuthService {
           iss: [this.config.serverId],
           leeway: this.config.auth.leeway,
           requiredSpecClaims: ['exp', 'iat', 'iss', 'sub'],
+          aud: ['https://affine.pro'],
         })
       ).data as UserClaim;
 
@@ -133,7 +136,7 @@ export class AuthService {
     return (
       !!outcome.success &&
       // skip hostname check in dev mode
-      (this.config.affineEnv === 'dev' || outcome.hostname === this.config.host)
+      (this.config.node.dev || outcome.hostname === this.config.host)
     );
   }
 
@@ -148,7 +151,10 @@ export class AuthService {
   async signIn(email: string, password: string): Promise<User> {
     const user = await this.prisma.user.findFirst({
       where: {
-        email,
+        email: {
+          equals: email,
+          mode: 'insensitive',
+        },
       },
     });
 
@@ -176,7 +182,10 @@ export class AuthService {
   async signUp(name: string, email: string, password: string): Promise<User> {
     const user = await this.prisma.user.findFirst({
       where: {
-        email,
+        email: {
+          equals: email,
+          mode: 'insensitive',
+        },
       },
     });
 
@@ -191,13 +200,14 @@ export class AuthService {
         name,
         email,
         password: hashedPassword,
+        // TODO(@forehalo): handle in event system
         features: {
           create: {
             reason: 'created by api sign up',
             activated: true,
             feature: {
               connect: {
-                feature_version: Quota_FreePlanV1,
+                feature_version: Quota_FreePlanV1_1,
               },
             },
           },
@@ -209,7 +219,10 @@ export class AuthService {
   async createAnonymousUser(email: string): Promise<User> {
     const user = await this.prisma.user.findFirst({
       where: {
-        email,
+        email: {
+          equals: email,
+          mode: 'insensitive',
+        },
       },
     });
 
@@ -227,7 +240,7 @@ export class AuthService {
             activated: true,
             feature: {
               connect: {
-                feature_version: Quota_FreePlanV1,
+                feature_version: Quota_FreePlanV1_1,
               },
             },
           },
@@ -237,9 +250,12 @@ export class AuthService {
   }
 
   async getUserByEmail(email: string): Promise<User | null> {
-    return this.prisma.user.findUnique({
+    return this.prisma.user.findFirst({
       where: {
-        email,
+        email: {
+          equals: email,
+          mode: 'insensitive',
+        },
       },
     });
   }
@@ -247,7 +263,10 @@ export class AuthService {
   async isUserHasPassword(email: string): Promise<boolean> {
     const user = await this.prisma.user.findFirst({
       where: {
-        email,
+        email: {
+          equals: email,
+          mode: 'insensitive',
+        },
       },
     });
     if (!user) {
@@ -257,9 +276,12 @@ export class AuthService {
   }
 
   async changePassword(email: string, newPassword: string): Promise<User> {
-    const user = await this.prisma.user.findUnique({
+    const user = await this.prisma.user.findFirst({
       where: {
-        email,
+        email: {
+          equals: email,
+          mode: 'insensitive',
+        },
         emailVerified: {
           not: null,
         },

packages/backend/server/src/core/auth/utils/jwt.ts

@@ -4,8 +4,7 @@ import { BadRequestException } from '@nestjs/common';
 import { Algorithm, sign, verify as jwtVerify } from '@node-rs/jsonwebtoken';
 import { JWT } from 'next-auth/jwt';
 
-import { Config } from '../../../config';
-import { PrismaService } from '../../../prisma';
+import { Config, PrismaService } from '../../../fundamentals';
 import { getUtcTimestamp, UserClaim } from '../service';
 
 export const jwtEncode = async (
@@ -38,7 +37,7 @@ export const jwtEncode = async (
       exp: now + (maxAge ?? config.auth.accessTokenExpiresIn),
       iss: config.serverId,
       sub: user.id,
-      aud: user.name,
+      aud: 'https://affine.pro',
       jti: randomUUID({
         disableEntropyCache: true,
       }),

packages/backend/server/src/core/auth/utils/send-mail.ts

@@ -2,9 +2,7 @@ import { Logger } from '@nestjs/common';
 import { nanoid } from 'nanoid';
 import type { SendVerificationRequestParams } from 'next-auth/providers/email';
 
-import { Config } from '../../../config';
-import { SessionService } from '../../../session';
-import { MailService } from '../mailer';
+import { Config, MailService, SessionService } from '../../../fundamentals';
 
 export async function sendVerificationRequest(
   config: Config,

packages/backend/server/src/core/config.ts

@@ -0,0 +1,71 @@
+import { Module } from '@nestjs/common';
+import { Field, ObjectType, Query, registerEnumType } from '@nestjs/graphql';
+
+import { DeploymentType } from '../fundamentals';
+
+export enum ServerFeature {
+  Payment = 'payment',
+}
+
+registerEnumType(ServerFeature, {
+  name: 'ServerFeature',
+});
+
+registerEnumType(DeploymentType, {
+  name: 'ServerDeploymentType',
+});
+
+const ENABLED_FEATURES: ServerFeature[] = [];
+export function ADD_ENABLED_FEATURES(feature: ServerFeature) {
+  ENABLED_FEATURES.push(feature);
+}
+
+@ObjectType()
+export class ServerConfigType {
+  @Field({
+    description:
+      'server identical name could be shown as badge on user interface',
+  })
+  name!: string;
+
+  @Field({ description: 'server version' })
+  version!: string;
+
+  @Field({ description: 'server base url' })
+  baseUrl!: string;
+
+  @Field(() => DeploymentType, { description: 'server type' })
+  type!: DeploymentType;
+
+  /**
+   * @deprecated
+   */
+  @Field({ description: 'server flavor', deprecationReason: 'use `features`' })
+  flavor!: string;
+
+  @Field(() => [ServerFeature], { description: 'enabled server features' })
+  features!: ServerFeature[];
+}
+export class ServerConfigResolver {
+  @Query(() => ServerConfigType, {
+    description: 'server config',
+  })
+  serverConfig(): ServerConfigType {
+    return {
+      name: AFFiNE.serverName,
+      version: AFFiNE.version,
+      baseUrl: AFFiNE.baseUrl,
+      type: AFFiNE.type,
+      // BACKWARD COMPATIBILITY
+      // the old flavors contains `selfhosted` but it actually not flavor but deployment type
+      // this field should be removed after frontend feature flags implemented
+      flavor: AFFiNE.type,
+      features: ENABLED_FEATURES,
+    };
+  }
+}
+
+@Module({
+  providers: [ServerConfigResolver],
+})
+export class ServerConfigModule {}

packages/backend/server/src/core/doc/history.ts

@@ -3,10 +3,13 @@ import { isDeepStrictEqual } from 'node:util';
 import { Injectable, Logger } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 
-import { Config } from '../../config';
-import { type EventPayload, OnEvent } from '../../event';
-import { metrics } from '../../metrics';
-import { PrismaService } from '../../prisma';
+import {
+  Config,
+  type EventPayload,
+  metrics,
+  OnEvent,
+  PrismaService,
+} from '../../fundamentals';
 import { QuotaService } from '../quota';
 import { Permission } from '../workspaces/types';
 import { isEmptyBuffer } from './manager';

packages/backend/server/src/core/doc/manager.ts

@@ -4,24 +4,29 @@ import {
   OnModuleDestroy,
   OnModuleInit,
 } from '@nestjs/common';
+import { Cron, CronExpression } from '@nestjs/schedule';
 import { Snapshot, Update } from '@prisma/client';
 import { chunk } from 'lodash-es';
 import { defer, retry } from 'rxjs';
 import {
   applyUpdate,
-  decodeStateVector,
   Doc,
   encodeStateAsUpdate,
   encodeStateVector,
   transact,
 } from 'yjs';
 
-import { Cache } from '../../cache';
-import { Config } from '../../config';
-import { EventEmitter, type EventPayload, OnEvent } from '../../event';
-import { metrics } from '../../metrics/metrics';
-import { PrismaService } from '../../prisma';
-import { mergeUpdatesInApplyWay as jwstMergeUpdates } from '../../storage';
+import {
+  Cache,
+  CallTimer,
+  Config,
+  EventEmitter,
+  type EventPayload,
+  mergeUpdatesInApplyWay as jwstMergeUpdates,
+  metrics,
+  OnEvent,
+  PrismaService,
+} from '../../fundamentals';
 
 function compare(yBinary: Buffer, jwstBinary: Buffer, strict = false): boolean {
   if (yBinary.equals(jwstBinary)) {
@@ -40,36 +45,6 @@ function compare(yBinary: Buffer, jwstBinary: Buffer, strict = false): boolean {
   return compare(yBinary, yBinary2, true);
 }
 
-/**
- * Detect whether rhs state is newer than lhs state.
- *
- * How could we tell a state is newer:
- *
- *   i. if the state vector size is larger, it's newer
- *  ii. if the state vector size is same, compare each client's state
- */
-function isStateNewer(lhs: Buffer, rhs: Buffer): boolean {
-  const lhsVector = decodeStateVector(lhs);
-  const rhsVector = decodeStateVector(rhs);
-
-  if (lhsVector.size < rhsVector.size) {
-    return true;
-  }
-
-  for (const [client, state] of lhsVector) {
-    const rstate = rhsVector.get(client);
-    if (!rstate) {
-      return false;
-    }
-
-    if (state < rstate) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
 export function isEmptyBuffer(buf: Buffer): boolean {
   return (
     buf.length === 0 ||
@@ -79,6 +54,7 @@ export function isEmptyBuffer(buf: Buffer): boolean {
 }
 
 const MAX_SEQ_NUM = 0x3fffffff; // u31
+const UPDATES_QUEUE_CACHE_KEY = 'doc:manager:updates';
 
 /**
  * Since we can't directly save all client updates into database, in which way the database will overload,
@@ -113,6 +89,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     this.destroy();
   }
 
+  @CallTimer('doc', 'yjs_recover_updates_to_doc')
   private recoverDoc(...updates: Buffer[]): Promise<Doc> {
     const doc = new Doc();
     const chunks = chunk(updates, 10);
@@ -148,11 +125,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     const doc = await this.recoverDoc(...updates);
 
     // test jwst codec
-    if (
-      this.config.affine.canary &&
-      this.config.doc.manager.experimentalMergeWithJwstCodec &&
-      updates.length < 100 /* avoid overloading */
-    ) {
+    if (this.config.doc.manager.experimentalMergeWithYOcto) {
       metrics.jwst.counter('codec_merge_counter').add(1);
       const yjsResult = Buffer.from(encodeStateAsUpdate(doc));
       let log = false;
@@ -203,7 +176,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     }, this.config.doc.manager.updatePollInterval);
 
     this.logger.log('Automation started');
-    if (this.config.doc.manager.experimentalMergeWithJwstCodec) {
+    if (this.config.doc.manager.experimentalMergeWithYOcto) {
       this.logger.warn(
         'Experimental feature enabled: merge updates with jwst codec is enabled'
       );
@@ -376,7 +349,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     const updates = await this.getUpdates(workspaceId, guid);
 
     if (updates.length) {
-      const doc = await this.squash(updates, snapshot);
+      const doc = await this.squash(snapshot, updates);
       return Buffer.from(encodeStateVector(doc));
     }
 
@@ -409,7 +382,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
       // take it ease, we don't want to overload db and or cpu
       // if we limit the taken number here,
       // user will never see the latest doc if there are too many updates pending to be merged.
-      take: 100,
+      take: this.config.doc.manager.maxUpdatesPullCount,
     });
 
     // perf(memory): avoid sorting in db
@@ -457,80 +430,92 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     });
   }
 
+  /**
+   * @returns whether the snapshot is updated to the latest, `undefined` means the doc to be upserted is outdated.
+   */
+  @CallTimer('doc', 'upsert')
   private async upsert(
     workspaceId: string,
     guid: string,
     doc: Doc,
     // we always delay the snapshot update to avoid db overload,
-    // so the value of `updatedAt` will not be accurate to user's real action time
+    // so the value of auto updated `updatedAt` by db will never be accurate to user's real action time
     updatedAt: Date,
-    initialSeq?: number
+    seq: number
   ) {
-    return this.lockSnapshotForUpsert(workspaceId, guid, async () => {
-      const blob = Buffer.from(encodeStateAsUpdate(doc));
+    const blob = Buffer.from(encodeStateAsUpdate(doc));
 
-      if (isEmptyBuffer(blob)) {
-        return false;
+    if (isEmptyBuffer(blob)) {
+      return undefined;
+    }
+
+    const state = Buffer.from(encodeStateVector(doc));
+
+    // CONCERNS:
+    //   i. Because we save the real user's last seen action time as `updatedAt`,
+    //      it's possible to simply compare the `updatedAt` to determine if the snapshot is older than the one we are going to save.
+    //
+    //  ii. Prisma doesn't support `upsert` with additional `where` condition along side unique constraint.
+    //      In our case, we need to manually check the `updatedAt` to avoid overriding the newer snapshot.
+    //      where: { id_workspaceId: {}, updatedAt: { lt: updatedAt } }
+    //                                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    //
+    // iii. Only set the seq number when creating the snapshot.
+    //      For updating scenario, the seq number will be updated when updates pushed to db.
+    try {
+      const result: { updatedAt: Date }[] = await this.db.$queryRaw`
+        INSERT INTO "snapshots" ("workspace_id", "guid", "blob", "state", "seq", "created_at", "updated_at")
+        VALUES (${workspaceId}, ${guid}, ${blob}, ${state}, ${seq}, DEFAULT, ${updatedAt})
+        ON CONFLICT ("workspace_id", "guid")
+        DO UPDATE SET "blob" = ${blob}, "state" = ${state}, "updated_at" = ${updatedAt}, "seq" = ${seq}
+        WHERE "snapshots"."workspace_id" = ${workspaceId} AND "snapshots"."guid" = ${guid} AND "snapshots"."updated_at" <= ${updatedAt}
+        RETURNING "snapshots"."workspace_id" as "workspaceId", "snapshots"."guid" as "id", "snapshots"."updated_at" as "updatedAt"
+      `;
+
+      // const result = await this.db.snapshot.upsert({
+      //   select: {
+      //     updatedAt: true,
+      //     seq: true,
+      //   },
+      //   where: {
+      //     id_workspaceId: {
+      //       workspaceId,
+      //       id: guid,
+      //     },
+      //     ⬇️ NOT SUPPORTED BY PRISMA YET
+      //     updatedAt: {
+      //       lt: updatedAt,
+      //     },
+      //   },
+      //   update: {
+      //     blob,
+      //     state,
+      //     updatedAt,
+      //   },
+      //   create: {
+      //     workspaceId,
+      //     id: guid,
+      //     blob,
+      //     state,
+      //     updatedAt,
+      //     seq,
+      //   },
+      // });
+
+      // if the condition `snapshot.updatedAt > updatedAt` is true, by which means the snapshot has already been updated by other process,
+      // the updates has been applied to current `doc` must have been seen by the other process as well.
+      // The `updatedSnapshot` will be `undefined` in this case.
+      const updatedSnapshot = result.at(0);
+
+      if (!updatedSnapshot) {
+        return undefined;
       }
 
-      const state = Buffer.from(encodeStateVector(doc));
-
-      return await this.db.$transaction(async db => {
-        const snapshot = await db.snapshot.findUnique({
-          where: {
-            id_workspaceId: {
-              id: guid,
-              workspaceId,
-            },
-          },
-        });
-
-        // update
-        if (snapshot) {
-          // only update if state is newer
-          if (isStateNewer(snapshot.state ?? Buffer.from([0]), state)) {
-            await db.snapshot.update({
-              select: {
-                seq: true,
-              },
-              where: {
-                id_workspaceId: {
-                  workspaceId,
-                  id: guid,
-                },
-              },
-              data: {
-                blob,
-                state,
-                updatedAt,
-              },
-            });
-
-            return true;
-          } else {
-            return false;
-          }
-        } else {
-          // create
-          await db.snapshot.create({
-            select: {
-              seq: true,
-            },
-            data: {
-              id: guid,
-              workspaceId,
-              blob,
-              state,
-              seq: initialSeq,
-              createdAt: updatedAt,
-              updatedAt,
-            },
-          });
-
-          return true;
-        }
-      });
-    });
+      return true;
+    } catch (e) {
+      this.logger.error('Failed to upsert snapshot', e);
+      return false;
+    }
   }
 
   private async _get(
@@ -542,7 +527,7 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
 
     if (updates.length) {
       return {
-        doc: await this.squash(updates, snapshot),
+        doc: await this.squash(snapshot, updates),
       };
     }
 
@@ -553,17 +538,17 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
    * Squash updates into a single update and save it as snapshot,
    * and delete the updates records at the same time.
    */
-  private async squash(updates: Update[], snapshot: Snapshot | null) {
+  @CallTimer('doc', 'squash')
+  private async squash(snapshot: Snapshot | null, updates: Update[]) {
     if (!updates.length) {
       throw new Error('No updates to squash');
     }
-    const first = updates[0];
-    const last = updates[updates.length - 1];
 
-    const { id, workspaceId } = first;
+    const last = updates[updates.length - 1];
+    const { id, workspaceId } = last;
 
     const doc = await this.applyUpdates(
-      first.id,
+      id,
       snapshot ? snapshot.blob : Buffer.from([0, 0]),
       ...updates.map(u => u.blob)
     );
@@ -594,19 +579,24 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
       );
     }
 
-    // always delete updates
-    // the upsert will return false if the state is not newer, so we don't need to worry about it
-    const { count } = await this.db.update.deleteMany({
-      where: {
-        id,
-        workspaceId,
-        seq: {
-          in: updates.map(u => u.seq),
+    // we will keep the updates only if the upsert failed on unknown reason
+    // `done === undefined` means the updates is outdated(have already been merged by other process), safe to be deleted
+    // `done === true` means the upsert is successful, safe to be deleted
+    if (done !== false) {
+      // always delete updates
+      // the upsert will return false if the state is not newer, so we don't need to worry about it
+      const { count } = await this.db.update.deleteMany({
+        where: {
+          id,
+          workspaceId,
+          seq: {
+            in: updates.map(u => u.seq),
+          },
         },
-      },
-    });
+      });
 
-    await this.updateCachedUpdatesCount(workspaceId, id, -count);
+      await this.updateCachedUpdatesCount(workspaceId, id, -count);
+    }
 
     return doc;
   }
@@ -663,26 +653,44 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     count: number
   ) {
     const result = await this.cache.mapIncrease(
-      `doc:manager:updates`,
+      UPDATES_QUEUE_CACHE_KEY,
       `${workspaceId}::${guid}`,
       count
     );
 
     if (result <= 0) {
       await this.cache.mapDelete(
-        `doc:manager:updates`,
+        UPDATES_QUEUE_CACHE_KEY,
         `${workspaceId}::${guid}`
       );
     }
   }
 
   private async getAutoSquashCandidateFromCache() {
-    const key = await this.cache.mapRandomKey('doc:manager:updates');
+    const key = await this.cache.mapRandomKey(UPDATES_QUEUE_CACHE_KEY);
 
     if (key) {
-      const count = await this.cache.mapGet<number>('doc:manager:updates', key);
-      if (typeof count === 'number' && count > 0) {
+      const cachedCount = await this.cache.mapIncrease(
+        UPDATES_QUEUE_CACHE_KEY,
+        key,
+        0
+      );
+
+      if (cachedCount > 0) {
         const [workspaceId, id] = key.split('::');
+        const count = await this.db.update.count({
+          where: {
+            workspaceId,
+            id,
+          },
+        });
+
+        // FIXME(@forehalo): somehow the update count in cache is not accurate
+        if (count === 0) {
+          await this.cache.mapDelete(UPDATES_QUEUE_CACHE_KEY, key);
+
+          return null;
+        }
         return { id, workspaceId };
       }
     }
@@ -690,22 +698,38 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     return null;
   }
 
-  private async doWithLock<T>(lock: string, job: () => Promise<T>) {
+  private async doWithLock<T>(
+    lockScope: string,
+    lockResource: string,
+    job: () => Promise<T>
+  ) {
+    const lock = `lock:${lockScope}:${lockResource}`;
     const acquired = await this.cache.setnx(lock, 1, {
       ttl: 60 * 1000,
     });
+    metrics.doc.counter('lock').add(1, { scope: lockScope });
 
     if (!acquired) {
+      metrics.doc.counter('lock_failed').add(1, { scope: lockScope });
       return;
     }
+    metrics.doc.counter('lock_required').add(1, { scope: lockScope });
 
     try {
       return await job();
     } finally {
-      await this.cache.delete(lock).catch(e => {
-        // safe, the lock will be expired when ttl ends
-        this.logger.error(`Failed to release lock ${lock}`, e);
-      });
+      await this.cache
+        .delete(lock)
+        .then(() => {
+          metrics.doc.counter('lock_released').add(1, { scope: lockScope });
+        })
+        .catch(e => {
+          metrics.doc
+            .counter('lock_release_failed')
+            .add(1, { scope: lockScope });
+          // safe, the lock will be expired when ttl ends
+          this.logger.error(`Failed to release lock ${lock}`, e);
+        });
     }
   }
 
@@ -715,19 +739,16 @@ export class DocManager implements OnModuleInit, OnModuleDestroy {
     job: () => Promise<T>
   ) {
     return this.doWithLock(
-      `doc:manager:updates-lock:${workspaceId}::${guid}`,
+      'doc:manager:updates',
+      `${workspaceId}::${guid}`,
       job
     );
   }
 
-  async lockSnapshotForUpsert<T>(
-    workspaceId: string,
-    guid: string,
-    job: () => Promise<T>
-  ) {
-    return this.doWithLock(
-      `doc:manager:snapshot-lock:${workspaceId}::${guid}`,
-      job
-    );
+  @Cron(CronExpression.EVERY_MINUTE)
+  async reportUpdatesQueueCount() {
+    metrics.doc
+      .gauge('updates_queue_count')
+      .record(await this.db.update.count());
   }
 }

packages/backend/server/src/core/features/feature.ts

@@ -1,4 +1,5 @@
-import { PrismaService } from '../../prisma';
+import { PrismaClient } from '@prisma/client';
+
 import { Feature, FeatureSchema, FeatureType } from './types';
 
 class FeatureConfig {
@@ -19,7 +20,20 @@ class FeatureConfig {
   }
 }
 
+export class CopilotFeatureConfig extends FeatureConfig {
+  override config!: Feature & { feature: FeatureType.Copilot };
+  constructor(data: any) {
+    super(data);
+
+    if (this.config.feature !== FeatureType.Copilot) {
+      throw new Error('Invalid feature config: type is not Copilot');
+    }
+  }
+}
+
 export class EarlyAccessFeatureConfig extends FeatureConfig {
+  override config!: Feature & { feature: FeatureType.EarlyAccess };
+
   constructor(data: any) {
     super(data);
 
@@ -27,27 +41,33 @@ export class EarlyAccessFeatureConfig extends FeatureConfig {
       throw new Error('Invalid feature config: type is not EarlyAccess');
     }
   }
+}
 
-  checkWhiteList(email: string) {
-    for (const domain in this.config.configs.whitelist) {
-      if (email.endsWith(domain)) {
-        return true;
-      }
+export class UnlimitedWorkspaceFeatureConfig extends FeatureConfig {
+  override config!: Feature & { feature: FeatureType.UnlimitedWorkspace };
+
+  constructor(data: any) {
+    super(data);
+
+    if (this.config.feature !== FeatureType.UnlimitedWorkspace) {
+      throw new Error('Invalid feature config: type is not UnlimitedWorkspace');
     }
-    return false;
   }
 }
 
 const FeatureConfigMap = {
+  [FeatureType.Copilot]: CopilotFeatureConfig,
   [FeatureType.EarlyAccess]: EarlyAccessFeatureConfig,
+  [FeatureType.UnlimitedWorkspace]: UnlimitedWorkspaceFeatureConfig,
 };
 
-const FeatureCache = new Map<
-  number,
-  InstanceType<(typeof FeatureConfigMap)[FeatureType]>
->();
+export type FeatureConfigType<F extends FeatureType> = InstanceType<
+  (typeof FeatureConfigMap)[F]
+>;
 
-export async function getFeature(prisma: PrismaService, featureId: number) {
+const FeatureCache = new Map<number, FeatureConfigType<FeatureType>>();
+
+export async function getFeature(prisma: PrismaClient, featureId: number) {
   const cachedQuota = FeatureCache.get(featureId);
 
   if (cachedQuota) {

packages/backend/server/src/core/features/index.ts

@@ -1,6 +1,5 @@
 import { Module } from '@nestjs/common';
 
-import { PrismaService } from '../../prisma';
 import { FeatureManagementService } from './management';
 import { FeatureService } from './service';
 
@@ -18,4 +17,4 @@ export class FeatureModule {}
 
 export { type CommonFeature, commonFeatureSchema } from './types';
 export { FeatureKind, Features, FeatureType } from './types';
-export { FeatureManagementService, FeatureService, PrismaService };
+export { FeatureManagementService, FeatureService };

packages/backend/server/src/core/features/management.ts

@@ -0,0 +1,117 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { Config, PrismaService } from '../../fundamentals';
+import { FeatureService } from './service';
+import { FeatureType } from './types';
+
+const STAFF = ['@toeverything.info'];
+
+@Injectable()
+export class FeatureManagementService {
+  protected logger = new Logger(FeatureManagementService.name);
+
+  constructor(
+    private readonly feature: FeatureService,
+    private readonly prisma: PrismaService,
+    private readonly config: Config
+  ) {}
+
+  // ======== Admin ========
+
+  // todo(@darkskygit): replace this with abac
+  isStaff(email: string) {
+    for (const domain of STAFF) {
+      if (email.endsWith(domain)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  // ======== Early Access ========
+
+  async addEarlyAccess(userId: string) {
+    return this.feature.addUserFeature(
+      userId,
+      FeatureType.EarlyAccess,
+      2,
+      'Early access user'
+    );
+  }
+
+  async removeEarlyAccess(userId: string) {
+    return this.feature.removeUserFeature(userId, FeatureType.EarlyAccess);
+  }
+
+  async listEarlyAccess() {
+    return this.feature.listFeatureUsers(FeatureType.EarlyAccess);
+  }
+
+  async isEarlyAccessUser(email: string) {
+    const user = await this.prisma.user.findFirst({
+      where: {
+        email: {
+          equals: email,
+          mode: 'insensitive',
+        },
+      },
+    });
+    if (user) {
+      const canEarlyAccess = await this.feature
+        .hasUserFeature(user.id, FeatureType.EarlyAccess)
+        .catch(() => false);
+
+      return canEarlyAccess;
+    }
+    return false;
+  }
+
+  /// check early access by email
+  async canEarlyAccess(email: string) {
+    if (this.config.featureFlags.earlyAccessPreview && !this.isStaff(email)) {
+      return this.isEarlyAccessUser(email);
+    } else {
+      return true;
+    }
+  }
+
+  // ======== Workspace Feature ========
+  async addWorkspaceFeatures(
+    workspaceId: string,
+    feature: FeatureType,
+    version?: number,
+    reason?: string
+  ) {
+    const latestVersions = await this.feature.getFeaturesVersion();
+    // use latest version if not specified
+    const latestVersion = version || latestVersions[feature];
+    if (!Number.isInteger(latestVersion)) {
+      throw new Error(`Version of feature ${feature} not found`);
+    }
+    return this.feature.addWorkspaceFeature(
+      workspaceId,
+      feature,
+      latestVersion,
+      reason || 'add feature by api'
+    );
+  }
+
+  async getWorkspaceFeatures(workspaceId: string) {
+    const features = await this.feature.getWorkspaceFeatures(workspaceId);
+    return features.filter(f => f.activated).map(f => f.feature.name);
+  }
+
+  async hasWorkspaceFeature(workspaceId: string, feature: FeatureType) {
+    return this.feature.hasWorkspaceFeature(workspaceId, feature);
+  }
+
+  async removeWorkspaceFeature(workspaceId: string, feature: FeatureType) {
+    return this.feature
+      .removeWorkspaceFeature(workspaceId, feature)
+      .then(c => c > 0);
+  }
+
+  async listFeatureWorkspaces(feature: FeatureType) {
+    return this.feature.listFeatureWorkspaces(feature);
+  }
+}

packages/backend/server/src/core/features/service.ts

@@ -0,0 +1,342 @@
+import { Injectable } from '@nestjs/common';
+
+import { PrismaService } from '../../fundamentals';
+import { UserType } from '../users/types';
+import { WorkspaceType } from '../workspaces/types';
+import { FeatureConfigType, getFeature } from './feature';
+import { FeatureKind, FeatureType } from './types';
+
+@Injectable()
+export class FeatureService {
+  constructor(private readonly prisma: PrismaService) {}
+
+  async getFeaturesVersion() {
+    const features = await this.prisma.features.findMany({
+      where: {
+        type: FeatureKind.Feature,
+      },
+      select: {
+        feature: true,
+        version: true,
+      },
+    });
+    return features.reduce(
+      (acc, feature) => {
+        // only keep the latest version
+        if (acc[feature.feature]) {
+          if (acc[feature.feature] < feature.version) {
+            acc[feature.feature] = feature.version;
+          }
+        } else {
+          acc[feature.feature] = feature.version;
+        }
+        return acc;
+      },
+      {} as Record<string, number>
+    );
+  }
+
+  async getFeature<F extends FeatureType>(
+    feature: F
+  ): Promise<FeatureConfigType<F> | undefined> {
+    const data = await this.prisma.features.findFirst({
+      where: {
+        feature,
+        type: FeatureKind.Feature,
+      },
+      select: { id: true },
+      orderBy: {
+        version: 'desc',
+      },
+    });
+    if (data) {
+      return getFeature(this.prisma, data.id) as FeatureConfigType<F>;
+    }
+    return undefined;
+  }
+
+  // ======== User Features ========
+
+  async addUserFeature(
+    userId: string,
+    feature: FeatureType,
+    version: number,
+    reason: string,
+    expiredAt?: Date | string
+  ) {
+    return this.prisma.$transaction(async tx => {
+      const latestFlag = await tx.userFeatures.findFirst({
+        where: {
+          userId,
+          feature: {
+            feature,
+            type: FeatureKind.Feature,
+          },
+          activated: true,
+        },
+        orderBy: {
+          createdAt: 'desc',
+        },
+      });
+      if (latestFlag) {
+        return latestFlag.id;
+      } else {
+        return tx.userFeatures
+          .create({
+            data: {
+              reason,
+              expiredAt,
+              activated: true,
+              user: {
+                connect: {
+                  id: userId,
+                },
+              },
+              feature: {
+                connect: {
+                  feature_version: {
+                    feature,
+                    version,
+                  },
+                  type: FeatureKind.Feature,
+                },
+              },
+            },
+          })
+          .then(r => r.id);
+      }
+    });
+  }
+
+  async removeUserFeature(userId: string, feature: FeatureType) {
+    return this.prisma.userFeatures
+      .updateMany({
+        where: {
+          userId,
+          feature: {
+            feature,
+            type: FeatureKind.Feature,
+          },
+          activated: true,
+        },
+        data: {
+          activated: false,
+        },
+      })
+      .then(r => r.count);
+  }
+
+  /**
+   * get user's features, will included inactivated features
+   * @param userId user id
+   * @returns list of features
+   */
+  async getUserFeatures(userId: string) {
+    const features = await this.prisma.userFeatures.findMany({
+      where: {
+        user: { id: userId },
+        feature: {
+          type: FeatureKind.Feature,
+        },
+      },
+      select: {
+        activated: true,
+        reason: true,
+        createdAt: true,
+        expiredAt: true,
+        featureId: true,
+      },
+    });
+
+    const configs = await Promise.all(
+      features.map(async feature => ({
+        ...feature,
+        feature: await getFeature(this.prisma, feature.featureId),
+      }))
+    );
+
+    return configs.filter(feature => !!feature.feature);
+  }
+
+  async listFeatureUsers(feature: FeatureType): Promise<UserType[]> {
+    return this.prisma.userFeatures
+      .findMany({
+        where: {
+          activated: true,
+          feature: {
+            feature: feature,
+            type: FeatureKind.Feature,
+          },
+        },
+        select: {
+          user: {
+            select: {
+              id: true,
+              name: true,
+              avatarUrl: true,
+              email: true,
+              emailVerified: true,
+              createdAt: true,
+            },
+          },
+        },
+      })
+      .then(users => users.map(user => user.user));
+  }
+
+  async hasUserFeature(userId: string, feature: FeatureType) {
+    return this.prisma.userFeatures
+      .count({
+        where: {
+          userId,
+          activated: true,
+          feature: {
+            feature,
+            type: FeatureKind.Feature,
+          },
+        },
+      })
+      .then(count => count > 0);
+  }
+
+  // ======== Workspace Features ========
+
+  async addWorkspaceFeature(
+    workspaceId: string,
+    feature: FeatureType,
+    version: number,
+    reason: string,
+    expiredAt?: Date | string
+  ) {
+    return this.prisma.$transaction(async tx => {
+      const latestFlag = await tx.workspaceFeatures.findFirst({
+        where: {
+          workspaceId,
+          feature: {
+            feature,
+            type: FeatureKind.Feature,
+          },
+          activated: true,
+        },
+        orderBy: {
+          createdAt: 'desc',
+        },
+      });
+      if (latestFlag) {
+        return latestFlag.id;
+      } else {
+        return tx.workspaceFeatures
+          .create({
+            data: {
+              reason,
+              expiredAt,
+              activated: true,
+              workspace: {
+                connect: {
+                  id: workspaceId,
+                },
+              },
+              feature: {
+                connect: {
+                  feature_version: {
+                    feature,
+                    version,
+                  },
+                  type: FeatureKind.Feature,
+                },
+              },
+            },
+          })
+          .then(r => r.id);
+      }
+    });
+  }
+
+  async removeWorkspaceFeature(workspaceId: string, feature: FeatureType) {
+    return this.prisma.workspaceFeatures
+      .updateMany({
+        where: {
+          workspaceId,
+          feature: {
+            feature,
+            type: FeatureKind.Feature,
+          },
+          activated: true,
+        },
+        data: {
+          activated: false,
+        },
+      })
+      .then(r => r.count);
+  }
+
+  /**
+   * get workspace's features, will included inactivated features
+   * @param workspaceId workspace id
+   * @returns list of features
+   */
+  async getWorkspaceFeatures(workspaceId: string) {
+    const features = await this.prisma.workspaceFeatures.findMany({
+      where: {
+        workspace: { id: workspaceId },
+        feature: {
+          type: FeatureKind.Feature,
+        },
+      },
+      select: {
+        activated: true,
+        reason: true,
+        createdAt: true,
+        expiredAt: true,
+        featureId: true,
+      },
+    });
+
+    const configs = await Promise.all(
+      features.map(async feature => ({
+        ...feature,
+        feature: await getFeature(this.prisma, feature.featureId),
+      }))
+    );
+
+    return configs.filter(feature => !!feature.feature);
+  }
+
+  async listFeatureWorkspaces(feature: FeatureType): Promise<WorkspaceType[]> {
+    return this.prisma.workspaceFeatures
+      .findMany({
+        where: {
+          activated: true,
+          feature: {
+            feature: feature,
+            type: FeatureKind.Feature,
+          },
+        },
+        select: {
+          workspace: {
+            select: {
+              id: true,
+              public: true,
+              createdAt: true,
+            },
+          },
+        },
+      })
+      .then(wss => wss.map(ws => ws.workspace as WorkspaceType));
+  }
+
+  async hasWorkspaceFeature(workspaceId: string, feature: FeatureType) {
+    return this.prisma.workspaceFeatures
+      .count({
+        where: {
+          workspaceId,
+          activated: true,
+          feature: {
+            feature,
+            type: FeatureKind.Feature,
+          },
+        },
+      })
+      .then(count => count > 0);
+  }
+}

packages/backend/server/src/core/features/types/common.ts

@@ -0,0 +1,12 @@
+import { registerEnumType } from '@nestjs/graphql';
+
+export enum FeatureType {
+  Copilot = 'copilot',
+  EarlyAccess = 'early_access',
+  UnlimitedWorkspace = 'unlimited_workspace',
+}
+
+registerEnumType(FeatureType, {
+  name: 'FeatureType',
+  description: 'The type of workspace feature',
+});

packages/backend/server/src/core/features/types/copilot.ts

@@ -0,0 +1,8 @@
+import { z } from 'zod';
+
+import { FeatureType } from './common';
+
+export const featureCopilot = z.object({
+  feature: z.literal(FeatureType.Copilot),
+  configs: z.object({}),
+});

packages/backend/server/src/core/features/types/early-access.ts

@@ -0,0 +1,11 @@
+import { z } from 'zod';
+
+import { FeatureType } from './common';
+
+export const featureEarlyAccess = z.object({
+  feature: z.literal(FeatureType.EarlyAccess),
+  configs: z.object({
+    // field polyfill, make it optional in the future
+    whitelist: z.string().array(),
+  }),
+});

packages/backend/server/src/core/features/types/index.ts

@@ -1,7 +1,10 @@
-import { URL } from 'node:url';
-
 import { z } from 'zod';
 
+import { FeatureType } from './common';
+import { featureCopilot } from './copilot';
+import { featureEarlyAccess } from './early-access';
+import { featureUnlimitedWorkspace } from './unlimited-workspace';
+
 /// ======== common schema ========
 
 export enum FeatureKind {
@@ -20,30 +23,13 @@ export type CommonFeature = z.infer<typeof commonFeatureSchema>;
 
 /// ======== feature define ========
 
-export enum FeatureType {
-  EarlyAccess = 'early_access',
-}
-
-function checkHostname(host: string) {
-  try {
-    return new URL(`https://${host}`).hostname === host;
-  } catch (_) {
-    return false;
-  }
-}
-
-const featureEarlyAccess = z.object({
-  feature: z.literal(FeatureType.EarlyAccess),
-  configs: z.object({
-    whitelist: z
-      .string()
-      .startsWith('@')
-      .refine(domain => checkHostname(domain.slice(1)))
-      .array(),
-  }),
-});
-
 export const Features: Feature[] = [
+  {
+    feature: FeatureType.Copilot,
+    type: FeatureKind.Feature,
+    version: 1,
+    configs: {},
+  },
   {
     feature: FeatureType.EarlyAccess,
     type: FeatureKind.Feature,
@@ -52,6 +38,20 @@ export const Features: Feature[] = [
       whitelist: ['@toeverything.info'],
     },
   },
+  {
+    feature: FeatureType.EarlyAccess,
+    type: FeatureKind.Feature,
+    version: 2,
+    configs: {
+      whitelist: [],
+    },
+  },
+  {
+    feature: FeatureType.UnlimitedWorkspace,
+    type: FeatureKind.Feature,
+    version: 1,
+    configs: {},
+  },
 ];
 
 /// ======== schema infer ========
@@ -60,6 +60,14 @@ export const FeatureSchema = commonFeatureSchema
   .extend({
     type: z.literal(FeatureKind.Feature),
   })
-  .and(z.discriminatedUnion('feature', [featureEarlyAccess]));
+  .and(
+    z.discriminatedUnion('feature', [
+      featureCopilot,
+      featureEarlyAccess,
+      featureUnlimitedWorkspace,
+    ])
+  );
 
 export type Feature = z.infer<typeof FeatureSchema>;
+
+export { FeatureType };

packages/backend/server/src/core/features/types/unlimited-workspace.ts

@@ -0,0 +1,8 @@
+import { z } from 'zod';
+
+import { FeatureType } from './common';
+
+export const featureUnlimitedWorkspace = z.object({
+  feature: z.literal(FeatureType.UnlimitedWorkspace),
+  configs: z.object({}),
+});

packages/backend/server/src/core/quota/index.ts

@@ -1,5 +1,7 @@
 import { Module } from '@nestjs/common';
 
+import { FeatureModule } from '../features';
+import { StorageModule } from '../storage';
 import { PermissionService } from '../workspaces/permission';
 import { QuotaService } from './service';
 import { QuotaManagementService } from './storage';
@@ -11,11 +13,12 @@ import { QuotaManagementService } from './storage';
  * - quota statistics
  */
 @Module({
+  imports: [FeatureModule, StorageModule],
   providers: [PermissionService, QuotaService, QuotaManagementService],
   exports: [QuotaService, QuotaManagementService],
 })
 export class QuotaModule {}
 
 export { QuotaManagementService, QuotaService };
-export { Quota_FreePlanV1, Quota_ProPlanV1, Quotas } from './schema';
-export { QuotaType } from './types';
+export { Quota_FreePlanV1_1, Quota_ProPlanV1, Quotas } from './schema';
+export { QuotaQueryType, QuotaType } from './types';

packages/backend/server/src/core/quota/quota.ts

@@ -1,4 +1,4 @@
-import { PrismaService } from '../../prisma';
+import { PrismaService } from '../../fundamentals';
 import { formatDate, formatSize, Quota, QuotaSchema } from './types';
 
 const QuotaCache = new Map<number, QuotaConfig>();
@@ -44,6 +44,10 @@ export class QuotaConfig {
     }
   }
 
+  get version() {
+    return this.config.version;
+  }
+
   /// feature name of quota
   get name() {
     return this.config.feature;
@@ -53,6 +57,12 @@ export class QuotaConfig {
     return this.config.configs.blobLimit;
   }
 
+  get businessBlobLimit() {
+    return (
+      this.config.configs.businessBlobLimit || this.config.configs.blobLimit
+    );
+  }
+
   get storageQuota() {
     return this.config.configs.storageQuota;
   }

packages/backend/server/src/core/quota/schema.ts

@@ -0,0 +1,106 @@
+import { FeatureKind } from '../features';
+import { OneDay, OneGB, OneMB } from './constant';
+import { Quota, QuotaType } from './types';
+
+export const Quotas: Quota[] = [
+  {
+    feature: QuotaType.FreePlanV1,
+    type: FeatureKind.Quota,
+    version: 1,
+    configs: {
+      // quota name
+      name: 'Free',
+      // single blob limit 10MB
+      blobLimit: 10 * OneMB,
+      // total blob limit 10GB
+      storageQuota: 10 * OneGB,
+      // history period of validity 7 days
+      historyPeriod: 7 * OneDay,
+      // member limit 3
+      memberLimit: 3,
+    },
+  },
+  {
+    feature: QuotaType.ProPlanV1,
+    type: FeatureKind.Quota,
+    version: 1,
+    configs: {
+      // quota name
+      name: 'Pro',
+      // single blob limit 100MB
+      blobLimit: 100 * OneMB,
+      // total blob limit 100GB
+      storageQuota: 100 * OneGB,
+      // history period of validity 30 days
+      historyPeriod: 30 * OneDay,
+      // member limit 10
+      memberLimit: 10,
+    },
+  },
+  {
+    feature: QuotaType.RestrictedPlanV1,
+    type: FeatureKind.Quota,
+    version: 1,
+    configs: {
+      // quota name
+      name: 'Restricted',
+      // single blob limit 10MB
+      blobLimit: OneMB,
+      // total blob limit 1GB
+      storageQuota: 10 * OneMB,
+      // history period of validity 30 days
+      historyPeriod: 30 * OneDay,
+      // member limit 10
+      memberLimit: 10,
+    },
+  },
+  {
+    feature: QuotaType.FreePlanV1,
+    type: FeatureKind.Quota,
+    version: 2,
+    configs: {
+      // quota name
+      name: 'Free',
+      // single blob limit 10MB
+      blobLimit: 100 * OneMB,
+      // total blob limit 10GB
+      storageQuota: 10 * OneGB,
+      // history period of validity 7 days
+      historyPeriod: 7 * OneDay,
+      // member limit 3
+      memberLimit: 3,
+    },
+  },
+  {
+    feature: QuotaType.FreePlanV1,
+    type: FeatureKind.Quota,
+    version: 3,
+    configs: {
+      // quota name
+      name: 'Free',
+      // single blob limit 10MB
+      blobLimit: 10 * OneMB,
+      // server limit will larger then client to handle a edge case:
+      // when a user downgrades from pro to free, he can still continue
+      // to upload previously added files that exceed the free limit
+      // NOTE: this is a product decision, may change in future
+      businessBlobLimit: 100 * OneMB,
+      // total blob limit 10GB
+      storageQuota: 10 * OneGB,
+      // history period of validity 7 days
+      historyPeriod: 7 * OneDay,
+      // member limit 3
+      memberLimit: 3,
+    },
+  },
+];
+
+export const Quota_FreePlanV1_1 = {
+  feature: Quotas[4].feature,
+  version: Quotas[4].version,
+};
+
+export const Quota_ProPlanV1 = {
+  feature: Quotas[1].feature,
+  version: Quotas[1].version,
+};

packages/backend/server/src/core/quota/service.ts

@@ -1,10 +1,12 @@
 import { Injectable } from '@nestjs/common';
 
-import { PrismaService } from '../../prisma';
+import { type EventPayload, OnEvent, PrismaService } from '../../fundamentals';
 import { FeatureKind } from '../features';
 import { QuotaConfig } from './quota';
 import { QuotaType } from './types';
 
+type Transaction = Parameters<Parameters<PrismaService['$transaction']>[0]>[0];
+
 @Injectable()
 export class QuotaService {
   constructor(private readonly prisma: PrismaService) {}
@@ -83,6 +85,13 @@ export class QuotaService {
     expiredAt?: Date
   ) {
     await this.prisma.$transaction(async tx => {
+      const hasSameActivatedQuota = await this.hasQuota(userId, quota, tx);
+
+      if (hasSameActivatedQuota) {
+        // don't need to switch
+        return;
+      }
+
       const latestPlanVersion = await tx.features.aggregate({
         where: {
           feature: quota,
@@ -130,8 +139,10 @@ export class QuotaService {
     });
   }
 
-  async hasQuota(userId: string, quota: QuotaType) {
-    return this.prisma.userFeatures
+  async hasQuota(userId: string, quota: QuotaType, transaction?: Transaction) {
+    const executor = transaction ?? this.prisma;
+
+    return executor.userFeatures
       .count({
         where: {
           userId,
@@ -144,4 +155,26 @@ export class QuotaService {
       })
       .then(count => count > 0);
   }
+
+  @OnEvent('user.subscription.activated')
+  async onSubscriptionUpdated({
+    userId,
+  }: EventPayload<'user.subscription.activated'>) {
+    await this.switchUserQuota(
+      userId,
+      QuotaType.ProPlanV1,
+      'subscription activated'
+    );
+  }
+
+  @OnEvent('user.subscription.canceled')
+  async onSubscriptionCanceled(
+    userId: EventPayload<'user.subscription.canceled'>
+  ) {
+    await this.switchUserQuota(
+      userId,
+      QuotaType.FreePlanV1,
+      'subscription canceled'
+    );
+  }
 }

packages/backend/server/src/core/quota/storage.ts

@@ -0,0 +1,112 @@
+import { Injectable, NotFoundException } from '@nestjs/common';
+
+import { FeatureService, FeatureType } from '../features';
+import { WorkspaceBlobStorage } from '../storage';
+import { PermissionService } from '../workspaces/permission';
+import { OneGB } from './constant';
+import { QuotaService } from './service';
+import { formatSize, QuotaQueryType } from './types';
+
+type QuotaBusinessType = QuotaQueryType & { businessBlobLimit: number };
+
+@Injectable()
+export class QuotaManagementService {
+  constructor(
+    private readonly feature: FeatureService,
+    private readonly quota: QuotaService,
+    private readonly permissions: PermissionService,
+    private readonly storage: WorkspaceBlobStorage
+  ) {}
+
+  async getUserQuota(userId: string) {
+    const quota = await this.quota.getUserQuota(userId);
+
+    return {
+      name: quota.feature.name,
+      reason: quota.reason,
+      createAt: quota.createdAt,
+      expiredAt: quota.expiredAt,
+      blobLimit: quota.feature.blobLimit,
+      businessBlobLimit: quota.feature.businessBlobLimit,
+      storageQuota: quota.feature.storageQuota,
+      historyPeriod: quota.feature.historyPeriod,
+      memberLimit: quota.feature.memberLimit,
+    };
+  }
+
+  // TODO: lazy calc, need to be optimized with cache
+  async getUserUsage(userId: string) {
+    const workspaces = await this.permissions.getOwnedWorkspaces(userId);
+
+    const sizes = await Promise.all(
+      workspaces.map(workspace => this.storage.totalSize(workspace))
+    );
+
+    return sizes.reduce((total, size) => total + size, 0);
+  }
+
+  // get workspace's owner quota and total size of used
+  // quota was apply to owner's account
+  async getWorkspaceUsage(workspaceId: string): Promise<QuotaBusinessType> {
+    const { user: owner } =
+      await this.permissions.getWorkspaceOwner(workspaceId);
+    if (!owner) throw new NotFoundException('Workspace owner not found');
+    const {
+      feature: {
+        name,
+        blobLimit,
+        businessBlobLimit,
+        historyPeriod,
+        memberLimit,
+        storageQuota,
+        humanReadable,
+      },
+    } = await this.quota.getUserQuota(owner.id);
+    // get all workspaces size of owner used
+    const usedSize = await this.getUserUsage(owner.id);
+
+    const quota = {
+      name,
+      blobLimit,
+      businessBlobLimit,
+      historyPeriod,
+      memberLimit,
+      storageQuota,
+      humanReadable,
+      usedSize,
+    };
+
+    // relax restrictions if workspace has unlimited feature
+    // todo(@darkskygit): need a mechanism to allow feature as a middleware to edit quota
+    const unlimited = await this.feature.hasWorkspaceFeature(
+      workspaceId,
+      FeatureType.UnlimitedWorkspace
+    );
+    if (unlimited) {
+      return this.mergeUnlimitedQuota(quota);
+    }
+
+    return quota;
+  }
+
+  private mergeUnlimitedQuota(orig: QuotaBusinessType) {
+    return {
+      ...orig,
+      storageQuota: 1000 * OneGB,
+      memberLimit: 1000,
+      humanReadable: {
+        ...orig.humanReadable,
+        name: 'Unlimited',
+        storageQuota: formatSize(1000 * OneGB),
+        memberLimit: '1000',
+      },
+    };
+  }
+
+  async checkBlobQuota(workspaceId: string, size: number) {
+    const { storageQuota, usedSize } =
+      await this.getWorkspaceUsage(workspaceId);
+
+    return storageQuota - (size + usedSize);
+  }
+}

packages/backend/server/src/core/quota/types.ts

@@ -0,0 +1,110 @@
+import { Field, ObjectType } from '@nestjs/graphql';
+import { SafeIntResolver } from 'graphql-scalars';
+import { z } from 'zod';
+
+import { commonFeatureSchema, FeatureKind } from '../features';
+import { ByteUnit, OneDay, OneKB } from './constant';
+
+/// ======== quota define ========
+
+/**
+ * naming rule:
+ * we append Vx to the end of the feature name to indicate the version of the feature
+ * x is a number, start from 1, this number will be change only at the time we change the schema of config
+ * for example, we change the value of `blobLimit` from 10MB to 100MB, then we will only change `version` field from 1 to 2
+ * but if we remove the `blobLimit` field or rename it, then we will change the Vx to Vx+1
+ */
+export enum QuotaType {
+  FreePlanV1 = 'free_plan_v1',
+  ProPlanV1 = 'pro_plan_v1',
+  // only for test, smaller quota
+  RestrictedPlanV1 = 'restricted_plan_v1',
+}
+
+const quotaPlan = z.object({
+  feature: z.enum([
+    QuotaType.FreePlanV1,
+    QuotaType.ProPlanV1,
+    QuotaType.RestrictedPlanV1,
+  ]),
+  configs: z.object({
+    name: z.string(),
+    blobLimit: z.number().positive().int(),
+    storageQuota: z.number().positive().int(),
+    historyPeriod: z.number().positive().int(),
+    memberLimit: z.number().positive().int(),
+    businessBlobLimit: z.number().positive().int().nullish(),
+  }),
+});
+
+/// ======== schema infer ========
+
+export const QuotaSchema = commonFeatureSchema
+  .extend({
+    type: z.literal(FeatureKind.Quota),
+  })
+  .and(z.discriminatedUnion('feature', [quotaPlan]));
+
+export type Quota = z.infer<typeof QuotaSchema>;
+
+/// ======== query types ========
+
+@ObjectType()
+export class HumanReadableQuotaType {
+  @Field(() => String)
+  name!: string;
+
+  @Field(() => String)
+  blobLimit!: string;
+
+  @Field(() => String)
+  storageQuota!: string;
+
+  @Field(() => String)
+  historyPeriod!: string;
+
+  @Field(() => String)
+  memberLimit!: string;
+}
+
+@ObjectType()
+export class QuotaQueryType {
+  @Field(() => String)
+  name!: string;
+
+  @Field(() => SafeIntResolver)
+  blobLimit!: number;
+
+  @Field(() => SafeIntResolver)
+  historyPeriod!: number;
+
+  @Field(() => SafeIntResolver)
+  memberLimit!: number;
+
+  @Field(() => SafeIntResolver)
+  storageQuota!: number;
+
+  @Field(() => HumanReadableQuotaType)
+  humanReadable!: HumanReadableQuotaType;
+
+  @Field(() => SafeIntResolver)
+  usedSize!: number;
+}
+
+/// ======== utils ========
+
+export function formatSize(bytes: number, decimals: number = 2): string {
+  if (bytes === 0) return '0 B';
+
+  const dm = decimals < 0 ? 0 : decimals;
+
+  const i = Math.floor(Math.log(bytes) / Math.log(OneKB));
+
+  return (
+    parseFloat((bytes / Math.pow(OneKB, i)).toFixed(dm)) + ' ' + ByteUnit[i]
+  );
+}
+
+export function formatDate(ms: number): string {
+  return `${(ms / OneDay).toFixed(0)} days`;
+}

packages/backend/server/src/core/storage/index.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+
+import { AvatarStorage, WorkspaceBlobStorage } from './wrappers';
+
+@Module({
+  providers: [WorkspaceBlobStorage, AvatarStorage],
+  exports: [WorkspaceBlobStorage, AvatarStorage],
+})
+export class StorageModule {}
+
+export { AvatarStorage, WorkspaceBlobStorage };

packages/backend/server/src/core/storage/wrappers/avatar.ts

@@ -0,0 +1,49 @@
+import { Injectable } from '@nestjs/common';
+
+import type {
+  BlobInputType,
+  EventPayload,
+  PutObjectMetadata,
+  StorageProvider,
+} from '../../../fundamentals';
+import { Config, OnEvent, StorageProviderFactory } from '../../../fundamentals';
+
+@Injectable()
+export class AvatarStorage {
+  public readonly provider: StorageProvider;
+  private readonly storageConfig: Config['storage']['storages']['avatar'];
+
+  constructor(
+    private readonly config: Config,
+    private readonly storageFactory: StorageProviderFactory
+  ) {
+    this.provider = this.storageFactory.create('avatar');
+    this.storageConfig = this.config.storage.storages.avatar;
+  }
+
+  async put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) {
+    await this.provider.put(key, blob, metadata);
+    let link = this.storageConfig.publicLinkFactory(key);
+
+    if (link.startsWith('/')) {
+      link = this.config.baseUrl + link;
+    }
+
+    return link;
+  }
+
+  get(key: string) {
+    return this.provider.get(key);
+  }
+
+  delete(key: string) {
+    return this.provider.delete(key);
+  }
+
+  @OnEvent('user.deleted')
+  async onUserDeleted(user: EventPayload<'user.deleted'>) {
+    if (user.avatarUrl) {
+      await this.delete(user.avatarUrl);
+    }
+  }
+}

packages/backend/server/src/core/storage/wrappers/blob.ts

@@ -0,0 +1,77 @@
+import { Injectable } from '@nestjs/common';
+
+import type {
+  BlobInputType,
+  EventPayload,
+  StorageProvider,
+} from '../../../fundamentals';
+import {
+  EventEmitter,
+  OnEvent,
+  StorageProviderFactory,
+} from '../../../fundamentals';
+
+@Injectable()
+export class WorkspaceBlobStorage {
+  public readonly provider: StorageProvider;
+
+  constructor(
+    private readonly event: EventEmitter,
+    private readonly storageFactory: StorageProviderFactory
+  ) {
+    this.provider = this.storageFactory.create('blob');
+  }
+
+  async put(workspaceId: string, key: string, blob: BlobInputType) {
+    await this.provider.put(`${workspaceId}/${key}`, blob);
+  }
+
+  async get(workspaceId: string, key: string) {
+    return this.provider.get(`${workspaceId}/${key}`);
+  }
+
+  async list(workspaceId: string) {
+    const blobs = await this.provider.list(workspaceId + '/');
+
+    blobs.forEach(item => {
+      // trim workspace prefix
+      item.key = item.key.slice(workspaceId.length + 1);
+    });
+
+    return blobs;
+  }
+
+  /**
+   * we won't really delete the blobs until the doc blobs manager is implemented sounded
+   */
+  async delete(_workspaceId: string, _key: string) {
+    // return this.provider.delete(`${workspaceId}/${key}`);
+  }
+
+  async totalSize(workspaceId: string) {
+    const blobs = await this.list(workspaceId);
+    // how could we ignore the ones get soft-deleted?
+    return blobs.reduce((acc, item) => acc + item.size, 0);
+  }
+
+  @OnEvent('workspace.deleted')
+  async onWorkspaceDeleted(workspaceId: EventPayload<'workspace.deleted'>) {
+    const blobs = await this.list(workspaceId);
+
+    // to reduce cpu time holding
+    blobs.forEach(blob => {
+      this.event.emit('workspace.blob.deleted', {
+        workspaceId: workspaceId,
+        name: blob.key,
+      });
+    });
+  }
+
+  @OnEvent('workspace.blob.deleted')
+  async onDeleteWorkspaceBlob({
+    workspaceId,
+    name,
+  }: EventPayload<'workspace.blob.deleted'>) {
+    await this.delete(workspaceId, name);
+  }
+}