chore: bump up @blocksuite/affine version to v0.17.32 (#8805)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

.cargo/config.toml

@@ -1,2 +1,4 @@
 [target.x86_64-pc-windows-msvc]
 rustflags = ["-C", "target-feature=+crt-static"]
+[target.aarch64-pc-windows-msvc]
+rustflags = ["-C", "target-feature=+crt-static"]

.devcontainer/build.sh

@@ -13,6 +13,3 @@ yarn workspace @affine/server-native build
 
 # Create database
 yarn workspace @affine/server prisma db push
-
-# Create user username: affine, password: affine
-echo "INSERT INTO \"users\"(\"id\",\"name\",\"email\",\"email_verified\",\"created_at\",\"password\") VALUES('99f3ad04-7c9b-441e-a6db-79f73aa64db9','affine','affine@affine.pro','2024-02-26 15:54:16.974','2024-02-26 15:54:16.974+00','\$argon2id\$v=19\$m=19456,t=2,p=1\$esDS3QCHRH0Kmeh87YPm5Q\$9S+jf+xzw2Hicj6nkWltvaaaXX3dQIxAFwCfFa9o38A');" | yarn workspace @affine/server prisma db execute --stdin

.env.template

@@ -1,5 +1,4 @@
 CHANGELOG_URL=
-ENABLE_PRELOADING=
 ENABLE_NEW_SETTING_UNSTABLE_API=
 ENABLE_CAPTCHA=
 CAPTCHA_SITE_KEY=

.eslintignore

@@ -12,4 +12,5 @@ static
 web-static
 public
 packages/frontend/i18n/src/i18n-generated.ts
+packages/frontend/i18n/src/i18n-completenesses.json
 packages/frontend/templates/*.gen.ts

.eslintrc.js

@@ -34,8 +34,8 @@ const createPattern = packageName => [
   {
     group: ['@affine/env/constant'],
     message:
-      'Do not import from @affine/env/constant. Use `environment.isDesktop` instead',
-    importNames: ['isDesktop'],
+      'Do not import from @affine/env/constant. Use `BUILD_CONFIG.isElectron` instead',
+    importNames: ['isElectron'],
   },
 ];
 
@@ -43,11 +43,14 @@ const allPackages = [
   'packages/backend/server',
   'packages/frontend/component',
   'packages/frontend/core',
-  'packages/frontend/electron',
+  'packages/frontend/apps/electron',
+  'packages/frontend/apps/web',
+  'packages/frontend/apps/mobile',
   'packages/frontend/graphql',
   'packages/frontend/i18n',
   'packages/frontend/native',
   'packages/frontend/templates',
+  'packages/frontend/track',
   'packages/common/debug',
   'packages/common/env',
   'packages/common/infra',
@@ -248,7 +251,7 @@ const config = {
           'warn',
           {
             additionalHooks:
-              '(useAsyncCallback|useCatchEventCallback|useDraggable|useDropTarget)',
+              '(useAsyncCallback|useCatchEventCallback|useDraggable|useDropTarget|useRefEffect)',
           },
         ],
       },

.github/actions/build-rust/action.yml

@@ -49,7 +49,7 @@ runs:
     - name: Build
       shell: bash
       run: |
-        yarn workspace ${{ inputs.package }} nx build ${{ inputs.package }} -- --target ${{ inputs.target }} --use-napi-cross
+        yarn workspace ${{ inputs.package }} build --target ${{ inputs.target }} --use-napi-cross
       env:
         NX_CLOUD_ACCESS_TOKEN: ${{ inputs.nx_token }}
         DEBUG: 'napi:*'

.github/actions/cluster-auth/action.yml

@@ -0,0 +1,36 @@
+name: 'Auth to Cluster'
+description: 'Auth to the GCP Cluster'
+inputs:
+  gcp-project-number:
+    description: 'GCP project number'
+    required: true
+  gcp-project-id:
+    description: 'GCP project id'
+    required: true
+  service-account:
+    description: 'Service account'
+  cluster-name:
+    description: 'Cluster name'
+  cluster-location:
+    description: 'Cluster location'
+
+runs:
+  using: 'composite'
+  steps:
+    - id: auth
+      uses: google-github-actions/auth@v2
+      with:
+        workload_identity_provider: 'projects/${{ inputs.gcp-project-number }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions-helm-deploy'
+        service_account: '${{ inputs.service-account }}'
+        token_format: 'access_token'
+        project_id: '${{ inputs.gcp-project-id }}'
+
+    - name: 'Setup gcloud cli'
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        install_components: 'gke-gcloud-auth-plugin'
+
+    - id: get-gke-credentials
+      shell: bash
+      run: |
+        gcloud container clusters get-credentials ${{ inputs.cluster-name }} --region ${{ inputs.cluster-location }} --project ${{ inputs.gcp-project-id }}

.github/actions/copilot-test/action.yml

@@ -0,0 +1,36 @@
+name: 'Run Copilot E2E Test'
+description: 'Run Copilot E2E Test'
+inputs:
+  script:
+    description: 'Script to run'
+    default: 'yarn workspace @affine-test/affine-cloud-copilot e2e --forbid-only'
+    required: false
+  openai-key:
+    description: 'OpenAI secret key'
+    required: true
+  fal-key:
+    description: 'Fal secret key'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Prepare Server Test Environment
+      uses: ./.github/actions/server-test-env
+
+    - name: Server Copilot E2E Test
+      shell: bash
+      run: ${{ inputs.script }}
+      env:
+        COPILOT: true
+        DEV_SERVER_URL: http://localhost:8080
+        COPILOT_OPENAI_API_KEY: ${{ inputs.openai-key }}
+        COPILOT_FAL_API_KEY: ${{ inputs.fal-key }}
+
+    - name: Upload test results
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-results-e2e-server-copilot
+        path: ./test-results
+        if-no-files-found: ignore

.github/actions/deploy/action.yml

@@ -24,24 +24,14 @@ runs:
       shell: bash
       run: |
         echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
-    - uses: azure/setup-helm@v4
-    - id: auth
-      uses: google-github-actions/auth@v2
+    - name: 'Auth to cluster'
+      uses: './.github/actions/cluster-auth'
       with:
-        workload_identity_provider: 'projects/${{ inputs.gcp-project-number }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions-helm-deploy'
-        service_account: '${{ inputs.service-account }}'
-        token_format: 'access_token'
-        project_id: '${{ inputs.gcp-project-id }}'
-
-    - name: 'Setup gcloud cli'
-      uses: 'google-github-actions/setup-gcloud@v2'
-      with:
-        install_components: 'gke-gcloud-auth-plugin'
-
-    - id: get-gke-credentials
-      shell: bash
-      run: |
-        gcloud container clusters get-credentials ${{ inputs.cluster-name }} --region ${{ inputs.cluster-location }} --project ${{ inputs.gcp-project-id }}
+        gcp-project-number: '${{ inputs.gcp-project-number }}'
+        gcp-project-id: '${{ inputs.gcp-project-id }}'
+        service-account: '${{ inputs.service-account }}'
+        cluster-name: '${{ inputs.cluster-name }}'
+        cluster-location: '${{ inputs.cluster-location }}'
 
     - name: Deploy
       shell: bash

.github/actions/deploy/deploy.mjs

@@ -40,6 +40,42 @@ const isProduction = buildType === 'stable';
 const isBeta = buildType === 'beta';
 const isInternal = buildType === 'internal';
 
+const replicaConfig = {
+  production: {
+    web: 3,
+    graphql: Number(process.env.PRODUCTION_GRAPHQL_REPLICA) || 3,
+    sync: Number(process.env.PRODUCTION_SYNC_REPLICA) || 3,
+    renderer: Number(process.env.PRODUCTION_RENDERER_REPLICA) || 3,
+  },
+  beta: {
+    web: 2,
+    graphql: Number(process.env.BETA_GRAPHQL_REPLICA) || 2,
+    sync: Number(process.env.BETA_SYNC_REPLICA) || 2,
+    renderer: Number(process.env.BETA_RENDERER_REPLICA) || 2,
+  },
+  canary: {
+    web: 2,
+    graphql: 2,
+    sync: 2,
+    renderer: 2,
+  },
+};
+
+const cpuConfig = {
+  beta: {
+    web: '300m',
+    graphql: '1',
+    sync: '1',
+    renderer: '300m',
+  },
+  canary: {
+    web: '300m',
+    graphql: '1',
+    sync: '1',
+    renderer: '300m',
+  },
+};
+
 const createHelmCommand = ({ isDryRun }) => {
   const flag = isDryRun ? '--dry-run' : '--atomic';
   const imageTag = `${buildType}-${GIT_SHORT_HASH}`;
@@ -67,17 +103,18 @@ const createHelmCommand = ({ isDryRun }) => {
           `--set-json   cloud-sql-proxy.nodeSelector=\"{ \\"iam.gke.io/gke-metadata-server-enabled\\": \\"true\\" }\"`,
         ]
       : [];
-  const webReplicaCount = isProduction ? 3 : isBeta ? 2 : 2;
-  const graphqlReplicaCount = isProduction
-    ? Number(process.env.PRODUCTION_GRAPHQL_REPLICA) || 3
-    : isBeta
-      ? Number(process.env.isBeta_GRAPHQL_REPLICA) || 2
-      : 2;
-  const syncReplicaCount = isProduction
-    ? Number(process.env.PRODUCTION_SYNC_REPLICA) || 3
-    : isBeta
-      ? Number(process.env.BETA_SYNC_REPLICA) || 2
-      : 2;
+
+  const cpu = cpuConfig[buildType];
+  const resources = cpu
+    ? [
+        `--set        web.resources.requests.cpu="${cpu.web}"`,
+        `--set        graphql.resources.requests.cpu="${cpu.graphql}"`,
+        `--set        sync.resources.requests.cpu="${cpu.sync}"`,
+      ]
+    : [];
+
+  const replica = replicaConfig[buildType] || replicaConfig.canary;
+
   const namespace = isProduction
     ? 'production'
     : isBeta
@@ -90,14 +127,19 @@ const createHelmCommand = ({ isDryRun }) => {
   const deployCommand = [
     `helm upgrade --install affine .github/helm/affine`,
     `--namespace  ${namespace}`,
+    `--set-string global.app.buildType="${buildType}"`,
     `--set        global.ingress.enabled=true`,
     `--set-json   global.ingress.annotations=\"{ \\"kubernetes.io/ingress.class\\": \\"gce\\", \\"kubernetes.io/ingress.allow-http\\": \\"true\\", \\"kubernetes.io/ingress.global-static-ip-name\\": \\"${STATIC_IP_NAME}\\" }\"`,
     `--set-string global.ingress.host="${host}"`,
+    `--set        global.objectStorage.r2.enabled=true`,
+    `--set-string global.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`,
+    `--set-string global.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`,
+    `--set-string global.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`,
     `--set-string global.version="${APP_VERSION}"`,
     ...redisAndPostgres,
-    `--set        web.replicaCount=${webReplicaCount}`,
+    `--set        web.replicaCount=${replica.web}`,
     `--set-string web.image.tag="${imageTag}"`,
-    `--set        graphql.replicaCount=${graphqlReplicaCount}`,
+    `--set        graphql.replicaCount=${replica.graphql}`,
     `--set-string graphql.image.tag="${imageTag}"`,
     `--set        graphql.app.host=${host}`,
     `--set        graphql.app.captcha.enabled=true`,
@@ -106,10 +148,6 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set-string graphql.app.copilot.openai.key="${COPILOT_OPENAI_API_KEY}"`,
     `--set-string graphql.app.copilot.fal.key="${COPILOT_FAL_API_KEY}"`,
     `--set-string graphql.app.copilot.unsplash.key="${COPILOT_UNSPLASH_API_KEY}"`,
-    `--set        graphql.app.objectStorage.r2.enabled=true`,
-    `--set-string graphql.app.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`,
-    `--set-string graphql.app.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`,
-    `--set-string graphql.app.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`,
     `--set-string graphql.app.mailer.sender="${MAILER_SENDER}"`,
     `--set-string graphql.app.mailer.user="${MAILER_USER}"`,
     `--set-string graphql.app.mailer.password="${MAILER_PASSWORD}"`,
@@ -123,9 +161,13 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set        graphql.app.experimental.enableJwstCodec=${namespace === 'dev'}`,
     `--set        graphql.app.features.earlyAccessPreview=false`,
     `--set        graphql.app.features.syncClientVersionCheck=true`,
-    `--set        sync.replicaCount=${syncReplicaCount}`,
+    `--set        sync.replicaCount=${replica.sync}`,
     `--set-string sync.image.tag="${imageTag}"`,
+    `--set-string renderer.image.tag="${imageTag}"`,
+    `--set        renderer.app.host=${host}`,
+    `--set        renderer.replicaCount=${replica.renderer}`,
     ...serviceAnnotations,
+    ...resources,
     `--timeout 10m`,
     flag,
   ].join(' ');

.github/actions/server-test-env/action.yml

@@ -0,0 +1,21 @@
+name: 'Prepare Server Test Environment'
+description: 'Prepare Server Test Environment'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Initialize database
+      shell: bash
+      run: |
+        psql -h localhost -U postgres -c "CREATE DATABASE affine;"
+        psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
+        psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
+      env:
+        PGPASSWORD: affine
+
+    - name: Run init-db script
+      shell: bash
+      run: |
+        yarn workspace @affine/server exec prisma generate
+        yarn workspace @affine/server exec prisma db push
+        yarn workspace @affine/server data-migration run

.github/actions/setup-node/action.yml

@@ -17,6 +17,10 @@ inputs:
     description: 'Download the Electron binary'
     required: false
     default: 'true'
+  corepack-install:
+    description: 'Install CorePack'
+    required: false
+    default: 'false'
   hard-link-nm:
     description: 'set nmMode to hardlinks-local in .yarnrc.yml'
     required: false
@@ -42,6 +46,11 @@ runs:
         registry-url: https://npm.pkg.github.com
         scope: '@toeverything'
 
+    - name: Init CorePack
+      if: ${{ inputs.corepack-install == 'true' }}
+      shell: bash
+      run: corepack enable
+
     - name: Set nmMode
       if: ${{ inputs.hard-link-nm == 'false' }}
       shell: bash
@@ -156,7 +165,7 @@ runs:
     - name: Install Playwright's dependencies
       shell: bash
       if: inputs.playwright-install == 'true'
-      run: yarn playwright install --with-deps chromium
+      run: yarn playwright install --with-deps chromium webkit
       env:
         PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/node_modules/.cache/ms-playwright
 

.github/deployment/front/Dockerfile

@@ -1,7 +1,8 @@
-FROM openresty/openresty:1.25.3.1-0-buster
+FROM openresty/openresty:1.27.1.1-0-buster
 WORKDIR /app
-COPY ./packages/frontend/web/dist ./dist
+COPY ./packages/frontend/apps/web/dist ./dist
 COPY ./packages/frontend/admin/dist ./admin
+COPY ./packages/frontend/apps/mobile/dist ./mobile
 COPY ./.github/deployment/front/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf
 COPY ./.github/deployment/front/affine.nginx.conf /etc/nginx/conf.d/affine.nginx.conf
 

.github/deployment/front/affine.nginx.conf

@@ -6,15 +6,33 @@ server {
     try_files $uri/index.html $uri/ $uri /admin/index.html;
   }
 
+  set $app_root_path /app/dist/;
+  set $mobile_root /app/dist/;
+  set_by_lua $affine_env 'return os.getenv("AFFINE_ENV")';
+
+  if ($affine_env = "dev") {
+    set $mobile_root /app/mobile/;
+  }
+
+  # https://gist.github.com/mariusom/6683dc52b1cad1a1f372e908bdb209d0
+  if ($http_user_agent ~* "(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino") {
+    set $app_root_path $mobile_root;
+  }
+
+  if ($http_user_agent ~* "^(1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-)") {
+    set $app_root_path $mobile_root;
+  }
+
   location ~ ^/(_plugin|assets|imgs|js|plugins|static)/ {
-    root /app/dist/;
+    root $app_root_path;
     try_files $uri $uri/ =404;
   }
 
   location / {
-    root /app/dist/;
+    root $app_root_path;
     index index.html;
     try_files $uri $uri/ /index.html;
+    add_header Cache-Control "private, no-cache, no-store, max-age=0, must-revalidate";
   }
 
   error_page 404 /404.html;

.github/deployment/front/nginx.conf

@@ -1,14 +1,15 @@
-worker_processes  4;
+worker_processes 4;
 error_log /var/log/nginx/error.log warn;
 pcre_jit on;
+env AFFINE_ENV;
 events {
-    worker_connections 1024;
+  worker_connections 1024;
 }
 http {
-    include       mime.types;
-    log_format    main  '$remote_addr [$time_local] "$request" '
-                        '$status $body_bytes_sent "$http_referer" '
-                        '"$http_user_agent" "$http_x_forwarded_for"';
-    access_log    /var/log/nginx/access.log  main;
-    include /etc/nginx/conf.d/*.conf;
+  include mime.types;
+  log_format main '$remote_addr [$time_local] "$request" '
+  '$status $body_bytes_sent "$http_referer" '
+  '"$http_user_agent" "$http_x_forwarded_for"';
+  access_log /var/log/nginx/access.log main;
+  include /etc/nginx/conf.d/*.conf;
 }

.github/deployment/node/Dockerfile

@@ -1,8 +1,9 @@
 FROM node:20-bookworm-slim
 
 COPY ./packages/backend/server /app
-COPY ./packages/frontend/web/dist /app/static
+COPY ./packages/frontend/apps/web/dist /app/static
 COPY ./packages/frontend/admin/dist /app/static/admin
+COPY ./packages/frontend/apps/mobile/dist /app/static/mobile
 WORKDIR /app
 
 RUN apt-get update && \

.github/deployment/self-host/compose.yaml

@@ -28,8 +28,6 @@ services:
       - REDIS_SERVER_HOST=redis
       - DATABASE_URL=postgres://affine:affine@postgres:5432/affine
       - NODE_ENV=production
-      - AFFINE_ADMIN_EMAIL=${AFFINE_ADMIN_EMAIL}
-      - AFFINE_ADMIN_PASSWORD=${AFFINE_ADMIN_PASSWORD}
       # Telemetry allows us to collect data on how you use the affine. This data will helps us improve the app and provide better features.
       # Uncomment next line if you wish to quit telemetry.
       # - TELEMETRY_ENABLE=false
@@ -45,7 +43,7 @@ services:
       timeout: 5s
       retries: 5
   postgres:
-    image: postgres
+    image: postgres:16
     container_name: affine_postgres
     restart: unless-stopped
     volumes:

.github/helm/affine/Chart.yaml

@@ -3,4 +3,4 @@ name: affine
 description: AFFiNE cloud chart
 type: application
 version: 0.0.0
-appVersion: "0.16.0"
+appVersion: "0.17.0"

.github/helm/affine/charts/graphql/Chart.yaml

@@ -3,7 +3,7 @@ name: graphql
 description: AFFiNE GraphQL server
 type: application
 version: 0.0.0
-appVersion: "0.16.0"
+appVersion: "0.17.0"
 dependencies:
   - name: gcloud-sql-proxy
     version: 0.0.0

.github/helm/affine/charts/graphql/templates/deployment.yaml

@@ -76,9 +76,7 @@ spec:
           - name: AFFINE_SERVER_HTTPS
             value: "{{ .Values.app.https }}"
           - name: ENABLE_R2_OBJECT_STORAGE
-            value: "{{ .Values.app.objectStorage.r2.enabled }}"
-          - name: ENABLE_CAPTCHA
-            value: "{{ .Values.app.captcha.enabled }}"
+            value: "{{ .Values.global.objectStorage.r2.enabled }}"
           - name: FEATURES_EARLY_ACCESS_PREVIEW
             value: "{{ .Values.app.features.earlyAccessPreview }}"
           - name: FEATURES_SYNC_CLIENT_VERSION_CHECK
@@ -124,21 +122,21 @@ spec:
           - name: DOC_MERGE_USE_JWST_CODEC
             value: "true"
           {{ end }}
-          {{ if .Values.app.objectStorage.r2.enabled }}
+          {{ if .Values.global.objectStorage.r2.enabled }}
           - name: R2_OBJECT_STORAGE_ACCOUNT_ID
             valueFrom:
               secretKeyRef:
-                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                name: "{{ .Values.global.objectStorage.r2.secretName }}"
                 key: accountId
           - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID
             valueFrom:
               secretKeyRef:
-                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                name: "{{ .Values.global.objectStorage.r2.secretName }}"
                 key: accessKeyId
           - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY
             valueFrom:
               secretKeyRef:
-                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                name: "{{ .Values.global.objectStorage.r2.secretName }}"
                 key: secretAccessKey
           {{ end }}
           {{ if .Values.app.captcha.enabled }}
@@ -204,12 +202,12 @@ spec:
               protocol: TCP
           livenessProbe:
             httpGet:
-              path: /
+              path: /info
               port: http
             initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }}
           readinessProbe:
             httpGet:
-              path: /
+              path: /info
               port: http
             initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }}
           resources:

.github/helm/affine/charts/graphql/templates/migration.yaml

@@ -37,21 +37,21 @@ spec:
           - name: DATABASE_URL
             value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.gcloud.cloudSqlInternal }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
           {{ end }}
-          {{ if .Values.app.objectStorage.r2.enabled }}
+          {{ if .Values.global.objectStorage.r2.enabled }}
           - name: R2_OBJECT_STORAGE_ACCOUNT_ID
             valueFrom:
               secretKeyRef:
-                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                name: "{{ .Values.global.objectStorage.r2.secretName }}"
                 key: accountId
           - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID
             valueFrom:
               secretKeyRef:
-                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                name: "{{ .Values.global.objectStorage.r2.secretName }}"
                 key: accessKeyId
           - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY
             valueFrom:
               secretKeyRef:
-                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                name: "{{ .Values.global.objectStorage.r2.secretName }}"
                 key: secretAccessKey
           {{ end }}
         resources:

.github/helm/affine/charts/graphql/templates/r2-secret.yaml

@@ -1,11 +0,0 @@
-{{- if .Values.app.objectStorage.r2.enabled -}}
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "{{ .Values.app.objectStorage.r2.secretName }}"
-type: Opaque
-data:
-  accountId: {{ .Values.app.objectStorage.r2.accountId | b64enc }}
-  accessKeyId: {{ .Values.app.objectStorage.r2.accessKeyId | b64enc }}
-  secretAccessKey: {{ .Values.app.objectStorage.r2.secretAccessKey | b64enc }}
-{{- end }}

.github/helm/affine/charts/graphql/templates/service.yaml

@@ -4,6 +4,10 @@ metadata:
   name: {{ include "graphql.fullname" . }}
   labels:
     {{- include "graphql.labels" . | nindent 4 }}
+  {{- with .Values.service.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   type: {{ .Values.service.type }}
   ports:

.github/helm/affine/charts/graphql/values.yaml

@@ -29,14 +29,7 @@ app:
     secretName: copilot
     openai:
       key: ''
-  objectStorage:
-    r2:
-      enabled: false
-      secretName: r2
-      accountId: ''
-      accessKeyId: ''
-      secretAccessKey: ''
-  oauth: 
+  oauth:
     google:
       enabled: false
       secretName: oauth-google

.github/helm/affine/charts/renderer/Chart.yaml

@@ -0,0 +1,11 @@
+apiVersion: v2
+name: renderer
+description: AFFiNE renderer server
+type: application
+version: 0.0.0
+appVersion: "0.16.0"
+dependencies:
+  - name: gcloud-sql-proxy
+    version: 0.0.0
+    repository: "file://../gcloud-sql-proxy"
+    condition: .global.database.gcloud.enabled

.github/helm/affine/charts/renderer/templates/NOTES.txt

@@ -0,0 +1,16 @@
+1. Get the application URL by running these commands:
+{{- if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "renderer.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "renderer.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "renderer.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "renderer.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}

.github/helm/affine/charts/renderer/templates/_helpers.tpl

@@ -0,0 +1,63 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "renderer.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "renderer.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "renderer.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "renderer.labels" -}}
+helm.sh/chart: {{ include "renderer.chart" . }}
+{{ include "renderer.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+monitoring: enabled
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "renderer.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "renderer.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "renderer.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "renderer.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

.github/helm/affine/charts/renderer/templates/deployment.yaml

@@ -0,0 +1,124 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "renderer.fullname" . }}
+  labels:
+    {{- include "renderer.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "renderer.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "renderer.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "renderer.serviceAccountName" . }}
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+          - name: AFFINE_PRIVATE_KEY
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.global.secret.secretName }}"
+                key: key
+          - name: NODE_ENV
+            value: "{{ .Values.env }}"
+          - name: NODE_OPTIONS
+            value: "--max-old-space-size=4096"
+          - name: NO_COLOR
+            value: "1"
+          - name: DEPLOYMENT_TYPE
+            value: "affine"
+          - name: SERVER_FLAVOR
+            value: "renderer"
+          - name: AFFINE_ENV
+            value: "{{ .Release.Namespace }}"
+          - name: DATABASE_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: pg-postgresql
+                key: postgres-password
+          - name: DATABASE_URL
+            value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.url }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
+          - name: REDIS_SERVER_ENABLED
+            value: "true"
+          - name: REDIS_SERVER_HOST
+            value: "{{ .Values.global.redis.host }}"
+          - name: REDIS_SERVER_PORT
+            value: "{{ .Values.global.redis.port }}"
+          - name: REDIS_SERVER_USER
+            value: "{{ .Values.global.redis.username }}"
+          - name: REDIS_SERVER_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: redis
+                key: redis-password
+          - name: REDIS_SERVER_DATABASE
+            value: "{{ .Values.global.redis.database }}"
+          - name: AFFINE_SERVER_PORT
+            value: "{{ .Values.service.port }}"
+          - name: AFFINE_SERVER_SUB_PATH
+            value: "{{ .Values.app.path }}"
+          - name: AFFINE_SERVER_HOST
+            value: "{{ .Values.app.host }}"
+          - name: AFFINE_SERVER_HTTPS
+            value: "{{ .Values.app.https }}"
+          - name: ENABLE_R2_OBJECT_STORAGE
+            value: "{{ .Values.global.objectStorage.r2.enabled }}"
+          {{ if .Values.global.objectStorage.r2.enabled }}
+          - name: R2_OBJECT_STORAGE_ACCOUNT_ID
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.global.objectStorage.r2.secretName }}"
+                key: accountId
+          - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.global.objectStorage.r2.secretName }}"
+                key: accessKeyId
+          - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.global.objectStorage.r2.secretName }}"
+                key: secretAccessKey
+          {{ end }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /info
+              port: http
+            initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }}
+          readinessProbe:
+            httpGet:
+              path: /info
+              port: http
+            initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

.github/helm/affine/charts/renderer/templates/service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "graphql.fullname" . }}
+  labels:
+    {{- include "graphql.labels" . | nindent 4 }}
+  {{- with .Values.service.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "graphql.selectorLabels" . | nindent 4 }}

.github/helm/affine/charts/renderer/templates/serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "graphql.serviceAccountName" . }}
+  labels:
+    {{- include "graphql.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

.github/helm/affine/charts/renderer/templates/tests/test-connection.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "renderer.fullname" . }}-test-connection"
+  labels:
+    {{- include "renderer.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "renderer.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never

.github/helm/affine/charts/renderer/values.yaml

@@ -0,0 +1,38 @@
+replicaCount: 1
+image:
+  repository: ghcr.io/toeverything/affine-graphql
+  pullPolicy: IfNotPresent
+  tag: ''
+
+imagePullSecrets: []
+nameOverride: ''
+fullnameOverride: ''
+# map to NODE_ENV environment variable
+env: 'production'
+app:
+  # AFFINE_SERVER_SUB_PATH
+  path: ''
+  # AFFINE_SERVER_HOST
+  host: '0.0.0.0'
+  https: true
+serviceAccount:
+  create: true
+  annotations: {}
+  name: 'affine-renderer'
+
+podAnnotations: {}
+
+podSecurityContext:
+  fsGroup: 2000
+
+resources:
+  requests:
+    cpu: '4'
+    memory: 4Gi
+
+probe:
+  initialDelaySeconds: 20
+
+nodeSelector: {}
+tolerations: []
+affinity: {}

.github/helm/affine/charts/sync/Chart.yaml

@@ -3,7 +3,7 @@ name: sync
 description: AFFiNE Sync Server
 type: application
 version: 0.0.0
-appVersion: "0.16.0"
+appVersion: "0.17.0"
 dependencies:
   - name: gcloud-sql-proxy
     version: 0.0.0

.github/helm/affine/charts/web/templates/deployment.yaml

@@ -27,6 +27,9 @@ spec:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+          - name: AFFINE_ENV
+            value: "{{ .Release.Namespace }}"
           ports:
             - name: http
               containerPort: {{ .Values.service.port }}

.github/helm/affine/templates/configmap.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-runtime-config
+data:
+  web-assets-manifest: |-
+    {{ .Files.Get "web-assets-manifest.json" | nindent 4 }}
+  mobile-assets-manifest: |-
+    {{ .Files.Get "mobile-assets-manifest.json" | nindent 4 }}

.github/helm/affine/templates/ingress.yaml

@@ -60,13 +60,13 @@ spec:
                 name: affine-graphql
                 port:
                   number: {{ .Values.graphql.service.port }}
-          - path: /oauth
+          - path: /workspace
             pathType: Prefix
             backend:
               service:
-                name: affine-graphql
+                name: affine-renderer
                 port:
-                  number: {{ .Values.graphql.service.port }}
+                  number: {{ .Values.renderer.service.port }}
           - path: /
             pathType: Prefix
             backend:
@@ -74,11 +74,4 @@ spec:
                 name: affine-web
                 port:
                   number: {{ .Values.web.service.port }}
-          - path: /js/worker.(.+).js
-            pathType: ImplementationSpecific
-            backend:
-              service:
-                name: affine-web
-                port:
-                  number: {{ .Values.web.service.port }}
 {{- end }}

.github/helm/affine/templates/r2-secret.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.global.objectStorage.r2.enabled -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "{{ .Values.global.objectStorage.r2.secretName }}"
+type: Opaque
+data:
+  accountId: {{ .Values.global.objectStorage.r2.accountId | b64enc }}
+  accessKeyId: {{ .Values.global.objectStorage.r2.accessKeyId | b64enc }}
+  secretAccessKey: {{ .Values.global.objectStorage.r2.secretAccessKey | b64enc }}
+{{- end }}

.github/helm/affine/values.yaml

@@ -1,4 +1,6 @@
 global:
+  app:
+    buildType: 'stable'
   ingress:
     enabled: false
     className: ''
@@ -28,6 +30,13 @@ global:
     username: ''
     password: ''
     database: 0
+  objectStorage:
+    r2:
+      enabled: false
+      secretName: r2
+      accountId: ''
+      accessKeyId: ''
+      secretAccessKey: ''
   gke:
     enabled: true
 
@@ -36,14 +45,21 @@ graphql:
     type: ClusterIP
     port: 3000
     annotations:
-      cloud.google.com/backend-config: '{"default": "affine-backendconfig"}'
+      cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}'
 
 sync:
   service:
     type: ClusterIP
     port: 3010
     annotations:
-      cloud.google.com/backend-config: '{"default": "affine-backendconfig"}'
+      cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}'
+
+renderer:
+  service:
+    type: ClusterIP
+    port: 3000
+    annotations:
+      cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}'
 
 web:
   service:

.github/helm/separate-config/backend-config.yaml

@@ -0,0 +1,10 @@
+apiVersion: cloud.google.com/v1
+kind: BackendConfig
+metadata:
+  name: "affine-api-backendconfig"
+spec:
+  healthCheck:
+    timeoutSec: 1
+    type: HTTP
+    requestPath: /info
+

.github/labeler.yml

@@ -77,7 +77,7 @@ app:core:
 app:electron:
   - changed-files:
       - any-glob-to-any-file:
-          - 'packages/frontend/electron/**/*'
+          - 'packages/frontend/apps/electron/**/*'
 
 app:server:
   - changed-files:

.github/renovate.json

@@ -23,12 +23,11 @@
       "groupName": "oxlint"
     },
     {
-      "groupName": "blocksuite-canary",
+      "groupName": "blocksuite",
       "matchPackagePatterns": ["^@blocksuite"],
       "excludePackageNames": ["@blocksuite/icons"],
       "rangeStrategy": "replace",
-      "followTag": "canary",
-      "enabled": false
+      "changelogUrl": "https://github.com/toeverything/blocksuite/blob/master/packages/blocks/CHANGELOG.md"
     },
     {
       "groupName": "all non-major dependencies",
@@ -41,6 +40,10 @@
       "groupName": "rust toolchain",
       "matchManagers": ["custom.regex"],
       "matchDepNames": ["rustc"]
+    },
+    {
+      "groupName": "nestjs",
+      "matchPackagePatterns": ["^@nestjs"]
     }
   ],
   "commitMessagePrefix": "chore: ",

.github/workflows/build-images.yml

@@ -6,11 +6,6 @@ on:
       flavor:
         type: string
         required: true
-  workflow_dispatch:
-    inputs:
-      flavor:
-        type: string
-        required: false
 
 env:
   NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
@@ -43,8 +38,8 @@ jobs:
           path: ./packages/backend/server/dist
           if-no-files-found: error
 
-  build-web-selfhost:
-    name: Build @affine/web selfhost
+  build-web:
+    name: Build @affine/web
     runs-on: ubuntu-latest
     environment: ${{ github.event.inputs.flavor }}
     steps:
@@ -57,21 +52,27 @@ jobs:
       - name: Build Core
         run: yarn nx build @affine/web --skip-nx-cache
         env:
+          R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
+          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           BUILD_TYPE: ${{ github.event.inputs.flavor }}
-          PUBLIC_PATH: '/'
-          SELF_HOSTED: true
+          CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
+          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+          SENTRY_PROJECT: 'affine-web'
+          SENTRY_RELEASE: ${{ steps.version.outputs.APP_VERSION }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+          PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
           MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
-      - name: Download selfhost fonts
-        run: node ./scripts/download-blocksuite-fonts.mjs
       - name: Upload web artifact
         uses: actions/upload-artifact@v4
         with:
-          name: selfhost-web
-          path: ./packages/frontend/web/dist
+          name: web
+          path: ./packages/frontend/apps/web/dist
           if-no-files-found: error
 
-  build-admin-selfhost:
-    name: Build @affine/admin selfhost
+  build-admin:
+    name: Build @affine/admin
     runs-on: ubuntu-latest
     environment: ${{ github.event.inputs.flavor }}
     steps:
@@ -81,20 +82,59 @@ jobs:
         uses: ./.github/actions/setup-version
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
-      - name: Build Core
+      - name: Build Admin
         run: yarn nx build @affine/admin --skip-nx-cache
         env:
+          R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
+          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           BUILD_TYPE: ${{ github.event.inputs.flavor }}
-          PUBLIC_PATH: '/admin/'
-          SELF_HOSTED: true
+          CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
+          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+          SENTRY_PROJECT: 'affine-admin'
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+          PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
           MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
       - name: Upload admin artifact
         uses: actions/upload-artifact@v4
         with:
-          name: selfhost-admin
+          name: admin
           path: ./packages/frontend/admin/dist
           if-no-files-found: error
 
+  build-mobile:
+    name: Build @affine/mobile
+    runs-on: ubuntu-latest
+    environment: ${{ github.event.inputs.flavor }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Version
+        id: version
+        uses: ./.github/actions/setup-version
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+      - name: Build Mobile
+        run: yarn nx build @affine/mobile --skip-nx-cache
+        env:
+          R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
+          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+          BUILD_TYPE: ${{ github.event.inputs.flavor }}
+          CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
+          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+          SENTRY_PROJECT: 'affine-mobile'
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+          PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
+          MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
+      - name: Upload mobile artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: mobile
+          path: ./packages/frontend/apps/mobile/dist
+          if-no-files-found: error
+
   build-server-native:
     name: Build Server native - ${{ matrix.targets.name }}
     runs-on: ubuntu-latest
@@ -131,13 +171,14 @@ jobs:
           path: ./packages/backend/native/server-native.node
           if-no-files-found: error
 
-  build-docker:
-    name: Build Docker
+  build-images:
+    name: Build Images
     runs-on: ubuntu-latest
     needs:
       - build-server
-      - build-web-selfhost
-      - build-admin-selfhost
+      - build-web
+      - build-mobile
+      - build-admin
       - build-server-native
     steps:
       - uses: actions/checkout@v4
@@ -195,16 +236,22 @@ jobs:
           registry-url: https://npm.pkg.github.com
           scope: '@toeverything'
 
-      - name: Download selfhost web artifact
+      - name: Download web artifact
         uses: actions/download-artifact@v4
         with:
-          name: selfhost-web
-          path: ./packages/frontend/web/dist
+          name: web
+          path: ./packages/frontend/apps/web/dist
 
-      - name: Download selfhost admin artifact
+      - name: Download mobile artifact
         uses: actions/download-artifact@v4
         with:
-          name: selfhost-admin
+          name: mobile
+          path: ./packages/frontend/apps/mobile/dist
+
+      - name: Download admin artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: admin
           path: ./packages/frontend/admin/dist
 
       - name: Install Node.js dependencies
@@ -220,6 +267,17 @@ jobs:
         id: version
         uses: ./.github/actions/setup-version
 
+      - name: Build front Dockerfile
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          pull: true
+          platforms: linux/amd64,linux/arm64
+          provenance: true
+          file: .github/deployment/front/Dockerfile
+          tags: ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}
+
       - name: Build graphql Dockerfile
         uses: docker/build-push-action@v6
         with:

.github/workflows/build-selfhost-image.yml

@@ -20,6 +20,6 @@ permissions:
 jobs:
   build-image:
     name: Build Image
-    uses: ./.github/workflows/build-server-image.yml
+    uses: ./.github/workflows/build-images.yml
     with:
       flavor: ${{ github.event.inputs.flavor }}

.github/workflows/build-test.yml

@@ -90,7 +90,7 @@ jobs:
           electron-install: false
           full-cache: true
       - name: Run i18n codegen
-        run: yarn i18n-codegen gen
+        run: yarn workspace @affine/i18n build
       - name: Run ESLint
         run: yarn lint:eslint --max-warnings=0
       - name: Run Prettier
@@ -117,7 +117,7 @@ jobs:
     name: E2E Test
     runs-on: ubuntu-latest
     env:
-      DISTRIBUTION: browser
+      DISTRIBUTION: web
       IN_CI_TEST: true
     strategy:
       fail-fast: false
@@ -143,11 +143,41 @@ jobs:
           path: ./test-results
           if-no-files-found: ignore
 
+  e2e-mobile-test:
+    name: E2E Mobile Test
+    runs-on: ubuntu-latest
+    env:
+      DISTRIBUTION: mobile
+      IN_CI_TEST: true
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4, 5]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          playwright-install: true
+          electron-install: false
+          full-cache: true
+
+      - name: Run playwright tests
+        run: yarn workspace @affine-test/affine-mobile e2e --forbid-only --shard=${{ matrix.shard }}/${{ strategy.job-total }}
+
+      - name: Upload test results
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-e2e-mobile-${{ matrix.shard }}
+          path: ./test-results
+          if-no-files-found: ignore
+
   e2e-migration-test:
     name: E2E Migration Test
     runs-on: ubuntu-latest
     env:
-      DISTRIBUTION: browser
+      DISTRIBUTION: web
     steps:
       - uses: actions/checkout@v4
       - name: Setup Node.js
@@ -174,13 +204,13 @@ jobs:
     needs:
       - build-native
     env:
-      DISTRIBUTION: browser
+      DISTRIBUTION: web
     steps:
       - uses: actions/checkout@v4
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
         with:
-          electron-install: false
+          electron-install: true
           full-cache: true
 
       - name: Download affine.linux-x64-gnu.node
@@ -266,8 +296,8 @@ jobs:
           path: ./packages/backend/native/server-native.node
           if-no-files-found: error
 
-  build-web:
-    name: Build @affine/web
+  build-electron-renderer:
+    name: Build @affine/electron renderer
     runs-on: ubuntu-latest
 
     steps:
@@ -277,13 +307,13 @@ jobs:
         with:
           electron-install: false
           full-cache: true
-      - name: Build Web
+      - name: Build Electron renderer
         # always skip cache because its fast, and cache configuration is always changing
-        run: yarn nx build @affine/web --skip-nx-cache
+        run: yarn build
         env:
-          DISTRIBUTION: 'desktop'
+          DISTRIBUTION: desktop
       - name: zip web
-        run: tar -czf dist.tar.gz --directory=packages/frontend/electron/renderer/dist .
+        run: tar -czf dist.tar.gz --directory=packages/frontend/apps/electron/renderer/dist .
       - name: Upload web artifact
         uses: actions/upload-artifact@v4
         with:
@@ -297,7 +327,8 @@ jobs:
     needs: build-server-native
     env:
       NODE_ENV: test
-      DISTRIBUTION: browser
+      DISTRIBUTION: web
+      DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
     services:
       postgres:
         image: postgres
@@ -330,27 +361,13 @@ jobs:
           name: server-native.node
           path: ./packages/backend/server
 
-      - name: Initialize database
-        run: |
-          psql -h localhost -U postgres -c "CREATE DATABASE affine;"
-          psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
-          psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
-        env:
-          PGPASSWORD: affine
-
-      - name: Run init-db script
-        run: |
-          yarn workspace @affine/server exec prisma generate
-          yarn workspace @affine/server exec prisma db push
-          yarn workspace @affine/server data-migration run
-        env:
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+      - name: Prepare Server Test Environment
+        uses: ./.github/actions/server-test-env
 
       - name: Run server tests
         run: yarn workspace @affine/server test:coverage
         env:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
-          DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
           COPILOT_OPENAI_API_KEY: 'use_fake_openai_api_key'
 
       - name: Upload server test coverage results
@@ -362,11 +379,157 @@ jobs:
           name: affine
           fail_ci_if_error: false
 
+  copilot-api-test:
+    name: Server Copilot Api Test
+    runs-on: ubuntu-latest
+    needs:
+      - build-server-native
+    env:
+      NODE_ENV: test
+      DISTRIBUTION: web
+      DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: affine
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+      mailer:
+        image: mailhog/mailhog
+        ports:
+          - 1025:1025
+          - 8025:8025
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check blocksuite update
+        id: check-blocksuite-update
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          if node ./scripts/detect-blocksuite-update.mjs "$BASE_REF"; then
+            echo "skip=false" >> $GITHUB_OUTPUT
+          else
+            echo "skip=true" >> $GITHUB_OUTPUT
+          fi
+
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            backend:
+              - 'packages/backend/server/src/**'
+
+      - name: Setup Node.js
+        if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.filter.outputs.backend == 'true' }}
+        uses: ./.github/actions/setup-node
+        with:
+          electron-install: false
+          full-cache: true
+
+      - name: Download server-native.node
+        if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.filter.outputs.backend == 'true' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: server-native.node
+          path: ./packages/backend/server
+
+      - name: Prepare Server Test Environment
+        if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.filter.outputs.backend == 'true' }}
+        uses: ./.github/actions/server-test-env
+
+      - name: Run server tests
+        if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.filter.outputs.backend == 'true' }}
+        run: yarn workspace @affine/server test:copilot:coverage --forbid-only
+        env:
+          CARGO_TARGET_DIR: '${{ github.workspace }}/target'
+          COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
+          COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
+
+      - name: Upload server test coverage results
+        if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.filter.outputs.backend == 'true' }}
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./packages/backend/server/.coverage/lcov.info
+          flags: server-test
+          name: affine
+          fail_ci_if_error: false
+
+  copilot-e2e-test:
+    name: Server Copilot E2E Test
+    runs-on: ubuntu-latest
+    env:
+      DISTRIBUTION: web
+      DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+      IN_CI_TEST: true
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3]
+        shardTotal: [3]
+    needs:
+      - build-server-native
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: affine
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check blocksuite update
+        id: check-blocksuite-update
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          if node ./scripts/detect-blocksuite-update.mjs "$BASE_REF"; then
+            echo "skip=false" >> $GITHUB_OUTPUT
+          else
+            echo "skip=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Setup Node.js
+        if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' }}
+        uses: ./.github/actions/setup-node
+        with:
+          playwright-install: true
+          electron-install: false
+          hard-link-nm: false
+
+      - name: Download server-native.node
+        if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: server-native.node
+          path: ./packages/backend/server
+
+      - name: Run Copilot E2E Test ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' }}
+        uses: ./.github/actions/copilot-test
+        with:
+          script: yarn workspace @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+          openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
+          fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
+
   server-e2e-test:
     name: ${{ matrix.tests.name }}
     runs-on: ubuntu-latest
     env:
-      DISTRIBUTION: browser
+      DISTRIBUTION: web
       DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
       IN_CI_TEST: true
     strategy:
@@ -382,6 +545,10 @@ jobs:
           - name: 'Server Desktop E2E Test'
             script: |
               yarn workspace @affine/electron build:dev
+              # Workaround for Electron apps failing to initialize on Ubuntu 24.04 due to AppArmor restrictions
+              # Disables unprivileged user namespaces restriction to allow Electron apps to run
+              # Reference: https://github.com/electron/electron/issues/42510
+              sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
               xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine-test/affine-desktop-cloud e2e
     needs:
       - build-server-native
@@ -424,19 +591,8 @@ jobs:
           name: affine.linux-x64-gnu.node
           path: ./packages/frontend/native
 
-      - name: Initialize database
-        run: |
-          psql -h localhost -U postgres -c "CREATE DATABASE affine;"
-          psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
-          psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
-        env:
-          PGPASSWORD: affine
-
-      - name: Run init-db script
-        run: |
-          yarn workspace @affine/server exec prisma generate
-          yarn workspace @affine/server exec prisma db push
-          yarn workspace @affine/server data-migration run
+      - name: Prepare Server Test Environment
+        uses: ./.github/actions/server-test-env
 
       - name: ${{ matrix.tests.name }}
         run: |
@@ -490,7 +646,7 @@ jobs:
               test: true,
             }
     needs:
-      - build-web
+      - build-electron-renderer
       - build-native
     steps:
       - uses: actions/checkout@v4
@@ -524,20 +680,25 @@ jobs:
       - name: Download web artifact
         uses: ./.github/actions/download-web
         with:
-          path: packages/frontend/electron/resources/web-static
+          path: packages/frontend/apps/electron/resources/web-static
 
       - name: Build Desktop Layers
         run: yarn workspace @affine/electron build
 
       - name: Run desktop tests
         if: ${{ matrix.spec.os == 'ubuntu-latest' }}
-        run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine-test/affine-desktop e2e
+        run: |
+          # Workaround for Electron apps failing to initialize on Ubuntu 24.04 due to AppArmor restrictions
+          # Disables unprivileged user namespaces restriction to allow Electron apps to run
+          # Reference: https://github.com/electron/electron/issues/42510
+          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine-test/affine-desktop e2e
 
       - name: Run desktop tests
         if: ${{ matrix.spec.test && matrix.spec.os != 'ubuntu-latest' }}
         run: yarn workspace @affine-test/affine-desktop e2e
 
-      - name: Make bundle
+      - name: Make bundle (macOS)
         if: ${{ matrix.spec.target == 'aarch64-apple-darwin' }}
         env:
           SKIP_BUNDLE: true
@@ -545,8 +706,15 @@ jobs:
           HOIST_NODE_MODULES: 1
         run: yarn workspace @affine/electron package --platform=darwin --arch=arm64
 
-      - name: Make AppImage
-        run: yarn workspace @affine/electron make --platform=linux --arch=x64
+      - name: Make Bundle (Linux)
+        run: |
+          sudo add-apt-repository universe
+          sudo apt install -y libfuse2 elfutils flatpak flatpak-builder
+          flatpak remote-add --user --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo
+          flatpak update
+          # some flatpak deps need git protocol.file.allow
+          git config --global protocol.file.allow always
+          yarn workspace @affine/electron make --platform=linux --arch=x64
         if: ${{ matrix.spec.target == 'x86_64-unknown-linux-gnu' }}
         env:
           SKIP_WEB_BUILD: 1
@@ -565,17 +733,32 @@ jobs:
           path: ./test-results
           if-no-files-found: ignore
 
+  test-build-mobile-app:
+    uses: ./.github/workflows/release-mobile.yml
+    with:
+      build-type: canary
+      build-target: development
+    secrets: inherit
+    permissions:
+      id-token: 'write'
+
   test-done:
     needs:
       - analyze
       - lint
       - check-yarn-binary
       - e2e-test
+      - e2e-mobile-test
       - e2e-migration-test
       - unit-test
+      - build-native
+      - build-server-native
+      - build-electron-renderer
       - server-test
+      - copilot-e2e-test
       - server-e2e-test
       - desktop-test
+      - test-build-mobile-app
     if: always()
     runs-on: ubuntu-latest
     name: 3, 2, 1 Launch

.github/workflows/copilot-test-automatically.yml

@@ -0,0 +1,29 @@
+name: Copilot Test Automatically
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-canary.[0-9]+'
+  schedule:
+    - cron: '0 8 * * *'
+  workflow_dispatch:
+
+permissions:
+  actions: write
+
+jobs:
+  dispatch-test:
+    runs-on: ubuntu-latest
+    name: Setup Test
+    steps:
+      - name: dispatch test by tag
+        if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: copilot-test.yml
+      - name: dispatch test by schedule
+        if: ${{ github.event_name == 'schedule' }}
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: copilot-test.yml
+          ref: canary

.github/workflows/copilot-test.yml

@@ -0,0 +1,190 @@
+name: Copilot Cron Test
+
+on:
+  workflow_dispatch:
+
+env:
+  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+  PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/node_modules/.cache/ms-playwright
+
+jobs:
+  build-server-native:
+    name: Build Server native
+    runs-on: ubuntu-latest
+    env:
+      CARGO_PROFILE_RELEASE_DEBUG: '1'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          extra-flags: workspaces focus @affine/server-native
+          electron-install: false
+      - name: Build Rust
+        uses: ./.github/actions/build-rust
+        with:
+          target: 'x86_64-unknown-linux-gnu'
+          package: '@affine/server-native'
+          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+      - name: Upload server-native.node
+        uses: actions/upload-artifact@v4
+        with:
+          name: server-native.node
+          path: ./packages/backend/native/server-native.node
+          if-no-files-found: error
+
+  copilot-api-test:
+    name: Server Copilot Api Test
+    runs-on: ubuntu-latest
+    needs:
+      - build-server-native
+    env:
+      NODE_ENV: test
+      DISTRIBUTION: web
+      DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: affine
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+      mailer:
+        image: mailhog/mailhog
+        ports:
+          - 1025:1025
+          - 8025:8025
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          playwright-install: true
+          electron-install: false
+          full-cache: true
+
+      - name: Download server-native.node
+        uses: actions/download-artifact@v4
+        with:
+          name: server-native.node
+          path: ./packages/backend/server
+
+      - name: Prepare Server Test Environment
+        uses: ./.github/actions/server-test-env
+
+      - name: Run server tests
+        run: yarn workspace @affine/server test:copilot:coverage --forbid-only
+        env:
+          CARGO_TARGET_DIR: '${{ github.workspace }}/target'
+          COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
+          COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
+
+      - name: Upload server test coverage results
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./packages/backend/server/.coverage/lcov.info
+          flags: server-test
+          name: affine
+          fail_ci_if_error: false
+
+  copilot-e2e-test:
+    name: Server Copilot E2E Test
+    runs-on: ubuntu-latest
+    env:
+      DISTRIBUTION: web
+      DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+      IN_CI_TEST: true
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3]
+        shardTotal: [3]
+    needs:
+      - build-server-native
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: affine
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          playwright-install: true
+          electron-install: false
+          hard-link-nm: false
+
+      - name: Download server-native.node
+        uses: actions/download-artifact@v4
+        with:
+          name: server-native.node
+          path: ./packages/backend/server
+
+      - name: Run Copilot E2E Test ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        uses: ./.github/actions/copilot-test
+        with:
+          script: yarn workspace @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+          openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
+          fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
+
+  test-done:
+    needs:
+      - copilot-api-test
+      - copilot-e2e-test
+    if: always()
+    runs-on: ubuntu-latest
+    name: Post test result message
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          extra-flags: 'workspaces focus @affine/copilot-result'
+          electron-install: false
+      - name: Post Success event to a Slack channel
+        if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
+        run: node ./tools/copilot-result/index.js
+        env:
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          BRANCH_SHA: ${{ github.sha }}
+          BRANCH_NAME: ${{ github.ref }}
+          COPILOT_RESULT: success
+      - name: Post Failed event to a Slack channel
+        id: failed-slack
+        if: ${{ always() && contains(needs.*.result, 'failure') }}
+        run: node ./tools/copilot-result/index.js
+        env:
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          BRANCH_SHA: ${{ github.sha }}
+          BRANCH_NAME: ${{ github.ref }}
+          COPILOT_RESULT: failed
+      - name: Post Cancel event to a Slack channel
+        id: cancel-slack
+        if: ${{ always() && contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'failure') }}
+        run: node ./tools/copilot-result/index.js
+        env:
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          BRANCH_SHA: ${{ github.sha }}
+          BRANCH_NAME: ${{ github.ref }}
+          COPILOT_RESULT: canceled

.github/workflows/deploy.yml

@@ -21,133 +21,60 @@ permissions:
   packages: 'write'
 
 jobs:
-  build-server-image:
-    name: Build Server Image
-    uses: ./.github/workflows/build-server-image.yml
+  output-prev-version:
+    name: Output previous version
+    runs-on: ubuntu-latest
+    environment: ${{ github.event.inputs.flavor }}
+    outputs:
+      prev: ${{ steps.print.outputs.version }}
+      namespace: ${{ steps.print.outputs.namespace }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Auth to Cluster
+        uses: './.github/actions/cluster-auth'
+        with:
+          gcp-project-number: ${{ secrets.GCP_PROJECT_NUMBER }}
+          gcp-project-id: ${{ secrets.GCP_PROJECT_ID }}
+          service-account: ${{ secrets.GCP_HELM_DEPLOY_SERVICE_ACCOUNT }}
+          cluster-name: ${{ secrets.GCP_CLUSTER_NAME }}
+          cluster-location: ${{ secrets.GCP_CLUSTER_LOCATION }}
+      - name: Output previous version
+        id: print
+        run: |
+          namespace=""
+          if [ "${{ github.event.inputs.flavor }}" = "canary" ]; then
+            namespace="dev"
+          elif [ "${{ github.event.inputs.flavor }}" = "beta" ]; then
+            namespace="beta"
+          elif [ "${{ github.event.inputs.flavor }}" = "stable" ]; then
+            namespace="production"
+          else
+            echo "Invalid flavor: ${{ github.event.inputs.flavor }}"
+            exit 1
+          fi
+
+          echo "Namespace set to: $namespace"
+
+          # Get the previous version from the deployment
+          prev_version=$(kubectl get deployment -n $namespace affine-graphql -o=jsonpath='{.spec.template.spec.containers[0].image}' | awk -F '-' '{print $3}')
+
+          echo "Previous version: $prev_version"
+          echo "version=$prev_version" >> $GITHUB_OUTPUT
+          echo "namesapce=$namespace" >> $GITHUB_OUTPUT
+
+  build-images:
+    name: Build Images
+    uses: ./.github/workflows/build-images.yml
+    secrets: inherit
     with:
       flavor: ${{ github.event.inputs.flavor }}
 
-  build-web:
-    name: Build @affine/web
-    runs-on: ubuntu-latest
-    environment: ${{ github.event.inputs.flavor }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Version
-        id: version
-        uses: ./.github/actions/setup-version
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-      - name: Build Core
-        run: yarn nx build @affine/web --skip-nx-cache
-        env:
-          R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
-          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
-          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-          BUILD_TYPE: ${{ github.event.inputs.flavor }}
-          CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
-          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
-          SENTRY_PROJECT: 'affine-web'
-          SENTRY_RELEASE: ${{ steps.version.outputs.APP_VERSION }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
-          PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
-          MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
-      - name: Upload web artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: web
-          path: ./packages/frontend/web/dist
-          if-no-files-found: error
-
-  build-admin:
-    name: Build @affine/admin
-    runs-on: ubuntu-latest
-    environment: ${{ github.event.inputs.flavor }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Version
-        id: version
-        uses: ./.github/actions/setup-version
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-      - name: Build Core
-        run: yarn nx build @affine/admin --skip-nx-cache
-        env:
-          R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
-          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
-          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-          BUILD_TYPE: ${{ github.event.inputs.flavor }}
-          CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
-          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
-          SENTRY_PROJECT: 'affine-admin'
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
-          PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
-          MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
-      - name: Upload admin artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: admin
-          path: ./packages/frontend/admin/dist
-          if-no-files-found: error
-
-  build-frontend-image:
-    name: Build Frontend Image
-    runs-on: ubuntu-latest
-    needs:
-      - build-web
-      - build-admin
-    steps:
-      - uses: actions/checkout@v4
-      - name: Download web artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: web
-          path: ./packages/frontend/web/dist
-      - name: Download admin artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: admin
-          path: ./packages/frontend/admin/dist
-      - name: Setup env
-        run: |
-          echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
-          if [ -z "${{ inputs.flavor }}" ]
-          then
-            echo "RELEASE_FLAVOR=canary" >> "$GITHUB_ENV"
-          else
-            echo "RELEASE_FLAVOR=${{ inputs.flavor }}" >> "$GITHUB_ENV"
-          fi
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          logout: false
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Build front Dockerfile
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          pull: true
-          platforms: linux/amd64,linux/arm64
-          provenance: true
-          file: .github/deployment/front/Dockerfile
-          tags: ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}
-
   deploy:
     name: Deploy to cluster
     if: ${{ github.event_name == 'workflow_dispatch' }}
     environment: ${{ github.event.inputs.flavor }}
     needs:
-      - build-frontend-image
-      - build-server-image
+      - build-images
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -193,3 +120,94 @@ jobs:
           STRIPE_API_KEY: ${{ secrets.STRIPE_API_KEY }}
           STRIPE_WEBHOOK_KEY: ${{ secrets.STRIPE_WEBHOOK_KEY }}
           STATIC_IP_NAME: ${{ secrets.STATIC_IP_NAME }}
+
+  deploy-done:
+    needs:
+      - output-prev-version
+      - build-images
+      - deploy
+    if: always()
+    runs-on: ubuntu-latest
+    name: Post deploy message
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/checkout@v4
+        with:
+          repository: toeverything/blocksuite
+          path: blocksuite
+          fetch-depth: 0
+          fetch-tags: true
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          extra-flags: 'workspaces focus @affine/changelog'
+          electron-install: false
+      - name: Output deployed info
+        if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
+        id: set_info
+        run: |
+          if [ "${{ github.event.inputs.flavor }}" = "canary" ]; then
+            echo "deployed_url=https://affine.fail" >> $GITHUB_OUTPUT
+          elif [ "${{ github.event.inputs.flavor }}" = "beta" ]; then
+            echo "deployed_url=https://insider.affine.pro" >> $GITHUB_OUTPUT
+          elif [ "${{ github.event.inputs.flavor }}" = "stable" ]; then
+            echo "deployed_url=https://app.affine.pro" >> $GITHUB_OUTPUT
+          else
+            exit 1
+          fi
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      - name: Post Success event to a Slack channel
+        if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
+        run: node ./tools/changelog/index.js
+        env:
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          DEPLOYED_URL: ${{ steps.set_info.outputs.deployed_url }}
+          PREV_VERSION: ${{ needs.output-prev-version.outputs.prev }}
+          NAMESPACE: ${{ needs.output-prev-version.outputs.namespace }}
+          DEPLOYMENT: 'SERVER'
+          FLAVOR: ${{ github.event.inputs.flavor }}
+          BLOCKSUITE_REPO_PATH: ${{ github.workspace }}/blocksuite
+      - name: Post Failed event to a Slack channel
+        id: failed-slack
+        uses: slackapi/slack-github-action@v1.27.0
+        if: ${{ always() && contains(needs.*.result, 'failure') }}
+        with:
+          channel-id: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Backend deploy failed `${{ github.event.inputs.flavor }}`>",
+                    "type": "mrkdwn"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+      - name: Post Cancel event to a Slack channel
+        id: cancel-slack
+        uses: slackapi/slack-github-action@v1.27.0
+        if: ${{ always() && contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'failure') }}
+        with:
+          channel-id: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Backend deploy cancelled `${{ github.event.inputs.flavor }}`>",
+                    "type": "mrkdwn"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/languages-sync.yml

@@ -1,35 +0,0 @@
-name: Languages Sync
-
-on:
-  push:
-    branches: ['canary']
-    paths:
-      - 'packages/frontend/i18n/**'
-      - '.github/workflows/languages-sync.yml'
-      - '!.github/actions/setup-node/action.yml'
-  pull_request_target:
-    branches: ['canary']
-    paths:
-      - 'packages/frontend/i18n/**'
-      - '.github/workflows/languages-sync.yml'
-      - '!.github/actions/setup-node/action.yml'
-  workflow_dispatch:
-
-jobs:
-  main:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-      - name: Check Language Key
-        if: github.ref != 'refs/heads/canary'
-        run: yarn workspace @affine/i18n run sync-languages:check
-        env:
-          TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}
-
-      - name: Sync Languages
-        if: github.ref == 'refs/heads/canary'
-        run: yarn workspace @affine/i18n run sync-languages
-        env:
-          TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}

.github/workflows/release-automatically.yml

@@ -1,4 +1,4 @@
-name: Release Desktop Automatically
+name: Release Desktop/Mobile Automatically
 
 on:
   push:
@@ -23,6 +23,7 @@ jobs:
         with:
           workflow: release-desktop.yml
           inputs: '{ "build-type": "canary", "is-draft": false, "is-pre-release": true }'
+
       - name: dispatch desktop release by schedule
         if: ${{ github.event_name == 'schedule' }}
         uses: benc-uk/workflow-dispatch@v1
@@ -30,3 +31,8 @@ jobs:
           workflow: release-desktop.yml
           inputs: '{ "build-type": "canary", "is-draft": false, "is-pre-release": true }'
           ref: canary
+      - name: dispatch desktop release by tag
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: release-mobile.yml
+          inputs: '{ "build-type": "canary", "build-target": "distribution" }'

.github/workflows/release-desktop.yml

@@ -32,7 +32,7 @@ permissions:
 
 env:
   BUILD_TYPE: ${{ github.event.inputs.build-type }}
-  DEBUG: napi:*
+  DEBUG: 'affine:*,napi:*'
   APP_NAME: affine
   MACOSX_DEPLOYMENT_TARGET: '10.13'
 
@@ -67,7 +67,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: web
-          path: packages/frontend/electron/resources/web-static
+          path: packages/frontend/apps/electron/resources/web-static
 
   make-distribution:
     strategy:
@@ -87,6 +87,7 @@ jobs:
             target: x86_64-unknown-linux-gnu
     runs-on: ${{ matrix.spec.runner }}
     needs: before-make
+    environment: ${{ github.event.inputs.build-type }}
     env:
       APPLE_ID: ${{ secrets.APPLE_ID }}
       APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
@@ -119,7 +120,7 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: web
-          path: packages/frontend/electron/resources/web-static
+          path: packages/frontend/apps/electron/resources/web-static
 
       - name: Build Desktop Layers
         run: yarn workspace @affine/electron build
@@ -131,35 +132,42 @@ jobs:
           p12-file-base64: ${{ secrets.CERTIFICATES_P12 }}
           p12-password: ${{ secrets.CERTIFICATES_P12_PASSWORD }}
 
-      - name: Install fuse on Linux (for patching AppImage)
+      - name: Install additional dependencies on Linux
         if: ${{ matrix.spec.platform == 'linux' }}
         run: |
           sudo add-apt-repository universe
-          sudo apt install libfuse2 -y
+          sudo apt install -y libfuse2 elfutils flatpak flatpak-builder
+          flatpak remote-add --user --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo
+          flatpak update
+          # some flatpak deps need git protocol.file.allow
+          git config --global protocol.file.allow always
 
       - name: make
         run: yarn workspace @affine/electron make --platform=${{ matrix.spec.platform }} --arch=${{ matrix.spec.arch }}
         env:
           SKIP_WEB_BUILD: 1
           HOIST_NODE_MODULES: 1
+          DEBUG: '*'
 
       - name: signing DMG
         if: ${{ matrix.spec.platform == 'darwin' }}
         run: |
-          codesign --force --sign "Developer ID Application: TOEVERYTHING PTE. LTD." packages/frontend/electron/out/${{ env.BUILD_TYPE }}/make/AFFiNE.dmg
+          codesign --force --sign "Developer ID Application: TOEVERYTHING PTE. LTD." packages/frontend/apps/electron/out/${{ env.BUILD_TYPE }}/make/AFFiNE.dmg
 
       - name: Save artifacts (mac)
         if: ${{ matrix.spec.platform == 'darwin' }}
         run: |
           mkdir -p builds
-          mv packages/frontend/electron/out/*/make/*.dmg ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-macos-${{ matrix.spec.arch }}.dmg
-          mv packages/frontend/electron/out/*/make/zip/darwin/${{ matrix.spec.arch }}/*.zip ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-macos-${{ matrix.spec.arch }}.zip
+          mv packages/frontend/apps/electron/out/*/make/*.dmg ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-macos-${{ matrix.spec.arch }}.dmg
+          mv packages/frontend/apps/electron/out/*/make/zip/darwin/${{ matrix.spec.arch }}/*.zip ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-macos-${{ matrix.spec.arch }}.zip
       - name: Save artifacts (linux)
         if: ${{ matrix.spec.platform == 'linux' }}
         run: |
           mkdir -p builds
-          mv packages/frontend/electron/out/*/make/zip/linux/x64/*.zip ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-x64.zip
-          mv packages/frontend/electron/out/*/make/*.AppImage ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-x64.appimage
+          mv packages/frontend/apps/electron/out/*/make/zip/linux/${{ matrix.spec.arch }}/*.zip ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ matrix.spec.arch }}.zip
+          mv packages/frontend/apps/electron/out/*/make/*.AppImage ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ matrix.spec.arch }}.appimage
+          mv packages/frontend/apps/electron/out/*/make/deb/${{ matrix.spec.arch }}/*.deb ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ matrix.spec.arch }}.deb
+          mv packages/frontend/apps/electron/out/*/make/flatpak/*/*.flatpak ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ matrix.spec.arch }}.flatpak
 
       - uses: actions/attest-build-provenance@v1
         if: ${{ matrix.spec.platform == 'darwin' }}
@@ -174,7 +182,7 @@ jobs:
           subject-path: |
             ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-x64.zip
             ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-x64.appimage
-
+            ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-x64.deb
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
         with:
@@ -182,6 +190,7 @@ jobs:
           path: builds
 
   package-distribution-windows:
+    environment: ${{ github.event.inputs.build-type }}
     strategy:
       matrix:
         spec:
@@ -189,10 +198,15 @@ jobs:
             platform: win32
             arch: x64
             target: x86_64-pc-windows-msvc
+          - runner: windows-latest
+            platform: win32
+            arch: arm64
+            target: aarch64-pc-windows-msvc
     runs-on: ${{ matrix.spec.runner }}
     needs: before-make
     outputs:
-      FILES_TO_BE_SIGNED: ${{ steps.get_files_to_be_signed.outputs.FILES_TO_BE_SIGNED }}
+      FILES_TO_BE_SIGNED_x64: ${{ steps.get_files_to_be_signed.outputs.FILES_TO_BE_SIGNED_x64 }}
+      FILES_TO_BE_SIGNED_arm64: ${{ steps.get_files_to_be_signed.outputs.FILES_TO_BE_SIGNED_arm64 }}
     env:
       SKIP_GENERATE_ASSETS: 1
       SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
@@ -221,7 +235,7 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: web
-          path: packages/frontend/electron/resources/web-static
+          path: packages/frontend/apps/electron/resources/web-static
 
       - name: Build Desktop Layers
         run: yarn workspace @affine/electron build
@@ -235,12 +249,12 @@ jobs:
       - name: get all files to be signed
         id: get_files_to_be_signed
         run: |
-          Set-Variable -Name FILES_TO_BE_SIGNED -Value ((Get-ChildItem -Path packages/frontend/electron/out -Recurse -File | Where-Object { $_.Extension -in @(".exe", ".node", ".dll", ".msi") } | ForEach-Object { '"' + $_.FullName.Replace((Get-Location).Path + '\packages\frontend\electron\out\', '') + '"' }) -join ' ')
-          "FILES_TO_BE_SIGNED=$FILES_TO_BE_SIGNED" >> $env:GITHUB_OUTPUT
+          Set-Variable -Name FILES_TO_BE_SIGNED -Value ((Get-ChildItem -Path packages/frontend/apps/electron/out -Recurse -File | Where-Object { $_.Extension -in @(".exe", ".node", ".dll", ".msi") } | ForEach-Object { '"' + $_.FullName.Replace((Get-Location).Path + '\packages\frontend\apps\electron\out\', '') + '"' }) -join ' ')
+          "FILES_TO_BE_SIGNED_${{ matrix.spec.arch }}=$FILES_TO_BE_SIGNED" >> $env:GITHUB_OUTPUT
           echo $FILES_TO_BE_SIGNED
 
       - name: Zip artifacts for faster upload
-        run: Compress-Archive -CompressionLevel Fastest -Path packages/frontend/electron/out/* -DestinationPath archive.zip
+        run: Compress-Archive -CompressionLevel Fastest -Path packages/frontend/apps/electron/out/* -DestinationPath archive.zip
 
       - name: Save packaged artifacts for signing
         uses: actions/upload-artifact@v4
@@ -250,25 +264,35 @@ jobs:
             archive.zip
             !**/*.map
 
-  sign-packaged-artifacts-windows:
+  sign-packaged-artifacts-windows_x64:
     needs: package-distribution-windows
     uses: ./.github/workflows/windows-signer.yml
     with:
-      files: ${{ needs.package-distribution-windows.outputs.FILES_TO_BE_SIGNED }}
+      files: ${{ needs.package-distribution-windows.outputs.FILES_TO_BE_SIGNED_x64 }}
       artifact-name: packaged-win32-x64
 
+  sign-packaged-artifacts-windows_arm64:
+    needs: package-distribution-windows
+    uses: ./.github/workflows/windows-signer.yml
+    with:
+      files: ${{ needs.package-distribution-windows.outputs.FILES_TO_BE_SIGNED_arm64 }}
+      artifact-name: packaged-win32-arm64
+
   make-windows-installer:
-    needs: sign-packaged-artifacts-windows
+    needs:
+      - sign-packaged-artifacts-windows_x64
+      - sign-packaged-artifacts-windows_arm64
     strategy:
       matrix:
         spec:
-          - runner: windows-latest
-            platform: win32
+          - platform: win32
             arch: x64
-            target: x86_64-pc-windows-msvc
-    runs-on: ${{ matrix.spec.runner }}
+          - platform: win32
+            arch: arm64
+    runs-on: windows-latest
     outputs:
-      FILES_TO_BE_SIGNED: ${{ steps.get_files_to_be_signed.outputs.FILES_TO_BE_SIGNED }}
+      FILES_TO_BE_SIGNED_x64: ${{ steps.get_files_to_be_signed.outputs.FILES_TO_BE_SIGNED_x64 }}
+      FILES_TO_BE_SIGNED_arm64: ${{ steps.get_files_to_be_signed.outputs.FILES_TO_BE_SIGNED_arm64 }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Version
@@ -281,13 +305,15 @@ jobs:
           extra-flags: workspaces focus @affine/electron @affine/monorepo
           hard-link-nm: false
           nmHoistingLimits: workspaces
+        env:
+          npm_config_arch: ${{ matrix.spec.arch }}
       - name: Download and overwrite packaged artifacts
         uses: actions/download-artifact@v4
         with:
           name: signed-packaged-${{ matrix.spec.platform }}-${{ matrix.spec.arch }}
           path: .
       - name: unzip file
-        run: Expand-Archive -Path signed.zip -DestinationPath packages/frontend/electron/out
+        run: Expand-Archive -Path signed.zip -DestinationPath packages/frontend/apps/electron/out
 
       - name: Make squirrel.windows installer
         run: yarn workspace @affine/electron make-squirrel --platform=${{ matrix.spec.platform }} --arch=${{ matrix.spec.arch }}
@@ -296,13 +322,13 @@ jobs:
         run: yarn workspace @affine/electron make-nsis --platform=${{ matrix.spec.platform }} --arch=${{ matrix.spec.arch }}
 
       - name: Zip artifacts for faster upload
-        run: Compress-Archive -CompressionLevel Fastest -Path packages/frontend/electron/out/${{ env.BUILD_TYPE }}/make/* -DestinationPath archive.zip
+        run: Compress-Archive -CompressionLevel Fastest -Path packages/frontend/apps/electron/out/${{ env.BUILD_TYPE }}/make/* -DestinationPath archive.zip
 
       - name: get all files to be signed
         id: get_files_to_be_signed
         run: |
-          Set-Variable -Name FILES_TO_BE_SIGNED -Value ((Get-ChildItem -Path packages/frontend/electron/out/${{ env.BUILD_TYPE }}/make -Recurse -File | Where-Object { $_.Extension -in @(".exe", ".node", ".dll", ".msi") } | ForEach-Object { '"' + $_.FullName.Replace((Get-Location).Path + '\packages\frontend\electron\out\${{ env.BUILD_TYPE }}\make\', '') + '"' }) -join ' ')
-          "FILES_TO_BE_SIGNED=$FILES_TO_BE_SIGNED" >> $env:GITHUB_OUTPUT
+          Set-Variable -Name FILES_TO_BE_SIGNED -Value ((Get-ChildItem -Path packages/frontend/apps/electron/out/${{ env.BUILD_TYPE }}/make -Recurse -File | Where-Object { $_.Extension -in @(".exe", ".node", ".dll", ".msi") } | ForEach-Object { '"' + $_.FullName.Replace((Get-Location).Path + '\packages\frontend\apps\electron\out\${{ env.BUILD_TYPE }}\make\', '') + '"' }) -join ' ')
+          "FILES_TO_BE_SIGNED_${{ matrix.spec.arch }}=$FILES_TO_BE_SIGNED" >> $env:GITHUB_OUTPUT
           echo $FILES_TO_BE_SIGNED
 
       - name: Save installer for signing
@@ -311,22 +337,36 @@ jobs:
           name: installer-${{ matrix.spec.platform }}-${{ matrix.spec.arch }}
           path: archive.zip
 
-  sign-installer-artifacts-windows:
+  sign-installer-artifacts-windows-x64:
     needs: make-windows-installer
     uses: ./.github/workflows/windows-signer.yml
     with:
-      files: ${{ needs.make-windows-installer.outputs.FILES_TO_BE_SIGNED }}
+      files: ${{ needs.make-windows-installer.outputs.FILES_TO_BE_SIGNED_x64 }}
       artifact-name: installer-win32-x64
 
+  sign-installer-artifacts-windows-arm64:
+    needs: make-windows-installer
+    uses: ./.github/workflows/windows-signer.yml
+    with:
+      files: ${{ needs.make-windows-installer.outputs.FILES_TO_BE_SIGNED_arm64 }}
+      artifact-name: installer-win32-arm64
+
   finalize-installer-windows:
-    needs: [sign-installer-artifacts-windows, before-make]
+    needs:
+      [
+        sign-installer-artifacts-windows-x64,
+        sign-installer-artifacts-windows-arm64,
+        before-make,
+      ]
     strategy:
       matrix:
         spec:
           - runner: windows-latest
             platform: win32
             arch: x64
-            target: x86_64-pc-windows-msvc
+          - runner: windows-latest
+            platform: win32
+            arch: arm64
     runs-on: ${{ matrix.spec.runner }}
     steps:
       - name: Download and overwrite installer artifacts
@@ -335,21 +375,21 @@ jobs:
           name: signed-installer-${{ matrix.spec.platform }}-${{ matrix.spec.arch }}
           path: .
       - name: unzip file
-        run: Expand-Archive -Path signed.zip -DestinationPath packages/frontend/electron/out/${{ env.BUILD_TYPE }}/make
+        run: Expand-Archive -Path signed.zip -DestinationPath packages/frontend/apps/electron/out/${{ env.BUILD_TYPE }}/make
 
       - name: Save artifacts
         run: |
           mkdir -p builds
-          mv packages/frontend/electron/out/*/make/zip/win32/x64/AFFiNE*-win32-x64-*.zip ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-x64.zip
-          mv packages/frontend/electron/out/*/make/squirrel.windows/x64/*.exe ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-x64.exe
-          mv packages/frontend/electron/out/*/make/nsis.windows/x64/*.exe ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-x64.nsis.exe
+          mv packages/frontend/apps/electron/out/*/make/zip/win32/${{ matrix.spec.arch }}/AFFiNE*-win32-${{ matrix.spec.arch }}-*.zip ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.zip
+          mv packages/frontend/apps/electron/out/*/make/squirrel.windows/${{ matrix.spec.arch }}/*.exe ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.exe
+          mv packages/frontend/apps/electron/out/*/make/nsis.windows/${{ matrix.spec.arch }}/*.exe ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.nsis.exe
 
       - uses: actions/attest-build-provenance@v1
         with:
           subject-path: |
-            ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-x64.zip
-            ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-x64.exe
-            ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-x64.nsis.exe
+            ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.zip
+            ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.exe
+            ./builds/affine-${{ needs.before-make.outputs.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.nsis.exe
 
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
@@ -384,6 +424,11 @@ jobs:
         with:
           name: affine-win32-x64-builds
           path: ./
+      - name: Download Artifacts (windows-arm64)
+        uses: actions/download-artifact@v4
+        with:
+          name: affine-win32-arm64-builds
+          path: ./
       - name: Download Artifacts (linux-x64)
         uses: actions/download-artifact@v4
         with:
@@ -394,7 +439,7 @@ jobs:
           node-version: 20
       - name: Generate Release yml
         run: |
-          node ./packages/frontend/electron/scripts/generate-yml.js
+          node ./packages/frontend/apps/electron/scripts/generate-yml.js
         env:
           RELEASE_VERSION: ${{ needs.before-make.outputs.RELEASE_VERSION }}
       - name: Create Release Draft
@@ -411,6 +456,8 @@ jobs:
             ./*.dmg
             ./*.exe
             ./*.appimage
+            ./*.deb
+            ./*.flatpak
             ./*.apk
             ./*.yml
       - name: Create Nightly Release Draft
@@ -433,5 +480,7 @@ jobs:
             ./*.dmg
             ./*.exe
             ./*.appimage
+            ./*.deb
             ./*.apk
+            ./*.flatpak
             ./*.yml

.github/workflows/release-mobile.yml

@@ -0,0 +1,211 @@
+name: Release Mobile App
+
+on:
+  workflow_call:
+    inputs:
+      build-target:
+        description: 'Build Target'
+        type: string
+        required: true
+      build-type:
+        description: 'Build Type'
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      build-target:
+        description: 'Build Target'
+        type: choice
+        required: true
+        default: distribution
+        options:
+          - development
+          - distribution
+      build-type:
+        description: 'Build Type'
+        type: choice
+        required: true
+        default: canary
+        options:
+          - canary
+          - beta
+          - stable
+env:
+  BUILD_TYPE: ${{ inputs.build-type || github.event.inputs.build-type }}
+  BUILD_TARGET: ${{ inputs.build-target || github.event.inputs.build-target }}
+  DEBUG: napi:*
+  KEYCHAIN_NAME: ${{ github.workspace }}/signing_temp
+
+jobs:
+  build-ios-web:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.build-type || github.event.inputs.build-type }}
+    outputs:
+      RELEASE_VERSION: ${{ steps.version.outputs.APP_VERSION }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Version
+        id: version
+        uses: ./.github/actions/setup-version
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+      - name: Setup @sentry/cli
+        uses: ./.github/actions/setup-sentry
+      - name: Build Mobile
+        run: yarn nx build @affine/ios --skip-nx-cache
+        env:
+          PUBLIC_PATH: '/'
+          MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
+          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+          SENTRY_PROJECT: 'affine'
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+          SENTRY_RELEASE: ${{ steps.version.outputs.APP_VERSION }}
+          RELEASE_VERSION: ${{ steps.version.outputs.APP_VERSION }}
+          SKIP_NX_CACHE: 'true'
+      - name: Upload ios artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ios
+          path: packages/frontend/apps/ios/dist
+
+  build-android-web:
+    runs-on: ubuntu-latest
+    environment: ${{ github.event.inputs.build-type || inputs.build-type }}
+    outputs:
+      RELEASE_VERSION: ${{ steps.version.outputs.APP_VERSION }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Version
+        id: version
+        uses: ./.github/actions/setup-version
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+      - name: Setup @sentry/cli
+        uses: ./.github/actions/setup-sentry
+      - name: Build Mobile
+        run: yarn nx build @affine/android --skip-nx-cache
+        env:
+          PUBLIC_PATH: '/'
+          MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
+          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+          SENTRY_PROJECT: 'affine'
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+          SENTRY_RELEASE: ${{ steps.version.outputs.APP_VERSION }}
+          RELEASE_VERSION: ${{ steps.version.outputs.APP_VERSION }}
+          SKIP_NX_CACHE: 'true'
+      - name: Upload android artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: android
+          path: packages/frontend/apps/android/dist
+
+  ios:
+    runs-on: macos-latest
+    needs:
+      - build-ios-web
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download mobile artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ios
+          path: packages/frontend/apps/ios/dist
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        timeout-minutes: 10
+        with:
+          extra-flags: workspaces focus @affine/ios
+          playwright-install: false
+          electron-install: false
+          hard-link-nm: false
+          enableScripts: false
+      - name: Cap sync
+        run: yarn workspace @affine/ios cap sync
+      - name: Signing By Apple Developer ID
+        uses: apple-actions/import-codesign-certs@v3
+        id: import-codesign-certs
+        with:
+          p12-file-base64: ${{ secrets.CERTIFICATES_P12_MOBILE }}
+          p12-password: ${{ secrets.CERTIFICATES_P12_PASSWORD_MOBILE }}
+      - uses: maxim-lobanov/setup-xcode@v1
+        with:
+          xcode-version: latest-stable
+      - name: Testflight
+        if: ${{ env.BUILD_TYPE != 'stable' }}
+        working-directory: packages/frontend/apps/ios/App
+        run: |
+          echo -n "${{ env.BUILD_PROVISION_PROFILE }}" | base64 --decode -o $PP_PATH
+          mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles
+          cp $PP_PATH ~/Library/MobileDevice/Provisioning\ Profiles
+          fastlane beta
+        env:
+          BUILD_PROVISION_PROFILE: ${{ secrets.BUILD_PROVISION_PROFILE }}
+          PP_PATH: ${{ runner.temp }}/build_pp.mobileprovision
+          APPLE_STORE_CONNECT_API_KEY_ID: ${{ secrets.APPLE_STORE_CONNECT_API_KEY_ID }}
+          APPLE_STORE_CONNECT_API_ISSUER_ID: ${{ secrets.APPLE_STORE_CONNECT_API_ISSUER_ID }}
+          APPLE_STORE_CONNECT_API_KEY: ${{ secrets.APPLE_STORE_CONNECT_API_KEY }}
+
+  android:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: 'write'
+    needs:
+      - build-android-web
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download mobile artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: android
+          path: packages/frontend/apps/android/dist
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        timeout-minutes: 10
+        with:
+          extra-flags: workspaces focus @affine/android @affine/playstore-auto-bump
+          playwright-install: false
+          electron-install: false
+          hard-link-nm: false
+          enableScripts: false
+      - name: Cap sync
+        run: yarn workspace @affine/android cap sync
+      - name: Auth gcloud
+        id: auth
+        uses: google-github-actions/auth@v2
+        if: ${{ env.BUILD_TARGET == 'distribution' }}
+        with:
+          workload_identity_provider: 'projects/${{ secrets.GCP_PROJECT_NUMBER }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions-helm-deploy'
+          service_account: '${{ secrets.GCP_HELM_DEPLOY_SERVICE_ACCOUNT }}'
+          token_format: 'access_token'
+          project_id: '${{ secrets.GCP_PROJECT_ID }}'
+          access_token_scopes: 'https://www.googleapis.com/auth/androidpublisher'
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - name: Auto increment version code
+        id: bump
+        if: ${{ env.BUILD_TARGET == 'distribution' }}
+        run: yarn workspace @affine/playstore-auto-bump bump
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.auth.outputs.credentials_file_path }}
+      - name: Build
+        run: |
+          echo -n "${{ env.AFFINE_ANDROID_SIGN_KEYSTORE }}" | base64 --decode > packages/frontend/apps/android/affine.keystore
+          yarn workspace @affine/android cap build android
+        env:
+          AFFINE_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.AFFINE_ANDROID_KEYSTORE_PASSWORD }}
+          AFFINE_ANDROID_KEYSTORE_ALIAS_PASSWORD: ${{ secrets.AFFINE_ANDROID_KEYSTORE_ALIAS_PASSWORD }}
+          AFFINE_ANDROID_SIGN_KEYSTORE: ${{ secrets.AFFINE_ANDROID_SIGN_KEYSTORE }}
+      - name: Upload to Google Play
+        uses: r0adkll/upload-google-play@v1
+        if: ${{ env.BUILD_TARGET == 'distribution' }}
+        with:
+          serviceAccountJson: ${{ steps.auth.outputs.credentials_file_path }}
+          packageName: app.affine.pro
+          releaseFiles: packages/frontend/apps/android/App/app/build/outputs/bundle/release/app-release-signed.aab
+          track: internal
+          status: draft
+          existingEditId: ${{ steps.bump.outputs.EDIT_ID }}

.github/workflows/sync-i18n.yml

@@ -0,0 +1,42 @@
+name: Sync I18n with Crowdin
+
+on:
+  push:
+    branches:
+      - canary
+    paths:
+      - 'packages/frontend/i18n/**'
+  workflow_dispatch:
+
+jobs:
+  synchronize-with-crowdin:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Crowdin action
+        uses: crowdin/github-action@v2
+        with:
+          upload_sources: true
+          upload_translations: true
+          download_translations: true
+          auto_approve_imported: true
+          import_eq_suggestions: true
+          export_only_approved: true
+          skip_untranslated_strings: true
+          localization_branch_name: l10n_crowdin_translations
+          create_pull_request: true
+          pull_request_title: 'chore(i18n): sync translations'
+          pull_request_body: 'New Crowdin translations by [Crowdin GH Action](https://github.com/crowdin/github-action)'
+          pull_request_base_branch_name: 'canary'
+          config: packages/frontend/i18n/crowdin.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
+          CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}

.github/workflows/workers.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Publish
-        uses: cloudflare/wrangler-action@v3.7.0
+        uses: cloudflare/wrangler-action@v3.12.1
         with:
           apiToken: ${{ secrets.CF_API_TOKEN }}
           accountId: ${{ secrets.CF_ACCOUNT_ID }}

.gitignore

@@ -59,7 +59,6 @@ Thumbs.db
 .vercel
 out/
 storybook-static
-i18n-generated.ts
 
 test-results
 playwright-report

.nvmrc

@@ -1 +1 @@
-20.15.1
+20.18.0

.prettierignore

@@ -14,6 +14,7 @@ public
 packages/backend/server/src/schema.gql
 packages/backend/server/src/fundamentals/error/errors.gen.ts
 packages/frontend/i18n/src/i18n-generated.ts
+packages/frontend/i18n/src/i18n-completenesses.json
 packages/frontend/graphql/src/graphql/index.ts
 tests/affine-legacy/**/static
 .yarnrc.yml

.yarnrc.yml

@@ -12,4 +12,4 @@ npmPublishAccess: public
 
 npmPublishRegistry: "https://registry.npmjs.org"
 
-yarnPath: .yarn/releases/yarn-4.3.1.cjs
+yarnPath: .yarn/releases/yarn-4.5.1.cjs

Cargo.toml

@@ -3,26 +3,27 @@ members  = ["./packages/backend/native", "./packages/frontend/native", "./packag
 resolver = "2"
 
 [workspace.dependencies]
-anyhow      = "1"
-chrono      = "0.4"
-dotenv      = "0.15"
-file-format = { version = "0.25", features = ["reader"] }
-mimalloc    = "0.1"
-napi        = { version = "3.0.0-alpha.1", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] }
-napi-build  = { version = "2" }
-napi-derive = { version = "3.0.0-alpha.1" }
-notify      = { version = "6", features = ["serde"] }
-once_cell   = "1"
-parking_lot = "0.12"
-rand        = "0.8"
-serde       = "1"
-serde_json  = "1"
-sha3        = "0.10"
-sqlx        = { version = "0.7", default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] }
-tiktoken-rs = "0.5"
-tokio       = "1.37"
-uuid        = "1.8"
-y-octo      = { git = "https://github.com/y-crdt/y-octo.git", branch = "main" }
+anyhow       = "1"
+chrono       = "0.4"
+dotenv       = "0.15"
+file-format  = { version = "0.26", features = ["reader"] }
+mimalloc     = "0.1"
+napi         = { version = "3.0.0-alpha.12", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] }
+napi-build   = { version = "2" }
+napi-derive  = { version = "3.0.0-alpha.12" }
+notify       = { version = "7", features = ["serde"] }
+once_cell    = "1"
+parking_lot  = "0.12"
+rand         = "0.8"
+serde        = "1"
+serde_json   = "1"
+sha3         = "0.10"
+sqlx         = { version = "0.8", default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] }
+tiktoken-rs  = "0.6"
+tokio        = "1.37"
+uuid         = "1.8"
+v_htmlescape = "0.15"
+y-octo       = { git = "https://github.com/y-crdt/y-octo.git", branch = "main" }
 
 [profile.dev.package.sqlx-macros]
 opt-level = 3

README.md

@@ -1,7 +1,7 @@
 <div align="center">
 
 <h1 style="border-bottom: none">
-    <b><a href="https://affine.pro">AFFiNE.PRO</a></b><br />
+    <b><a href="https://affine.pro">AFFiNE.Pro</a></b><br />
     Write, Draw and Plan All at Once
     <br>
 </h1>
@@ -33,7 +33,6 @@
 [![Releases](https://img.shields.io/github/downloads/toeverything/AFFiNE/total)](https://github.com/toeverything/AFFiNE/releases/latest)
 [![All Contributors][all-contributors-badge]](#contributors)
 [![TypeScript-version-icon]](https://www.typescriptlang.org/)
-[![Rust-version-icon]](https://www.rust-lang.org/)
 
 </div>
 
@@ -55,7 +54,7 @@ Star us, and you will receive all release notifications from GitHub without any
 
 ## What is AFFiNE
 
-AFFiNE is an open-source, all-in-one workspace and an operating system for all the building blocks that assemble your knowledge base and much more -- wiki, knowledge management, presentation and digital assets. It's a better alternative to Notion and Miro.
+[AFFiNE](https://affine.pro) is an open-source, all-in-one workspace and an operating system for all the building blocks that assemble your knowledge base and much more -- wiki, knowledge management, presentation and digital assets. It's a better alternative to Notion and Miro.
 
 ## Features
 
@@ -65,7 +64,7 @@ AFFiNE is an open-source, all-in-one workspace and an operating system for all t
 
 **Multimodal AI partner ready to kick in any work**
 
-- Write up professional work report? Turn an outline into expressive and presentable slides? Summary an article into a well-structured mindmap? Sorting your job plan and backlog for tasks? Or... draw and code prototype apps and web pages directly all with one prompt? With you, AFFiNE AI pushes your creativity to the edge of your imagination.
+- Write up professional work report? Turn an outline into expressive and presentable slides? Summary an article into a well-structured mindmap? Sorting your job plan and backlog for tasks? Or... draw and code prototype apps and web pages directly all with one prompt? With you, [AFFiNE AI](https://affine.pro/ai) pushes your creativity to the edge of your imagination,just like [Canvas AI](https://affine.pro/blog/best-canvas-ai) to generate mind map for brainstorming.
 
 **Local-first & Real-time collaborative**
 
@@ -108,6 +107,37 @@ Looking for **other ways to contribute** and wondering where to start? Check out
 
 If you have questions, you are welcome to contact us. One of the best places to get more info and learn more is in the [AFFiNE Community](https://community.affine.pro) where you can engage with other like-minded individuals.
 
+## Templates
+
+AFFiNE now provides pre-built [templates](https://affine.pro/templates) from our team. Following are the Top 10 most popular templates among AFFiNE users,if you want to contribute, you can contribute your own template so other people can use it too.
+
+- [vision board template](https://affine.pro/templates/category-vision-board-template)
+- [one pager template](https://affine.pro/templates/category-one-pager-template-free)
+- [sample lesson plan math template](https://affine.pro/templates/sample-lesson-plan-math-template)
+- [grr lesson plan template free](https://affine.pro/templates/grr-lesson-plan-template-free)
+- [free editable lesson plan template for pre k](https://affine.pro/templates/free-editable-lesson-plan-template-for-pre-k)
+- [high note collection planners](https://affine.pro/templates/high-note-collection-planners)
+- [digital planner](https://affine.pro/templates/category-digital-planner)
+- [ADHD Planner](https://affine.pro/templates/adhd-planner)
+- [Reading Log](https://affine.pro/templates/reading-log)
+- [Cornell Notes Template](https://affine.pro/templates/category-cornell-notes-template)
+
+## Blog
+
+Welcome to the AFFiNE blog section! Here, you’ll find the latest insights, tips, and guides on how to maximize your experience with AFFiNE and AFFiNE AI, the leading Canvas AI tool for flexible note-taking and creative organization.
+
+- [vision board template](https://affine.pro/blog/8-free-printable-vision-board-templates-examples-2023)
+- [itinerary template](https://affine.pro/blog/free-customized-travel-itinerary-planner-templates)
+- [one pager template](https://affine.pro/blog/top-12-one-pager-examples-how-to-create-your-own)
+- [cornell notes template](https://affine.pro/blog/the-cornell-notes-template-and-system-learning-tips)
+- [swot chart template](https://affine.pro/blog/top-10-free-editable-swot-analysis-template-examples)
+- [apps like luna task](https://affine.pro/blog/apps-like-luna-task)
+- [note taking ai from rough notes to mind map](https://affine.pro/blog/dynamic-AI-notes)
+- [canvas ai](https://affine.pro/blog/best-canvas-ai)
+- [one pager](https://affine.pro/blog/top-12-one-pager-examples-how-to-create-your-own)
+- [SOP Template](https://affine.pro/blog/how-to-write-sop-step-by-step-guide-5-best-free-tools-templates)
+- [Chore Chart](https://affine.pro/blog/10-best-free-chore-chart-templates-kids-adults)
+
 ## Ecosystem
 
 | Name                                             |                            |                                                                                                                                         |
@@ -176,6 +206,12 @@ Thanks to [Chromatic](https://www.chromatic.com/) for providing the visual testi
 
 ## License
 
+### Editions
+
+- AFFiNE Community Edition (CE) is the current available version, it's free for self-host under the MIT license.
+
+- AFFiNE Enterprise Edition (EE) is yet to be published, it will have more advanced features and enterprise-oriented offerings, including but not exclusive to rebranding and SSO, advanced admin and audit, etc., you may refer to https://affine.pro/pricing for more information
+
 See [LICENSE] for details.
 
 [all-contributors-badge]: https://img.shields.io/github/contributors/toeverything/AFFiNE
@@ -185,7 +221,6 @@ See [LICENSE] for details.
 [jobs available]: ./docs/jobs.md
 [latest packages]: https://github.com/toeverything/AFFiNE/pkgs/container/affine-self-hosted
 [contributor license agreement]: https://github.com/toeverything/affine/edit/canary/.github/CLA.md
-[rust-version-icon]: https://img.shields.io/badge/Rust-1.79.0-dea584
 [stars-icon]: https://img.shields.io/github/stars/toeverything/AFFiNE.svg?style=flat&logo=github&colorB=red&label=stars
 [codecov]: https://codecov.io/gh/toeverything/affine/branch/canary/graphs/badge.svg?branch=canary
 [node-version-icon]: https://img.shields.io/badge/node-%3E=18.16.1-success

SECURITY.md

@@ -6,8 +6,8 @@ We recommend users to always use the latest major version. Security updates will
 
 | Version         | Supported          |
 | --------------- | ------------------ |
-| 0.15.x (stable) | :white_check_mark: |
-| < 0.15.x        | :x:                |
+| 0.17.x (stable) | :white_check_mark: |
+| < 0.17.x        | :x:                |
 
 ## Reporting a Vulnerability
 

docs/developing-server.md

@@ -29,7 +29,7 @@ docker run --rm --name mailhog -p 1025:1025 -p 8025:8025 mailhog/mailhog
 
 ```
 docker ps
-docker exec -it CONTAINER_ID psql -U postgres ## change container_id
+docker exec -it affine-postgres psql -U postgres ## `affine-postgres` is the container name from the previous step
 ```
 
 ### in the terminal, following the example to user & table
@@ -96,6 +96,12 @@ yarn workspace @affine/native build
 yarn workspace @affine/server dev
 ```
 
+when server started, it will created a default user:
+
+email: dev@affine.pro
+name: Dev User
+password: dev
+
 ## start core (web)
 
 ```

docs/reference/package.json

@@ -19,5 +19,5 @@
     ],
     "ext": "ts,md,json"
   },
-  "version": "0.16.0"
+  "version": "0.17.0"
 }

nx.json

@@ -11,9 +11,7 @@
       }
     }
   },
-  "affected": {
-    "defaultBase": "canary"
-  },
+  "defaultBase": "canary",
   "namedInputs": {
     "default": ["{projectRoot}/**/*", "sharedGlobals"],
     "sharedGlobals": [
@@ -81,9 +79,6 @@
     "test": {
       "outputs": ["{workspaceRoot}/.nyc_output"],
       "inputs": [
-        {
-          "env": "ENABLE_PRELOADING"
-        },
         {
           "env": "COVERAGE"
         }
@@ -92,9 +87,6 @@
     "test:ui": {
       "outputs": ["{workspaceRoot}/.nyc_output"],
       "inputs": [
-        {
-          "env": "ENABLE_PRELOADING"
-        },
         {
           "env": "COVERAGE"
         }
@@ -102,11 +94,7 @@
     },
     "test:coverage": {
       "outputs": ["{workspaceRoot}/.nyc_output"],
-      "inputs": [
-        {
-          "env": "ENABLE_PRELOADING"
-        }
-      ]
+      "inputs": []
     }
   }
 }

oxlint.json

@@ -1,4 +1,8 @@
 {
+  "categories": {
+    "correctness": "error",
+    "perf": "error"
+  },
   "rules": {
     // allow
     "import/named": "allow",

package.json

@@ -1,12 +1,13 @@
 {
   "name": "@affine/monorepo",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "private": true,
   "author": "toeverything",
   "license": "MIT",
   "workspaces": [
     ".",
     "packages/*/*",
+    "packages/frontend/apps/*",
     "tools/*",
     "docs/reference",
     "tools/@types/*",
@@ -18,8 +19,8 @@
   },
   "scripts": {
     "dev": "yarn workspace @affine/cli dev",
+    "build": "yarn workspace @affine/cli bundle",
     "dev:electron": "yarn workspace @affine/electron dev",
-    "build": "yarn nx build @affine/web",
     "build:electron": "yarn nx build @affine/electron",
     "build:server-native": "yarn nx run-many -t build -p @affine/server-native",
     "start:web-static": "yarn workspace @affine/web static-server",
@@ -28,14 +29,14 @@
     "lint:eslint:fix": "yarn lint:eslint --fix",
     "lint:prettier": "prettier --ignore-unknown --cache --check .",
     "lint:prettier:fix": "prettier --ignore-unknown --cache --write .",
-    "lint:ox": "oxlint -c oxlint.json --deny-warnings --import-plugin -D correctness -D perf",
+    "lint:ox": "oxlint -c oxlint.json --deny-warnings --import-plugin",
     "lint": "yarn lint:eslint && yarn lint:prettier",
     "lint:fix": "yarn lint:eslint:fix && yarn lint:prettier:fix",
     "test": "vitest --run",
     "test:ui": "vitest --ui",
     "test:coverage": "vitest run --coverage",
     "typecheck": "tsc -b tsconfig.json",
-    "postinstall": "node ./scripts/check-version.mjs && yarn i18n-codegen gen && yarn husky install",
+    "postinstall": "node ./scripts/check-version.mjs && yarn workspace @affine/i18n i18n-codegen gen && yarn husky install",
     "prepare": "husky"
   },
   "lint-staged": {
@@ -52,30 +53,25 @@
     ]
   },
   "devDependencies": {
-    "@affine-test/kit": "workspace:*",
     "@affine/cli": "workspace:*",
-    "@commitlint/cli": "^19.2.1",
-    "@commitlint/config-conventional": "^19.1.0",
-    "@faker-js/faker": "^8.4.1",
+    "@capacitor/cli": "^6.1.2",
+    "@faker-js/faker": "^9.0.0",
     "@istanbuljs/schema": "^0.1.3",
     "@magic-works/i18n-codegen": "^0.6.0",
-    "@nx/vite": "^19.5.3",
-    "@playwright/test": "=1.44.1",
+    "@playwright/test": "=1.48.2",
+    "@swc/core": "^1.9.1",
     "@taplo/cli": "^0.7.0",
-    "@testing-library/react": "^16.0.0",
     "@toeverything/infra": "workspace:*",
     "@types/affine__env": "workspace:*",
-    "@types/eslint": "^8.56.7",
+    "@types/eslint": "^9.0.0",
     "@types/node": "^20.12.7",
     "@typescript-eslint/eslint-plugin": "^7.6.0",
     "@typescript-eslint/parser": "^7.6.0",
     "@vanilla-extract/vite-plugin": "^4.0.7",
-    "@vanilla-extract/webpack-plugin": "^2.3.7",
-    "@vitejs/plugin-react-swc": "^3.6.0",
-    "@vitest/coverage-istanbul": "1.6.0",
-    "@vitest/ui": "1.6.0",
+    "@vitest/coverage-istanbul": "2.1.4",
+    "@vitest/ui": "2.1.4",
     "cross-env": "^7.0.3",
-    "electron": "~30.2.0",
+    "electron": "^33.0.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-import-x": "^0.5.0",
@@ -86,31 +82,22 @@
     "eslint-plugin-sonarjs": "^0.25.1",
     "eslint-plugin-unicorn": "^52.0.0",
     "eslint-plugin-unused-imports": "^3.1.0",
-    "eslint-plugin-vue": "^9.24.1",
-    "fake-indexeddb": "6.0.0",
-    "happy-dom": "^14.7.1",
+    "happy-dom": "^15.0.0",
     "husky": "^9.0.11",
     "lint-staged": "^15.2.2",
     "msw": "^2.3.0",
-    "nanoid": "^5.0.7",
-    "nx": "^19.0.0",
-    "nyc": "^17.0.0",
-    "oxlint": "0.7.1",
-    "prettier": "^3.2.5",
+    "nx": "^20.0.3",
+    "nx-cloud": "^19.1.0",
+    "oxlint": "0.11.1",
+    "prettier": "^3.3.3",
     "semver": "^7.6.0",
     "serve": "^14.2.1",
-    "string-width": "^7.1.0",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.4.5",
+    "typescript": "^5.6.3",
     "unplugin-swc": "^1.4.5",
     "vite": "^5.2.8",
-    "vite-plugin-istanbul": "^6.0.0",
-    "vite-plugin-static-copy": "^1.0.2",
-    "vitest": "1.6.0",
-    "vitest-fetch-mock": "^0.3.0",
-    "vitest-mock-extended": "^2.0.0"
+    "vitest": "2.1.4"
   },
-  "packageManager": "yarn@4.3.1",
+  "packageManager": "yarn@4.5.1",
   "resolutions": {
     "array-buffer-byte-length": "npm:@nolyfill/array-buffer-byte-length@latest",
     "array-includes": "npm:@nolyfill/array-includes@latest",
@@ -167,7 +154,7 @@
     "unbox-primitive": "npm:@nolyfill/unbox-primitive@latest",
     "which-boxed-primitive": "npm:@nolyfill/which-boxed-primitive@latest",
     "which-typed-array": "npm:@nolyfill/which-typed-array@latest",
-    "@reforged/maker-appimage/@electron-forge/maker-base": "7.4.0",
+    "@reforged/maker-appimage/@electron-forge/maker-base": "7.5.0",
     "macos-alias": "npm:@napi-rs/macos-alias@0.0.4",
     "fs-xattr": "npm:@napi-rs/xattr@latest"
   }

packages/backend/native/Cargo.toml

@@ -7,14 +7,15 @@ version = "1.0.0"
 crate-type = ["cdylib"]
 
 [dependencies]
-chrono      = { workspace = true }
-file-format = { workspace = true }
-napi        = { workspace = true }
-napi-derive = { workspace = true }
-rand        = { workspace = true }
-sha3        = { workspace = true }
-tiktoken-rs = { workspace = true }
-y-octo      = { workspace = true }
+chrono       = { workspace = true }
+file-format  = { workspace = true }
+napi         = { workspace = true }
+napi-derive  = { workspace = true }
+rand         = { workspace = true }
+sha3         = { workspace = true }
+tiktoken-rs  = { workspace = true }
+v_htmlescape = { workspace = true }
+y-octo       = { workspace = true }
 
 [target.'cfg(not(target_os = "linux"))'.dependencies]
 mimalloc = { workspace = true }

packages/backend/native/benchmark/index.js

@@ -7,6 +7,7 @@ import { fromModelName } from '../index.js';
 
 const bench = new Bench({
   iterations: 100,
+  warmup: true,
 });
 
 const FIXTURE = `Please extract the items that can be used as tasks from the following content, and send them to me in the format provided by the template. The extracted items should cover as much of the following content as possible.
@@ -36,7 +37,6 @@ bench
     fromModelName('gpt-4o').count(FIXTURE);
   });
 
-await bench.warmup();
 await bench.run();
 
 console.table(bench.table());

packages/backend/native/index.d.ts

@@ -8,6 +8,8 @@ export declare function fromModelName(modelName: string): Tokenizer | null
 
 export declare function getMime(input: Uint8Array): string
 
+export declare function htmlSanitize(input: string): string
+
 /**
  * Merge updates in form like `Y.applyUpdate(doc, update)` way and return the
  * result binary.

packages/backend/native/index.js

@@ -11,3 +11,4 @@ export const mintChallengeResponse = binding.mintChallengeResponse;
 export const getMime = binding.getMime;
 export const Tokenizer = binding.Tokenizer;
 export const fromModelName = binding.fromModelName;
+export const htmlSanitize = binding.htmlSanitize;

packages/backend/native/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@affine/server-native",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "engines": {
     "node": ">= 10.16.0 < 11 || >= 11.8.0"
   },
@@ -33,12 +33,12 @@
     "build:debug": "napi build"
   },
   "devDependencies": {
-    "@napi-rs/cli": "3.0.0-alpha.60",
+    "@napi-rs/cli": "3.0.0-alpha.64",
     "lib0": "^0.2.93",
-    "nx": "^19.0.0",
-    "nx-cloud": "^19.0.0",
+    "nx": "^20.0.3",
+    "nx-cloud": "^19.1.0",
     "tiktoken": "^1.0.15",
-    "tinybench": "^2.8.0",
+    "tinybench": "^3.0.0",
     "yjs": "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch"
   }
 }

packages/backend/native/project.json

@@ -12,12 +12,18 @@
         "script": "build"
       },
       "inputs": [
-        { "runtime": "rustc --version" },
-        { "runtime": "node -v" },
-        { "runtime": "clang --version" },
-        { "runtime": "cargo tree" }
+        { "fileset": "{workspaceRoot}/rust-toolchain.toml" },
+        { "fileset": "{workspaceRoot}/Cargo.lock" },
+        { "fileset": "{workspaceRoot}/packages/backend/native/**/*.rs" },
+        { "fileset": "{workspaceRoot}/packages/backend/native/Cargo.toml" },
+        {
+          "runtime": "rustc --version"
+        },
+        {
+          "externalDependencies": ["nx"]
+        }
       ],
-      "outputs": ["{projectRoot}/*.node", "{workspaceRoot}/*.node"]
+      "outputs": ["{projectRoot}/*.node"]
     }
   }
 }

packages/backend/native/src/html_sanitize.rs

@@ -0,0 +1,4 @@
+#[napi]
+pub fn html_sanitize(input: String) -> String {
+  v_htmlescape::escape(&input).to_string()
+}

packages/backend/native/src/lib.rs

@@ -2,6 +2,7 @@
 
 pub mod file_type;
 pub mod hashcash;
+pub mod html_sanitize;
 pub mod tiktoken;
 
 use std::fmt::{Debug, Display};

packages/backend/server/.gitignore

@@ -1 +1,2 @@
 .env
+static/

packages/backend/server/migrations/20240813095727_prompt_updated_field/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "ai_prompts_metadata" ADD COLUMN     "modified" BOOLEAN NOT NULL DEFAULT false,
+ADD COLUMN     "updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/backend/server/migrations/20240815064332_userspace/migration.sql

@@ -0,0 +1,13 @@
+-- CreateTable
+CREATE TABLE "user_snapshots" (
+    "user_id" VARCHAR NOT NULL,
+    "id" VARCHAR NOT NULL,
+    "blob" BYTEA NOT NULL,
+    "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMPTZ(3) NOT NULL,
+
+    CONSTRAINT "user_snapshots_pkey" PRIMARY KEY ("user_id","id")
+);
+
+-- AddForeignKey
+ALTER TABLE "user_snapshots" ADD CONSTRAINT "user_snapshots_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/backend/server/migrations/20240815084618_update_doc_tables/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - The primary key for the `updates` table will be changed. If it partially fails, the table could be left without primary key constraint.
+
+*/
+-- AlterTable
+ALTER TABLE "snapshots" ALTER COLUMN "seq" DROP NOT NULL;
+
+-- AlterTable
+ALTER TABLE "updates" DROP CONSTRAINT "updates_pkey",
+ALTER COLUMN "created_at" DROP DEFAULT,
+ALTER COLUMN "seq" DROP NOT NULL,
+ADD CONSTRAINT "updates_pkey" PRIMARY KEY ("workspace_id", "guid", "created_at");

packages/backend/server/migrations/20240826033024_editor_record/migration.sql

@@ -0,0 +1,21 @@
+-- AlterTable
+ALTER TABLE "snapshot_histories" ADD COLUMN     "created_by" VARCHAR;
+
+-- AlterTable
+ALTER TABLE "snapshots" ADD COLUMN     "created_by" VARCHAR,
+ADD COLUMN     "updated_by" VARCHAR;
+
+-- AlterTable
+ALTER TABLE "updates" ADD COLUMN     "created_by" VARCHAR;
+
+-- AddForeignKey
+ALTER TABLE "snapshots" ADD CONSTRAINT "snapshots_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "snapshots" ADD CONSTRAINT "snapshots_updated_by_fkey" FOREIGN KEY ("updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "updates" ADD CONSTRAINT "updates_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "snapshot_histories" ADD CONSTRAINT "snapshot_histories_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/backend/server/migrations/20240903033137_workspace_url_preview/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "workspaces" ADD COLUMN     "enable_url_preview" BOOLEAN NOT NULL DEFAULT false;

packages/backend/server/migrations/20240908111944_index_permission_user_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "workspace_user_permissions_user_id_idx" ON "workspace_user_permissions"("user_id");

packages/backend/server/migrations/20240908130725_add_workspace_level_snapshot_index/migration.sql

@@ -0,0 +1,12 @@
+/*
+  Warnings:
+
+  - The primary key for the `snapshots` table will be changed. If it partially fails, the table could be left without primary key constraint.
+
+*/
+-- AlterTable
+ALTER TABLE "snapshots" DROP CONSTRAINT "snapshots_pkey",
+ADD CONSTRAINT "snapshots_pkey" PRIMARY KEY ("workspace_id", "guid");
+
+-- CreateIndex
+CREATE INDEX "snapshots_workspace_id_updated_at_idx" ON "snapshots"("workspace_id", "updated_at");

packages/backend/server/migrations/20240909090850_ai_table_index/migration.sql

@@ -0,0 +1,8 @@
+-- CreateIndex
+CREATE INDEX "ai_sessions_messages_session_id_idx" ON "ai_sessions_messages"("session_id");
+
+-- CreateIndex
+CREATE INDEX "ai_sessions_metadata_user_id_idx" ON "ai_sessions_metadata"("user_id");
+
+-- CreateIndex
+CREATE INDEX "ai_sessions_metadata_user_id_workspace_id_idx" ON "ai_sessions_metadata"("user_id", "workspace_id");

packages/backend/server/migrations/20240909100000_lower_index_user_email/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "users_email_lowercase_idx" ON "users"(lower("email"))

packages/backend/server/migrations/20240924024058_onetime_payment_subscription/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "user_subscriptions" ADD COLUMN     "variant" VARCHAR(20);

packages/backend/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/server",
   "private": true,
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Affine Node.js server",
   "type": "module",
   "bin": {
@@ -12,7 +12,9 @@
     "start": "node --loader ts-node/esm/transpile-only.mjs ./src/index.ts",
     "dev": "nodemon ./src/index.ts",
     "test": "ava --concurrency 1 --serial",
+    "test:copilot": "ava --concurrency 1 --serial \"tests/**/copilot-*.e2e.ts\"",
     "test:coverage": "c8 ava --concurrency 1 --serial",
+    "test:copilot:coverage": "c8 ava --timeout=5m --concurrency 1 --serial \"tests/**/copilot-*.e2e.ts\"",
     "postinstall": "prisma generate",
     "data-migration": "node --loader ts-node/esm/transpile-only.mjs ./src/data/index.ts",
     "predeploy": "yarn prisma migrate deploy && node --import ./scripts/register.js ./dist/data/index.js run",
@@ -21,11 +23,10 @@
   "dependencies": {
     "@apollo/server": "^4.10.2",
     "@aws-sdk/client-s3": "^3.620.0",
-    "@fal-ai/serverless-client": "^0.13.0",
-    "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.19.0",
+    "@fal-ai/serverless-client": "^0.15.0",
+    "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.20.0",
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.2.0",
     "@google-cloud/opentelemetry-resource-util": "^2.2.0",
-    "@keyv/redis": "^2.8.4",
     "@nestjs/apollo": "^12.1.0",
     "@nestjs/common": "^10.3.7",
     "@nestjs/core": "^10.3.7",
@@ -34,26 +35,24 @@
     "@nestjs/platform-express": "^10.3.7",
     "@nestjs/platform-socket.io": "^10.3.7",
     "@nestjs/schedule": "^4.0.1",
-    "@nestjs/serve-static": "^4.0.2",
-    "@nestjs/throttler": "5.2.0",
+    "@nestjs/throttler": "6.2.1",
     "@nestjs/websockets": "^10.3.7",
-    "@node-rs/argon2": "^1.8.0",
+    "@node-rs/argon2": "^2.0.0",
     "@node-rs/crc32": "^1.10.0",
-    "@node-rs/jsonwebtoken": "^0.5.2",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/core": "^1.25.0",
-    "@opentelemetry/exporter-prometheus": "^0.52.0",
+    "@opentelemetry/exporter-prometheus": "^0.54.0",
     "@opentelemetry/exporter-zipkin": "^1.25.0",
     "@opentelemetry/host-metrics": "^0.35.2",
-    "@opentelemetry/instrumentation": "^0.52.0",
-    "@opentelemetry/instrumentation-graphql": "^0.42.0",
-    "@opentelemetry/instrumentation-http": "^0.52.0",
-    "@opentelemetry/instrumentation-ioredis": "^0.42.0",
-    "@opentelemetry/instrumentation-nestjs-core": "^0.39.0",
-    "@opentelemetry/instrumentation-socket.io": "^0.41.0",
+    "@opentelemetry/instrumentation": "^0.54.0",
+    "@opentelemetry/instrumentation-graphql": "^0.44.0",
+    "@opentelemetry/instrumentation-http": "^0.54.0",
+    "@opentelemetry/instrumentation-ioredis": "^0.44.0",
+    "@opentelemetry/instrumentation-nestjs-core": "^0.41.0",
+    "@opentelemetry/instrumentation-socket.io": "^0.43.0",
     "@opentelemetry/resources": "^1.25.0",
     "@opentelemetry/sdk-metrics": "^1.25.0",
-    "@opentelemetry/sdk-node": "^0.52.0",
+    "@opentelemetry/sdk-node": "^0.54.0",
     "@opentelemetry/sdk-trace-node": "^1.25.0",
     "@opentelemetry/semantic-conventions": "^1.25.0",
     "@prisma/client": "^5.15.0",
@@ -61,50 +60,42 @@
     "@socket.io/redis-adapter": "^8.3.0",
     "cookie-parser": "^1.4.6",
     "dotenv": "^16.4.5",
-    "dotenv-cli": "^7.4.1",
     "express": "^4.19.2",
     "fast-xml-parser": "^4.4.0",
     "get-stream": "^9.0.1",
     "graphql": "^16.8.1",
     "graphql-scalars": "^1.23.0",
-    "graphql-type-json": "^0.3.2",
-    "graphql-upload": "^16.0.2",
+    "graphql-upload": "^17.0.0",
     "html-validate": "^8.20.1",
     "ioredis": "^5.3.2",
-    "keyv": "^4.5.4",
+    "is-mobile": "^5.0.0",
+    "keyv": "^5.0.0",
     "lodash-es": "^4.17.21",
     "mixpanel": "^0.18.0",
     "mustache": "^4.2.0",
     "nanoid": "^5.0.7",
     "nest-commander": "^3.12.5",
-    "nestjs-throttler-storage-redis": "^0.4.1",
+    "nestjs-throttler-storage-redis": "^0.5.0",
     "nodemailer": "^6.9.13",
     "on-headers": "^1.0.2",
     "openai": "^4.33.0",
-    "parse-duration": "^1.1.0",
     "piscina": "^4.5.1",
-    "pretty-time": "^1.1.0",
     "prisma": "^5.12.1",
-    "prom-client": "^15.1.1",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.1",
-    "semver": "^7.6.0",
     "ses": "^1.4.1",
     "socket.io": "^4.7.5",
-    "stripe": "^16.0.0",
+    "stripe": "^17.0.0",
     "ts-node": "^10.9.2",
-    "typescript": "^5.4.5",
-    "ws": "^8.16.0",
+    "typescript": "^5.6.3",
     "yjs": "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch",
     "zod": "^3.22.4"
   },
   "devDependencies": {
     "@affine-test/kit": "workspace:*",
     "@affine/server-native": "workspace:*",
-    "@napi-rs/image": "^1.9.1",
     "@nestjs/testing": "^10.3.7",
     "@types/cookie-parser": "^1.4.7",
-    "@types/engine.io": "^3.1.10",
     "@types/express": "^4.17.21",
     "@types/graphql-upload": "^16.0.7",
     "@types/keyv": "^4.2.0",
@@ -114,14 +105,12 @@
     "@types/node": "^20.12.7",
     "@types/nodemailer": "^6.4.14",
     "@types/on-headers": "^1.0.3",
-    "@types/pretty-time": "^1.1.5",
     "@types/sinon": "^17.0.3",
     "@types/supertest": "^6.0.2",
-    "@types/ws": "^8.5.10",
     "ava": "^6.1.2",
     "c8": "^10.0.0",
     "nodemon": "^3.1.0",
-    "sinon": "^18.0.0",
+    "sinon": "^19.0.0",
     "supertest": "^7.0.0"
   },
   "ava": {
@@ -138,6 +127,7 @@
     ],
     "watchMode": {
       "ignoreChanges": [
+        "static/**",
         "**/*.gen.*"
       ]
     },

packages/backend/server/schema.prisma

@@ -32,6 +32,11 @@ model User {
   sessions              UserSession[]
   aiSessions            AiSession[]
   updatedRuntimeConfigs RuntimeConfig[]
+  userSnapshots         UserSnapshot[]
+  createdSnapshot       Snapshot[]                    @relation("createdSnapshot")
+  updatedSnapshot       Snapshot[]                    @relation("updatedSnapshot")
+  createdUpdate         Update[]                      @relation("createdUpdate")
+  createdHistory        SnapshotHistory[]             @relation("createdHistory")
 
   @@index([email])
   @@map("users")
@@ -57,12 +62,13 @@ model ConnectedAccount {
 }
 
 model Session {
-  id        String    @id @default(uuid()) @db.VarChar
-  expiresAt DateTime? @map("expires_at") @db.Timestamptz(3)
-  createdAt DateTime  @default(now()) @map("created_at") @db.Timestamptz(3)
-
+  id           String        @id @default(uuid()) @db.VarChar
+  createdAt    DateTime      @default(now()) @map("created_at") @db.Timestamptz(3)
   userSessions UserSession[]
 
+  // @deprecated use [UserSession.expiresAt]
+  deprecated_expiresAt DateTime? @map("expires_at") @db.Timestamptz(3)
+
   @@map("multiple_users_sessions")
 }
 
@@ -91,9 +97,10 @@ model VerificationToken {
 }
 
 model Workspace {
-  id        String   @id @default(uuid()) @db.VarChar
-  public    Boolean
-  createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
+  id               String   @id @default(uuid()) @db.VarChar
+  public           Boolean
+  enableUrlPreview Boolean  @default(false) @map("enable_url_preview")
+  createdAt        DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
 
   pages           WorkspacePage[]
   permissions     WorkspaceUserPermission[]
@@ -135,6 +142,8 @@ model WorkspaceUserPermission {
   workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)
 
   @@unique([workspaceId, userId])
+  // optimize for quering user's workspace permissions
+  @@index(userId)
   @@map("workspace_user_permissions")
 }
 
@@ -235,25 +244,61 @@ model Snapshot {
   workspaceId String   @map("workspace_id") @db.VarChar
   id          String   @default(uuid()) @map("guid") @db.VarChar
   blob        Bytes    @db.ByteA
-  seq         Int      @default(0) @db.Integer
   state       Bytes?   @db.ByteA
   createdAt   DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
   // the `updated_at` field will not record the time of record changed,
   // but the created time of last seen update that has been merged into snapshot.
   updatedAt   DateTime @map("updated_at") @db.Timestamptz(3)
+  createdBy   String?  @map("created_by") @db.VarChar
+  updatedBy   String?  @map("updated_by") @db.VarChar
 
-  @@id([id, workspaceId])
+  // should not delete origin snapshot even if user is deleted
+  // we only delete the snapshot if the workspace is deleted
+  createdByUser User? @relation(name: "createdSnapshot", fields: [createdBy], references: [id], onDelete: SetNull)
+  updatedByUser User? @relation(name: "updatedSnapshot", fields: [updatedBy], references: [id], onDelete: SetNull)
+
+  // @deprecated use updatedAt only
+  seq Int? @default(0) @db.Integer
+
+  // we need to clear all hanging updates and snapshots before enable the foreign key on workspaceId
+  // workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)
+
+  @@id([workspaceId, id])
+  @@index([workspaceId, updatedAt])
   @@map("snapshots")
 }
 
+// user snapshots are special snapshots for user storage like personal app settings, distinguished from workspace snapshots
+// basically they share the same structure with workspace snapshots
+// but for convenience, we don't fork the updates queue and hisotry for user snapshots, until we have to
+// which means all operation on user snapshot will happen in-pace
+model UserSnapshot {
+  userId    String   @map("user_id") @db.VarChar
+  id        String   @map("id") @db.VarChar
+  blob      Bytes    @db.ByteA
+  createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
+  updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz(3)
+
+  user User @relation(fields: [userId], references: [id], onDelete: Cascade)
+
+  @@id([userId, id])
+  @@map("user_snapshots")
+}
+
 model Update {
   workspaceId String   @map("workspace_id") @db.VarChar
   id          String   @map("guid") @db.VarChar
-  seq         Int      @db.Integer
   blob        Bytes    @db.ByteA
-  createdAt   DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
+  createdAt   DateTime @map("created_at") @db.Timestamptz(3)
+  createdBy   String?  @map("created_by") @db.VarChar
 
-  @@id([workspaceId, id, seq])
+  // will delete createor record if createor's account is deleted
+  createdByUser User? @relation(name: "createdUpdate", fields: [createdBy], references: [id], onDelete: SetNull)
+
+  // @deprecated use createdAt only
+  seq Int? @db.Integer
+
+  @@id([workspaceId, id, createdAt])
   @@map("updates")
 }
 
@@ -264,6 +309,10 @@ model SnapshotHistory {
   blob        Bytes    @db.ByteA
   state       Bytes?   @db.ByteA
   expiredAt   DateTime @map("expired_at") @db.Timestamptz(3)
+  createdBy   String?  @map("created_by") @db.VarChar
+
+  // will delete createor record if creator's account is deleted
+  createdByUser User? @relation(name: "createdHistory", fields: [createdBy], references: [id], onDelete: SetNull)
 
   @@id([workspaceId, id, timestamp])
   @@map("snapshot_histories")
@@ -283,9 +332,11 @@ model UserSubscription {
   id                   Int       @id @default(autoincrement()) @db.Integer
   userId               String    @map("user_id") @db.VarChar
   plan                 String    @db.VarChar(20)
-  // yearly/monthly
+  // yearly/monthly/lifetime
   recurring            String    @db.VarChar(20)
-  // subscription.id, null for linefetime payment
+  // onetime subscription or anything else
+  variant              String?    @db.VarChar(20)
+  // subscription.id, null for linefetime payment or one time payment subscription
   stripeSubscriptionId String?   @unique @map("stripe_subscription_id")
   // subscription.status, active/past_due/canceled/unpaid...
   status               String    @db.VarChar(20)
@@ -367,6 +418,9 @@ model AiPrompt {
   model     String   @db.VarChar
   config    Json?    @db.Json
   createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
+  updatedAt DateTime @default(now()) @map("updated_at") @db.Timestamptz(3)
+  // whether the prompt is modified by the admin panel
+  modified  Boolean  @default(false)
 
   messages AiPromptMessage[]
   sessions AiSession[]
@@ -386,6 +440,7 @@ model AiSessionMessage {
 
   session AiSession @relation(fields: [sessionId], references: [id], onDelete: Cascade)
 
+  @@index([sessionId])
   @@map("ai_sessions_messages")
 }
 
@@ -406,6 +461,8 @@ model AiSession {
   prompt   AiPrompt           @relation(fields: [promptName], references: [name], onDelete: Cascade)
   messages AiSessionMessage[]
 
+  @@index([userId])
+  @@index([userId, workspaceId])
   @@map("ai_sessions_metadata")
 }
 

packages/backend/server/src/app.controller.ts

@@ -3,7 +3,7 @@ import { Controller, Get } from '@nestjs/common';
 import { Public } from './core/auth';
 import { Config, SkipThrottle } from './fundamentals';
 
-@Controller('/')
+@Controller('/info')
 export class AppController {
   constructor(private readonly config: Config) {}
 
@@ -15,7 +15,7 @@ export class AppController {
       compatibility: this.config.version,
       message: `AFFiNE ${this.config.version} Server`,
       type: this.config.type,
-      flavor: this.config.flavor,
+      flavor: this.config.flavor.type,
     };
   }
 }

packages/backend/server/src/app.module.ts

@@ -1,5 +1,3 @@
-import { join } from 'node:path';
-
 import {
   DynamicModule,
   ForwardReference,
@@ -7,16 +5,17 @@ import {
   Module,
 } from '@nestjs/common';
 import { ScheduleModule } from '@nestjs/schedule';
-import { ServeStaticModule } from '@nestjs/serve-static';
 import { get } from 'lodash-es';
 
 import { AppController } from './app.controller';
 import { AuthModule } from './core/auth';
 import { ADD_ENABLED_FEATURES, ServerConfigModule } from './core/config';
-import { DocModule } from './core/doc';
+import { DocStorageModule } from './core/doc';
+import { DocRendererModule } from './core/doc-renderer';
 import { FeatureModule } from './core/features';
+import { PermissionModule } from './core/permission';
 import { QuotaModule } from './core/quota';
-import { CustomSetupModule } from './core/setup';
+import { SelfhostModule } from './core/selfhost';
 import { StorageModule } from './core/storage';
 import { SyncModule } from './core/sync';
 import { UserModule } from './core/user';
@@ -44,7 +43,6 @@ import { ENABLED_PLUGINS } from './plugins/registry';
 
 export const FunctionalityModules = [
   ConfigModule.forRoot(),
-  ScheduleModule.forRoot(),
   EventModule,
   CacheModule,
   MutexModule,
@@ -137,7 +135,7 @@ export class AppModuleBuilder {
   compile() {
     @Module({
       imports: this.modules,
-      controllers: this.config.isSelfhosted ? [] : [AppController],
+      controllers: [AppController],
     })
     class AppModule {}
 
@@ -145,20 +143,20 @@ export class AppModuleBuilder {
   }
 }
 
-function buildAppModule() {
+export function buildAppModule() {
   AFFiNE = mergeConfigOverride(AFFiNE);
   const factor = new AppModuleBuilder(AFFiNE);
 
   factor
-    // common fundamental modules
+    // basic
     .use(...FunctionalityModules)
     .useIf(config => config.flavor.sync, WebSocketModule)
 
     // auth
-    .use(UserModule, AuthModule)
+    .use(UserModule, AuthModule, PermissionModule)
 
     // business modules
-    .use(DocModule)
+    .use(FeatureModule, QuotaModule, DocStorageModule)
 
     // sync server only
     .useIf(config => config.flavor.sync, SyncModule)
@@ -166,27 +164,16 @@ function buildAppModule() {
     // graphql server only
     .useIf(
       config => config.flavor.graphql,
-      ServerConfigModule,
+      ScheduleModule.forRoot(),
       GqlModule,
       StorageModule,
-      WorkspaceModule,
-      FeatureModule,
-      QuotaModule
+      ServerConfigModule,
+      WorkspaceModule
     )
 
     // self hosted server only
-    .useIf(
-      config => config.isSelfhosted,
-      CustomSetupModule,
-      ServeStaticModule.forRoot({
-        rootPath: join('/app', 'static'),
-        exclude: ['/admin*'],
-      }),
-      ServeStaticModule.forRoot({
-        rootPath: join('/app', 'static', 'admin'),
-        serveRoot: '/admin',
-      })
-    );
+    .useIf(config => config.isSelfhosted, SelfhostModule)
+    .useIf(config => config.flavor.renderer, DocRendererModule);
 
   // plugin modules
   ENABLED_PLUGINS.forEach(name => {

packages/backend/server/src/app.ts

@@ -5,6 +5,7 @@ import cookieParser from 'cookie-parser';
 import graphqlUploadExpress from 'graphql-upload/graphqlUploadExpress.mjs';
 
 import { AuthGuard } from './core/auth';
+import { ENABLED_FEATURES } from './core/config/server-feature';
 import {
   CacheInterceptor,
   CloudThrottlerGuard,
@@ -23,6 +24,10 @@ export async function createApp() {
     logger: AFFiNE.affine.stable ? ['log'] : ['verbose'],
   });
 
+  if (AFFiNE.server.path) {
+    app.setGlobalPrefix(AFFiNE.server.path);
+  }
+
   app.use(serverTimingAndCache);
 
   app.use(
@@ -56,6 +61,7 @@ export async function createApp() {
       .init(AFFiNE.metrics.telemetry.token)
       .track('selfhost-server-started', {
         version: AFFiNE.version,
+        features: Array.from(ENABLED_FEATURES),
       });
   }
 

packages/backend/server/src/config/affine.env.ts

@@ -25,6 +25,7 @@ AFFiNE.ENV_MAP = {
   OAUTH_OIDC_CLAIM_MAP_EMAIL: 'plugins.oauth.providers.oidc.args.claim_email',
   OAUTH_OIDC_CLAIM_MAP_NAME: 'plugins.oauth.providers.oidc.args.claim_name',
   METRICS_CUSTOMER_IO_TOKEN: ['metrics.customerIo.token', 'string'],
+  CAPTCHA_TURNSTILE_SECRET: ['plugins.captcha.turnstile.secret', 'string'],
   COPILOT_OPENAI_API_KEY: 'plugins.copilot.openai.apiKey',
   COPILOT_FAL_API_KEY: 'plugins.copilot.fal.apiKey',
   COPILOT_UNSPLASH_API_KEY: 'plugins.copilot.unsplashKey',

packages/backend/server/src/config/affine.self.ts

@@ -71,6 +71,14 @@ AFFiNE.use('payment', {
 });
 AFFiNE.use('oauth');
 
+/* Captcha Plugin Default Config */
+AFFiNE.use('captcha', {
+  turnstile: {},
+  challenge: {
+    bits: 20,
+  },
+});
+
 if (AFFiNE.deploy) {
   AFFiNE.mailer = {
     service: 'gmail',

packages/backend/server/src/config/affine.ts

@@ -95,6 +95,15 @@ AFFiNE.server.port = 3010;
 // });
 //
 //
+// /* Captcha Plugin Default Config */
+// AFFiNE.plugins.use('captcha', {
+//   turnstile: {},
+//   challenge: {
+//     bits: 20,
+//   },
+// });
+//
+//
 // /* Cloudflare R2 Plugin */
 // /* Enable if you choose to store workspace blobs or user avatars in Cloudflare R2 Storage Service */
 // AFFiNE.use('cloudflare-r2', {

packages/backend/server/src/core/auth/config.ts

@@ -40,6 +40,11 @@ export interface AuthRuntimeConfigurations {
    */
   allowSignup: boolean;
 
+  /**
+   * Whether require email domain record verification before access restricted resources
+   */
+  requireEmailDomainVerification: boolean;
+
   /**
    * Whether require email verification before access restricted resources
    */
@@ -76,6 +81,10 @@ defineRuntimeConfig('auth', {
     desc: 'Whether allow new registrations',
     default: true,
   },
+  requireEmailDomainVerification: {
+    desc: 'Whether require email domain record verification before accessing restricted resources',
+    default: false,
+  },
   requireEmailVerification: {
     desc: 'Whether require email verification before accessing restricted resources',
     default: true,

packages/backend/server/src/core/auth/controller.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from 'node:crypto';
+import { resolveMx, resolveTxt, setServers } from 'node:dns/promises';
 
 import {
   Body,
@@ -18,26 +18,34 @@ import {
   EarlyAccessRequired,
   EmailTokenNotFound,
   InternalServerError,
+  InvalidEmail,
   InvalidEmailToken,
   SignUpForbidden,
   Throttle,
   URLHelper,
+  UseNamedGuard,
 } from '../../fundamentals';
 import { UserService } from '../user';
 import { validators } from '../utils/validators';
-import { CurrentUser } from './current-user';
 import { Public } from './guard';
-import { AuthService, parseAuthUserSeqNum } from './service';
+import { AuthService } from './service';
+import { CurrentUser, Session } from './session';
 import { TokenService, TokenType } from './token';
 
-class SignInCredential {
-  email!: string;
-  password?: string;
+interface PreflightResponse {
+  registered: boolean;
+  hasPassword: boolean;
 }
 
-class MagicLinkCredential {
-  email!: string;
-  token!: string;
+interface SignInCredential {
+  email: string;
+  password?: string;
+  callbackUrl?: string;
+}
+
+interface MagicLinkCredential {
+  email: string;
+  token: string;
 }
 
 @Throttle('strict')
@@ -49,16 +57,56 @@ export class AuthController {
     private readonly user: UserService,
     private readonly token: TokenService,
     private readonly config: Config
-  ) {}
+  ) {
+    if (config.node.dev) {
+      // set DNS servers in dev mode
+      // NOTE: some network debugging software uses DNS hijacking
+      // to better debug traffic, but their DNS servers may not
+      // handle the non dns query(like txt, mx) correctly, so we
+      // set a public DNS server here to avoid this issue.
+      setServers(['1.1.1.1', '8.8.8.8']);
+    }
+  }
 
   @Public()
+  @Post('/preflight')
+  async preflight(
+    @Body() params?: { email: string }
+  ): Promise<PreflightResponse> {
+    if (!params?.email) {
+      throw new InvalidEmail();
+    }
+    validators.assertValidEmail(params.email);
+
+    const user = await this.user.findUserWithHashedPasswordByEmail(
+      params.email
+    );
+
+    if (!user) {
+      return {
+        registered: false,
+        hasPassword: false,
+      };
+    }
+
+    return {
+      registered: user.registered,
+      hasPassword: !!user.password,
+    };
+  }
+
+  @Public()
+  @UseNamedGuard('captcha')
   @Post('/sign-in')
   @Header('content-type', 'application/json')
   async signIn(
     @Req() req: Request,
     @Res() res: Response,
     @Body() credential: SignInCredential,
-    @Query('redirect_uri') redirectUri = this.url.home
+    /**
+     * @deprecated
+     */
+    @Query('redirect_uri') redirectUri?: string
   ) {
     validators.assertValidEmail(credential.email);
     const canSignIn = await this.auth.canSignIn(credential.email);
@@ -67,80 +115,117 @@ export class AuthController {
     }
 
     if (credential.password) {
-      const user = await this.auth.signIn(
+      await this.passwordSignIn(
+        req,
+        res,
         credential.email,
         credential.password
       );
-
-      await this.auth.setCookie(req, res, user);
-      res.status(HttpStatus.OK).send(user);
     } else {
-      // send email magic link
-      const user = await this.user.findUserByEmail(credential.email);
-      if (!user) {
-        const allowSignup = await this.config.runtime.fetch('auth/allowSignup');
-        if (!allowSignup) {
-          throw new SignUpForbidden();
-        }
-      }
-
-      const result = await this.sendSignInEmail(
-        { email: credential.email, signUp: !user },
+      await this.sendMagicLink(
+        req,
+        res,
+        credential.email,
+        credential.callbackUrl,
         redirectUri
       );
+    }
+  }
 
-      if (result.rejected.length) {
-        throw new InternalServerError('Failed to send sign-in email.');
+  async passwordSignIn(
+    req: Request,
+    res: Response,
+    email: string,
+    password: string
+  ) {
+    const user = await this.auth.signIn(email, password);
+
+    await this.auth.setCookies(req, res, user.id);
+    res.status(HttpStatus.OK).send(user);
+  }
+
+  async sendMagicLink(
+    _req: Request,
+    res: Response,
+    email: string,
+    callbackUrl = '/magic-link',
+    redirectUrl?: string
+  ) {
+    // send email magic link
+    const user = await this.user.findUserByEmail(email);
+    if (!user) {
+      const allowSignup = await this.config.runtime.fetch('auth/allowSignup');
+      if (!allowSignup) {
+        throw new SignUpForbidden();
       }
 
-      res.status(HttpStatus.OK).send({
-        email: credential.email,
-      });
+      const requireEmailDomainVerification = await this.config.runtime.fetch(
+        'auth/requireEmailDomainVerification'
+      );
+      if (requireEmailDomainVerification) {
+        // verify domain has MX, SPF, DMARC records
+        const [name, domain, ...rest] = email.split('@');
+        if (rest.length || !domain) {
+          throw new InvalidEmail();
+        }
+        const [mx, spf, dmarc] = await Promise.allSettled([
+          resolveMx(domain).then(t => t.map(mx => mx.exchange).filter(Boolean)),
+          resolveTxt(domain).then(t =>
+            t.map(([k]) => k).filter(txt => txt.includes('v=spf1'))
+          ),
+          resolveTxt('_dmarc.' + domain).then(t =>
+            t.map(([k]) => k).filter(txt => txt.includes('v=DMARC1'))
+          ),
+        ]).then(t => t.filter(t => t.status === 'fulfilled').map(t => t.value));
+        if (!mx?.length || !spf?.length || !dmarc?.length) {
+          throw new InvalidEmail();
+        }
+        // filter out alias emails
+        if (name.includes('+') || name.includes('.')) {
+          throw new InvalidEmail();
+        }
+      }
     }
-  }
 
-  async sendSignInEmail(
-    { email, signUp }: { email: string; signUp: boolean },
-    redirectUri: string
-  ) {
     const token = await this.token.createToken(TokenType.SignIn, email);
 
-    const magicLink = this.url.link('/magic-link', {
+    const magicLink = this.url.link(callbackUrl, {
       token,
       email,
-      redirect_uri: redirectUri,
+      ...(redirectUrl
+        ? {
+            redirect_uri: redirectUrl,
+          }
+        : {}),
     });
 
-    const result = await this.auth.sendSignInEmail(email, magicLink, signUp);
+    const result = await this.auth.sendSignInEmail(email, magicLink, !user);
 
-    return result;
+    if (result.rejected.length) {
+      throw new InternalServerError('Failed to send sign-in email.');
+    }
+
+    res.status(HttpStatus.OK).send({
+      email: email,
+    });
   }
 
+  @Public()
   @Get('/sign-out')
   async signOut(
-    @Req() req: Request,
     @Res() res: Response,
-    @Query('redirect_uri') redirectUri?: string
+    @Session() session: Session | undefined,
+    @Query('user_id') userId: string | undefined
   ) {
-    const session = await this.auth.signOut(
-      req.cookies[AuthService.sessionCookieName],
-      parseAuthUserSeqNum(req.headers[AuthService.authUserSeqHeaderName])
-    );
-
-    if (session) {
-      res.cookie(AuthService.sessionCookieName, session.id, {
-        expires: session.expiresAt ?? void 0, // expiredAt is `string | null`
-        ...this.auth.cookieOptions,
-      });
-    } else {
-      res.clearCookie(AuthService.sessionCookieName);
+    if (!session) {
+      res.status(HttpStatus.OK).send({});
+      return;
     }
 
-    if (redirectUri) {
-      return this.url.safeRedirect(res, redirectUri);
-    } else {
-      return res.send(null);
-    }
+    await this.auth.signOut(session.sessionId, userId);
+    await this.auth.refreshCookies(res, session.sessionId);
+
+    res.status(HttpStatus.OK).send({});
   }
 
   @Public()
@@ -156,11 +241,11 @@ export class AuthController {
 
     validators.assertValidEmail(email);
 
-    const valid = await this.token.verifyToken(TokenType.SignIn, token, {
+    const tokenRecord = await this.token.verifyToken(TokenType.SignIn, token, {
       credential: email,
     });
 
-    if (!valid) {
+    if (!tokenRecord) {
       throw new InvalidEmailToken();
     }
 
@@ -169,9 +254,8 @@ export class AuthController {
       registered: true,
     });
 
-    await this.auth.setCookie(req, res, user);
-
-    res.send({ id: user.id, email: user.email, name: user.name });
+    await this.auth.setCookies(req, res, user.id);
+    res.send({ id: user.id });
   }
 
   @Throttle('default', { limit: 1200 })
@@ -198,14 +282,4 @@ export class AuthController {
       users: await this.auth.getUserList(token),
     };
   }
-
-  @Public()
-  @Get('/challenge')
-  async challenge() {
-    // TODO(@darksky): impl in following PR
-    return {
-      challenge: randomUUID(),
-      resource: randomUUID(),
-    };
-  }
 }

packages/backend/server/src/core/auth/guard.ts

@@ -4,28 +4,20 @@ import type {
   FactoryProvider,
   OnModuleInit,
 } from '@nestjs/common';
-import { Injectable, SetMetadata, UseGuards } from '@nestjs/common';
+import { Injectable, SetMetadata } from '@nestjs/common';
 import { ModuleRef, Reflector } from '@nestjs/core';
-import type { Request } from 'express';
+import type { Request, Response } from 'express';
+import { Socket } from 'socket.io';
 
 import {
   AuthenticationRequired,
   Config,
   getRequestResponseFromContext,
-  mapAnyError,
   parseCookies,
 } from '../../fundamentals';
 import { WEBSOCKET_OPTIONS } from '../../fundamentals/websocket';
-import { CurrentUser, UserSession } from './current-user';
-import { AuthService, parseAuthUserSeqNum } from './service';
-
-function extractTokenFromHeader(authorization: string) {
-  if (!/^Bearer\s/i.test(authorization)) {
-    return;
-  }
-
-  return authorization.substring(7);
-}
+import { AuthService } from './service';
+import { Session } from './session';
 
 const PUBLIC_ENTRYPOINT_SYMBOL = Symbol('public');
 
@@ -45,9 +37,9 @@ export class AuthGuard implements CanActivate, OnModuleInit {
   async canActivate(context: ExecutionContext) {
     const { req, res } = getRequestResponseFromContext(context);
 
-    const userSession = await this.signIn(req);
-    if (res && userSession && userSession.session.expiresAt) {
-      await this.auth.refreshUserSessionIfNeeded(req, res, userSession.session);
+    const userSession = await this.signIn(req, res);
+    if (res && userSession && userSession.expiresAt) {
+      await this.auth.refreshUserSessionIfNeeded(res, userSession);
     }
 
     // api is public
@@ -60,43 +52,28 @@ export class AuthGuard implements CanActivate, OnModuleInit {
       return true;
     }
 
-    if (!req.user) {
+    if (!userSession) {
       throw new AuthenticationRequired();
     }
+
     return true;
   }
 
-  async signIn(
-    req: Request
-  ): Promise<{ user: CurrentUser; session: UserSession } | null> {
-    if (req.user && req.session) {
-      return {
-        user: req.user,
-        session: req.session,
+  async signIn(req: Request, res?: Response): Promise<Session | null> {
+    if (req.session) {
+      return req.session;
+    }
+
+    // TODO(@forehalo): a cache for user session
+    const userSession = await this.auth.getUserSessionFromRequest(req, res);
+
+    if (userSession) {
+      req.session = {
+        ...userSession.session,
+        user: userSession.user,
       };
-    }
 
-    parseCookies(req);
-    let sessionToken: string | undefined =
-      req.cookies[AuthService.sessionCookieName];
-
-    if (!sessionToken && req.headers.authorization) {
-      sessionToken = extractTokenFromHeader(req.headers.authorization);
-    }
-
-    if (sessionToken) {
-      const userSeq = parseAuthUserSeqNum(
-        req.headers[AuthService.authUserSeqHeaderName]
-      );
-
-      const userSession = await this.auth.getUserSession(sessionToken, userSeq);
-
-      if (userSession) {
-        req.session = userSession.session;
-        req.user = userSession.user;
-      }
-
-      return userSession;
+      return req.session;
     }
 
     return null;
@@ -104,26 +81,8 @@ export class AuthGuard implements CanActivate, OnModuleInit {
 }
 
 /**
- * This guard is used to protect routes/queries/mutations that require a user to be logged in.
- *
- * The `@CurrentUser()` parameter decorator used in a `Auth` guarded queries would always give us the user because the `Auth` guard will
- * fast throw if user is not logged in.
- *
- * @example
- *
- * ```typescript
- * \@Auth()
- * \@Query(() => UserType)
- * user(@CurrentUser() user: CurrentUser) {
- *   return user;
- * }
- * ```
+ * Mark api to be public accessible
  */
-export const Auth = () => {
-  return UseGuards(AuthGuard);
-};
-
-// api is public accessible
 export const Public = () => SetMetadata(PUBLIC_ENTRYPOINT_SYMBOL, true);
 
 export const AuthWebsocketOptionsProvider: FactoryProvider = {
@@ -131,27 +90,22 @@ export const AuthWebsocketOptionsProvider: FactoryProvider = {
   useFactory: (config: Config, guard: AuthGuard) => {
     return {
       ...config.websocket,
-      allowRequest: async (
-        req: any,
-        pass: (err: string | null | undefined, success: boolean) => void
-      ) => {
-        if (!config.websocket.requireAuthentication) {
-          return pass(null, true);
-        }
+      canActivate: async (socket: Socket) => {
+        const upgradeReq = socket.client.request as Request;
+        const handshake = socket.handshake;
 
-        try {
-          const authentication = await guard.signIn(req);
+        // compatibility with websocket request
+        parseCookies(upgradeReq);
 
-          if (authentication) {
-            return pass(null, true);
-          } else {
-            return pass('unauthenticated', false);
-          }
-        } catch (e) {
-          const error = mapAnyError(e);
-          error.log('Websocket');
-          return pass('unauthenticated', false);
-        }
+        upgradeReq.cookies = {
+          [AuthService.sessionCookieName]: handshake.auth.token,
+          [AuthService.userCookieName]: handshake.auth.userId,
+          ...upgradeReq.cookies,
+        };
+
+        const session = await guard.signIn(upgradeReq);
+
+        return !!session;
       },
     };
   },