From fe1eefdbb26584ac10e77973c78bd28812bd1e68 Mon Sep 17 00:00:00 2001 From: Brooooooklyn Date: Tue, 10 Sep 2024 04:03:58 +0000 Subject: [PATCH] feat: init renderer server (#8088) --- .github/actions/deploy/deploy.mjs | 11 +- .github/deployment/front/affine.nginx.conf | 10 +- .github/deployment/node/Dockerfile | 1 + .../charts/graphql/templates/deployment.yaml | 10 +- .../charts/graphql/templates/migration.yaml | 8 +- .../charts/graphql/templates/r2-secret.yaml | 11 - .../helm/affine/charts/graphql/values.yaml | 9 +- .../helm/affine/charts/renderer/Chart.yaml | 11 + .../charts/renderer/templates/NOTES.txt | 16 ++ .../charts/renderer/templates/_helpers.tpl | 63 ++++++ .../charts/renderer/templates/deployment.yaml | 124 +++++++++++ .../charts/renderer/templates/service.yaml | 19 ++ .../renderer/templates/serviceaccount.yaml | 12 ++ .../templates/tests/test-connection.yaml | 15 ++ .../helm/affine/charts/renderer/values.yaml | 38 ++++ .github/helm/affine/templates/configmap.yaml | 9 + .github/helm/affine/templates/ingress.yaml | 9 + .../graphql => }/templates/pg-secret.yaml | 0 .github/helm/affine/templates/r2-secret.yaml | 11 + .../graphql => }/templates/redis-secret.yaml | 0 .github/helm/affine/values.yaml | 16 ++ ...uild-server-image.yml => build-images.yml} | 176 +++++++++++++++- .github/workflows/build-selfhost-image.yml | 2 +- .github/workflows/build-test.yml | 12 +- .github/workflows/deploy.yml | 168 +-------------- Cargo.lock | 7 + Cargo.toml | 41 ++-- packages/backend/native/Cargo.toml | 17 +- packages/backend/native/index.d.ts | 2 + packages/backend/native/index.js | 1 + packages/backend/native/src/html_sanitize.rs | 4 + packages/backend/native/src/lib.rs | 1 + packages/backend/server/package.json | 2 +- .../src/core/doc-renderer/controller.ts | 192 +++++++++++++----- .../server/src/core/doc-renderer/service.ts | 12 +- .../server/src/core/doc/adapters/workspace.ts | 11 +- .../backend/server/src/core/selfhost/index.ts | 13 +- .../server/src/fundamentals/event/def.ts | 6 +- packages/backend/server/src/native.ts | 1 + .../backend/server/tests/app/selfhost.e2e.ts | 13 +- packages/common/env/src/global.ts | 4 +- .../frontend/component/.storybook/main.ts | 3 +- packages/frontend/graphql/src/schema.ts | 4 + packages/frontend/web/package.json | 2 +- scripts/setup/global.ts | 2 +- tools/cli/src/bin/build.ts | 2 +- tools/cli/src/bin/dev.ts | 6 +- tools/cli/src/config/cwd.cjs | 6 +- tools/cli/src/config/index.ts | 2 +- tools/cli/src/webpack/template.html | 1 + tools/cli/src/webpack/webpack.config.ts | 30 ++- yarn.lock | 11 +- 52 files changed, 827 insertions(+), 330 deletions(-) delete mode 100644 .github/helm/affine/charts/graphql/templates/r2-secret.yaml create mode 100644 .github/helm/affine/charts/renderer/Chart.yaml create mode 100644 .github/helm/affine/charts/renderer/templates/NOTES.txt create mode 100644 .github/helm/affine/charts/renderer/templates/_helpers.tpl create mode 100644 .github/helm/affine/charts/renderer/templates/deployment.yaml create mode 100644 .github/helm/affine/charts/renderer/templates/service.yaml create mode 100644 .github/helm/affine/charts/renderer/templates/serviceaccount.yaml create mode 100644 .github/helm/affine/charts/renderer/templates/tests/test-connection.yaml create mode 100644 .github/helm/affine/charts/renderer/values.yaml create mode 100644 .github/helm/affine/templates/configmap.yaml rename .github/helm/affine/{charts/graphql => }/templates/pg-secret.yaml (100%) create mode 100644 .github/helm/affine/templates/r2-secret.yaml rename .github/helm/affine/{charts/graphql => }/templates/redis-secret.yaml (100%) rename .github/workflows/{build-server-image.yml => build-images.yml} (55%) create mode 100644 packages/backend/native/src/html_sanitize.rs diff --git a/.github/actions/deploy/deploy.mjs b/.github/actions/deploy/deploy.mjs index 4afce51374..43e9bee706 100644 --- a/.github/actions/deploy/deploy.mjs +++ b/.github/actions/deploy/deploy.mjs @@ -90,9 +90,14 @@ const createHelmCommand = ({ isDryRun }) => { const deployCommand = [ `helm upgrade --install affine .github/helm/affine`, `--namespace ${namespace}`, + `--set-string global.app.buildType="${buildType}"`, `--set global.ingress.enabled=true`, `--set-json global.ingress.annotations=\"{ \\"kubernetes.io/ingress.class\\": \\"gce\\", \\"kubernetes.io/ingress.allow-http\\": \\"true\\", \\"kubernetes.io/ingress.global-static-ip-name\\": \\"${STATIC_IP_NAME}\\" }\"`, `--set-string global.ingress.host="${host}"`, + `--set global.objectStorage.r2.enabled=true`, + `--set-string global.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`, + `--set-string global.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`, + `--set-string global.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`, `--set-string global.version="${APP_VERSION}"`, ...redisAndPostgres, `--set web.replicaCount=${webReplicaCount}`, @@ -106,10 +111,6 @@ const createHelmCommand = ({ isDryRun }) => { `--set-string graphql.app.copilot.openai.key="${COPILOT_OPENAI_API_KEY}"`, `--set-string graphql.app.copilot.fal.key="${COPILOT_FAL_API_KEY}"`, `--set-string graphql.app.copilot.unsplash.key="${COPILOT_UNSPLASH_API_KEY}"`, - `--set graphql.app.objectStorage.r2.enabled=true`, - `--set-string graphql.app.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`, - `--set-string graphql.app.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`, - `--set-string graphql.app.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`, `--set-string graphql.app.mailer.sender="${MAILER_SENDER}"`, `--set-string graphql.app.mailer.user="${MAILER_USER}"`, `--set-string graphql.app.mailer.password="${MAILER_PASSWORD}"`, @@ -125,6 +126,8 @@ const createHelmCommand = ({ isDryRun }) => { `--set graphql.app.features.syncClientVersionCheck=true`, `--set sync.replicaCount=${syncReplicaCount}`, `--set-string sync.image.tag="${imageTag}"`, + `--set-string renderer.image.tag="${imageTag}"`, + `--set renderer.app.host=${host}`, ...serviceAnnotations, `--timeout 10m`, flag, diff --git a/.github/deployment/front/affine.nginx.conf b/.github/deployment/front/affine.nginx.conf index fffed2fe76..8e2a2d352c 100644 --- a/.github/deployment/front/affine.nginx.conf +++ b/.github/deployment/front/affine.nginx.conf @@ -6,11 +6,6 @@ server { try_files $uri/index.html $uri/ $uri /admin/index.html; } - location ~ ^/(_plugin|assets|imgs|js|plugins|static)/ { - root /app/dist/; - try_files $uri $uri/ =404; - } - set $app_root_path /app/dist/; set $mobile_root /app/dist/; set_by_lua $affine_env 'return os.getenv("AFFINE_ENV")'; @@ -28,6 +23,11 @@ server { set $app_root_path $mobile_root; } + location ~ ^/(_plugin|assets|imgs|js|plugins|static)/ { + root $app_root_path; + try_files $uri $uri/ =404; + } + location / { root $app_root_path; index index.html; diff --git a/.github/deployment/node/Dockerfile b/.github/deployment/node/Dockerfile index 6190734232..5dcba66169 100644 --- a/.github/deployment/node/Dockerfile +++ b/.github/deployment/node/Dockerfile @@ -3,6 +3,7 @@ FROM node:20-bookworm-slim COPY ./packages/backend/server /app COPY ./packages/frontend/web/dist /app/static COPY ./packages/frontend/admin/dist /app/static/admin +COPY ./packages/frontend/mobile/dist /app/static/mobile WORKDIR /app RUN apt-get update && \ diff --git a/.github/helm/affine/charts/graphql/templates/deployment.yaml b/.github/helm/affine/charts/graphql/templates/deployment.yaml index 2f74b5cde3..36a04270c5 100644 --- a/.github/helm/affine/charts/graphql/templates/deployment.yaml +++ b/.github/helm/affine/charts/graphql/templates/deployment.yaml @@ -76,7 +76,7 @@ spec: - name: AFFINE_SERVER_HTTPS value: "{{ .Values.app.https }}" - name: ENABLE_R2_OBJECT_STORAGE - value: "{{ .Values.app.objectStorage.r2.enabled }}" + value: "{{ .Values.global.objectStorage.r2.enabled }}" - name: FEATURES_EARLY_ACCESS_PREVIEW value: "{{ .Values.app.features.earlyAccessPreview }}" - name: FEATURES_SYNC_CLIENT_VERSION_CHECK @@ -122,21 +122,21 @@ spec: - name: DOC_MERGE_USE_JWST_CODEC value: "true" {{ end }} - {{ if .Values.app.objectStorage.r2.enabled }} + {{ if .Values.global.objectStorage.r2.enabled }} - name: R2_OBJECT_STORAGE_ACCOUNT_ID valueFrom: secretKeyRef: - name: "{{ .Values.app.objectStorage.r2.secretName }}" + name: "{{ .Values.global.objectStorage.r2.secretName }}" key: accountId - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: "{{ .Values.app.objectStorage.r2.secretName }}" + name: "{{ .Values.global.objectStorage.r2.secretName }}" key: accessKeyId - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: "{{ .Values.app.objectStorage.r2.secretName }}" + name: "{{ .Values.global.objectStorage.r2.secretName }}" key: secretAccessKey {{ end }} {{ if .Values.app.captcha.enabled }} diff --git a/.github/helm/affine/charts/graphql/templates/migration.yaml b/.github/helm/affine/charts/graphql/templates/migration.yaml index 714d4d86c5..32b88d0dd5 100644 --- a/.github/helm/affine/charts/graphql/templates/migration.yaml +++ b/.github/helm/affine/charts/graphql/templates/migration.yaml @@ -37,21 +37,21 @@ spec: - name: DATABASE_URL value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.gcloud.cloudSqlInternal }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }} {{ end }} - {{ if .Values.app.objectStorage.r2.enabled }} + {{ if .Values.global.objectStorage.r2.enabled }} - name: R2_OBJECT_STORAGE_ACCOUNT_ID valueFrom: secretKeyRef: - name: "{{ .Values.app.objectStorage.r2.secretName }}" + name: "{{ .Values.global.objectStorage.r2.secretName }}" key: accountId - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: "{{ .Values.app.objectStorage.r2.secretName }}" + name: "{{ .Values.global.objectStorage.r2.secretName }}" key: accessKeyId - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: "{{ .Values.app.objectStorage.r2.secretName }}" + name: "{{ .Values.global.objectStorage.r2.secretName }}" key: secretAccessKey {{ end }} resources: diff --git a/.github/helm/affine/charts/graphql/templates/r2-secret.yaml b/.github/helm/affine/charts/graphql/templates/r2-secret.yaml deleted file mode 100644 index a521c27131..0000000000 --- a/.github/helm/affine/charts/graphql/templates/r2-secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.app.objectStorage.r2.enabled -}} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ .Values.app.objectStorage.r2.secretName }}" -type: Opaque -data: - accountId: {{ .Values.app.objectStorage.r2.accountId | b64enc }} - accessKeyId: {{ .Values.app.objectStorage.r2.accessKeyId | b64enc }} - secretAccessKey: {{ .Values.app.objectStorage.r2.secretAccessKey | b64enc }} -{{- end }} diff --git a/.github/helm/affine/charts/graphql/values.yaml b/.github/helm/affine/charts/graphql/values.yaml index 61b8ec3ea8..9dd154d616 100644 --- a/.github/helm/affine/charts/graphql/values.yaml +++ b/.github/helm/affine/charts/graphql/values.yaml @@ -29,14 +29,7 @@ app: secretName: copilot openai: key: '' - objectStorage: - r2: - enabled: false - secretName: r2 - accountId: '' - accessKeyId: '' - secretAccessKey: '' - oauth: + oauth: google: enabled: false secretName: oauth-google diff --git a/.github/helm/affine/charts/renderer/Chart.yaml b/.github/helm/affine/charts/renderer/Chart.yaml new file mode 100644 index 0000000000..d448d65c50 --- /dev/null +++ b/.github/helm/affine/charts/renderer/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: renderer +description: AFFiNE renderer server +type: application +version: 0.0.0 +appVersion: "0.16.0" +dependencies: + - name: gcloud-sql-proxy + version: 0.0.0 + repository: "file://../gcloud-sql-proxy" + condition: .global.database.gcloud.enabled diff --git a/.github/helm/affine/charts/renderer/templates/NOTES.txt b/.github/helm/affine/charts/renderer/templates/NOTES.txt new file mode 100644 index 0000000000..805c45e12a --- /dev/null +++ b/.github/helm/affine/charts/renderer/templates/NOTES.txt @@ -0,0 +1,16 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "renderer.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "renderer.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "renderer.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "renderer.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/.github/helm/affine/charts/renderer/templates/_helpers.tpl b/.github/helm/affine/charts/renderer/templates/_helpers.tpl new file mode 100644 index 0000000000..6a77a56d13 --- /dev/null +++ b/.github/helm/affine/charts/renderer/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "renderer.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "renderer.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "renderer.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "renderer.labels" -}} +helm.sh/chart: {{ include "renderer.chart" . }} +{{ include "renderer.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +monitoring: enabled +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "renderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "renderer.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "renderer.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "renderer.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/renderer/templates/deployment.yaml b/.github/helm/affine/charts/renderer/templates/deployment.yaml new file mode 100644 index 0000000000..6a6edc380a --- /dev/null +++ b/.github/helm/affine/charts/renderer/templates/deployment.yaml @@ -0,0 +1,124 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "renderer.fullname" . }} + labels: + {{- include "renderer.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "renderer.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "renderer.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "renderer.serviceAccountName" . }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: AFFINE_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: "{{ .Values.global.secret.secretName }}" + key: key + - name: NODE_ENV + value: "{{ .Values.env }}" + - name: NODE_OPTIONS + value: "--max-old-space-size=4096" + - name: NO_COLOR + value: "1" + - name: DEPLOYMENT_TYPE + value: "affine" + - name: SERVER_FLAVOR + value: "renderer" + - name: AFFINE_ENV + value: "{{ .Release.Namespace }}" + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: pg-postgresql + key: postgres-password + - name: DATABASE_URL + value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.url }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }} + - name: REDIS_SERVER_ENABLED + value: "true" + - name: REDIS_SERVER_HOST + value: "{{ .Values.global.redis.host }}" + - name: REDIS_SERVER_PORT + value: "{{ .Values.global.redis.port }}" + - name: REDIS_SERVER_USER + value: "{{ .Values.global.redis.username }}" + - name: REDIS_SERVER_PASSWORD + valueFrom: + secretKeyRef: + name: redis + key: redis-password + - name: REDIS_SERVER_DATABASE + value: "{{ .Values.global.redis.database }}" + - name: AFFINE_SERVER_PORT + value: "{{ .Values.service.port }}" + - name: AFFINE_SERVER_SUB_PATH + value: "{{ .Values.app.path }}" + - name: AFFINE_SERVER_HOST + value: "{{ .Values.app.host }}" + - name: AFFINE_SERVER_HTTPS + value: "{{ .Values.app.https }}" + - name: ENABLE_R2_OBJECT_STORAGE + value: "{{ .Values.global.objectStorage.r2.enabled }}" + {{ if .Values.global.objectStorage.r2.enabled }} + - name: R2_OBJECT_STORAGE_ACCOUNT_ID + valueFrom: + secretKeyRef: + name: "{{ .Values.global.objectStorage.r2.secretName }}" + key: accountId + - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: "{{ .Values.global.objectStorage.r2.secretName }}" + key: accessKeyId + - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: "{{ .Values.global.objectStorage.r2.secretName }}" + key: secretAccessKey + {{ end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /info + port: http + initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} + readinessProbe: + httpGet: + path: /info + port: http + initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/.github/helm/affine/charts/renderer/templates/service.yaml b/.github/helm/affine/charts/renderer/templates/service.yaml new file mode 100644 index 0000000000..4c34622c41 --- /dev/null +++ b/.github/helm/affine/charts/renderer/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "graphql.fullname" . }} + labels: + {{- include "graphql.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "graphql.selectorLabels" . | nindent 4 }} diff --git a/.github/helm/affine/charts/renderer/templates/serviceaccount.yaml b/.github/helm/affine/charts/renderer/templates/serviceaccount.yaml new file mode 100644 index 0000000000..14dac586bf --- /dev/null +++ b/.github/helm/affine/charts/renderer/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "graphql.serviceAccountName" . }} + labels: + {{- include "graphql.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/renderer/templates/tests/test-connection.yaml b/.github/helm/affine/charts/renderer/templates/tests/test-connection.yaml new file mode 100644 index 0000000000..d3b5be0e91 --- /dev/null +++ b/.github/helm/affine/charts/renderer/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "renderer.fullname" . }}-test-connection" + labels: + {{- include "renderer.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "renderer.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/.github/helm/affine/charts/renderer/values.yaml b/.github/helm/affine/charts/renderer/values.yaml new file mode 100644 index 0000000000..50482f5759 --- /dev/null +++ b/.github/helm/affine/charts/renderer/values.yaml @@ -0,0 +1,38 @@ +replicaCount: 1 +image: + repository: ghcr.io/toeverything/affine-graphql + pullPolicy: IfNotPresent + tag: '' + +imagePullSecrets: [] +nameOverride: '' +fullnameOverride: '' +# map to NODE_ENV environment variable +env: 'production' +app: + # AFFINE_SERVER_SUB_PATH + path: '' + # AFFINE_SERVER_HOST + host: '0.0.0.0' + https: true +serviceAccount: + create: true + annotations: {} + name: 'affine-renderer' + +podAnnotations: {} + +podSecurityContext: + fsGroup: 2000 + +resources: + requests: + cpu: '4' + memory: 4Gi + +probe: + initialDelaySeconds: 20 + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/.github/helm/affine/templates/configmap.yaml b/.github/helm/affine/templates/configmap.yaml new file mode 100644 index 0000000000..ce0603f86d --- /dev/null +++ b/.github/helm/affine/templates/configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-runtime-config +data: + web-assets-manifest: |- + {{ .Files.Get "web-assets-manifest.json" | nindent 4 }} + mobile-assets-manifest: |- + {{ .Files.Get "mobile-assets-manifest.json" | nindent 4 }} diff --git a/.github/helm/affine/templates/ingress.yaml b/.github/helm/affine/templates/ingress.yaml index b4067c943a..5b35a8a703 100644 --- a/.github/helm/affine/templates/ingress.yaml +++ b/.github/helm/affine/templates/ingress.yaml @@ -60,6 +60,15 @@ spec: name: affine-graphql port: number: {{ .Values.graphql.service.port }} + {{- if eq .Values.global.app.buildType "canary" }} + - path: /workspace + pathType: Prefix + backend: + service: + name: affine-renderer + port: + number: {{ .Values.graphql.service.port }} + {{- end }} - path: / pathType: Prefix backend: diff --git a/.github/helm/affine/charts/graphql/templates/pg-secret.yaml b/.github/helm/affine/templates/pg-secret.yaml similarity index 100% rename from .github/helm/affine/charts/graphql/templates/pg-secret.yaml rename to .github/helm/affine/templates/pg-secret.yaml diff --git a/.github/helm/affine/templates/r2-secret.yaml b/.github/helm/affine/templates/r2-secret.yaml new file mode 100644 index 0000000000..d5c49fb2fb --- /dev/null +++ b/.github/helm/affine/templates/r2-secret.yaml @@ -0,0 +1,11 @@ +{{- if .Values.global.objectStorage.r2.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ .Values.global.objectStorage.r2.secretName }}" +type: Opaque +data: + accountId: {{ .Values.global.objectStorage.r2.accountId | b64enc }} + accessKeyId: {{ .Values.global.objectStorage.r2.accessKeyId | b64enc }} + secretAccessKey: {{ .Values.global.objectStorage.r2.secretAccessKey | b64enc }} +{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/redis-secret.yaml b/.github/helm/affine/templates/redis-secret.yaml similarity index 100% rename from .github/helm/affine/charts/graphql/templates/redis-secret.yaml rename to .github/helm/affine/templates/redis-secret.yaml diff --git a/.github/helm/affine/values.yaml b/.github/helm/affine/values.yaml index 5367047004..1f2f9e216d 100644 --- a/.github/helm/affine/values.yaml +++ b/.github/helm/affine/values.yaml @@ -1,4 +1,6 @@ global: + app: + buildType: 'stable' ingress: enabled: false className: '' @@ -28,6 +30,13 @@ global: username: '' password: '' database: 0 + objectStorage: + r2: + enabled: false + secretName: r2 + accountId: '' + accessKeyId: '' + secretAccessKey: '' gke: enabled: true @@ -45,6 +54,13 @@ sync: annotations: cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}' +renderer: + service: + type: ClusterIP + port: 3000 + annotations: + cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}' + web: service: type: ClusterIP diff --git a/.github/workflows/build-server-image.yml b/.github/workflows/build-images.yml similarity index 55% rename from .github/workflows/build-server-image.yml rename to .github/workflows/build-images.yml index 77eecb33d8..c416b5083b 100644 --- a/.github/workflows/build-server-image.yml +++ b/.github/workflows/build-images.yml @@ -6,11 +6,6 @@ on: flavor: type: string required: true - workflow_dispatch: - inputs: - flavor: - type: string - required: false env: NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} @@ -43,6 +38,103 @@ jobs: path: ./packages/backend/server/dist if-no-files-found: error + build-web: + name: Build @affine/web + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.flavor }} + steps: + - uses: actions/checkout@v4 + - name: Setup Version + id: version + uses: ./.github/actions/setup-version + - name: Setup Node.js + uses: ./.github/actions/setup-node + - name: Build Core + run: yarn nx build @affine/web --skip-nx-cache + env: + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + BUILD_TYPE: ${{ github.event.inputs.flavor }} + CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }} + SENTRY_ORG: ${{ secrets.SENTRY_ORG }} + SENTRY_PROJECT: 'affine-web' + SENTRY_RELEASE: ${{ steps.version.outputs.APP_VERSION }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }} + MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }} + - name: Upload web artifact + uses: actions/upload-artifact@v4 + with: + name: web + path: ./packages/frontend/web/dist + if-no-files-found: error + + build-admin: + name: Build @affine/admin + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.flavor }} + steps: + - uses: actions/checkout@v4 + - name: Setup Version + id: version + uses: ./.github/actions/setup-version + - name: Setup Node.js + uses: ./.github/actions/setup-node + - name: Build Admin + run: yarn nx build @affine/admin --skip-nx-cache + env: + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + BUILD_TYPE: ${{ github.event.inputs.flavor }} + CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }} + SENTRY_ORG: ${{ secrets.SENTRY_ORG }} + SENTRY_PROJECT: 'affine-admin' + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }} + MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }} + - name: Upload admin artifact + uses: actions/upload-artifact@v4 + with: + name: admin + path: ./packages/frontend/admin/dist + if-no-files-found: error + + build-mobile: + name: Build @affine/mobile + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.flavor }} + steps: + - uses: actions/checkout@v4 + - name: Setup Version + id: version + uses: ./.github/actions/setup-version + - name: Setup Node.js + uses: ./.github/actions/setup-node + - name: Build Mobile + run: yarn nx build @affine/mobile --skip-nx-cache + env: + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + BUILD_TYPE: ${{ github.event.inputs.flavor }} + CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }} + SENTRY_ORG: ${{ secrets.SENTRY_ORG }} + SENTRY_PROJECT: 'affine-mobile' + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }} + MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }} + - name: Upload mobile artifact + uses: actions/upload-artifact@v4 + with: + name: mobile + path: ./packages/frontend/mobile/dist + if-no-files-found: error + build-web-selfhost: name: Build @affine/web selfhost runs-on: ubuntu-latest @@ -70,6 +162,31 @@ jobs: path: ./packages/frontend/web/dist if-no-files-found: error + build-mobile-selfhost: + name: Build @affine/mobile selfhost + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.flavor }} + steps: + - uses: actions/checkout@v4 + - name: Setup Version + id: version + uses: ./.github/actions/setup-version + - name: Setup Node.js + uses: ./.github/actions/setup-node + - name: Build Mobile + run: yarn nx build @affine/mobile --skip-nx-cache + env: + BUILD_TYPE: ${{ github.event.inputs.flavor }} + PUBLIC_PATH: '/' + SELF_HOSTED: true + MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }} + - name: Upload mobile artifact + uses: actions/upload-artifact@v4 + with: + name: selfhost-mobile + path: ./packages/frontend/mobile/dist + if-no-files-found: error + build-admin-selfhost: name: Build @affine/admin selfhost runs-on: ubuntu-latest @@ -81,7 +198,7 @@ jobs: uses: ./.github/actions/setup-version - name: Setup Node.js uses: ./.github/actions/setup-node - - name: Build Core + - name: Build admin run: yarn nx build @affine/admin --skip-nx-cache env: BUILD_TYPE: ${{ github.event.inputs.flavor }} @@ -131,12 +248,16 @@ jobs: path: ./packages/backend/native/server-native.node if-no-files-found: error - build-docker: - name: Build Docker + build-images: + name: Build Images runs-on: ubuntu-latest needs: - build-server + - build-web + - build-mobile + - build-admin - build-web-selfhost + - build-mobile-selfhost - build-admin-selfhost - build-server-native steps: @@ -195,17 +316,41 @@ jobs: registry-url: https://npm.pkg.github.com scope: '@toeverything' + - name: Download web artifact + uses: actions/download-artifact@v4 + with: + name: web + path: ./packages/frontend/web/dist + + - name: Download mobile artifact + uses: actions/download-artifact@v4 + with: + name: mobile + path: ./packages/frontend/mobile/dist + + - name: Download admin artifact + uses: actions/download-artifact@v4 + with: + name: admin + path: ./packages/frontend/admin/dist + - name: Download selfhost web artifact uses: actions/download-artifact@v4 with: name: selfhost-web - path: ./packages/frontend/web/dist + path: ./packages/frontend/web/dist/selfhost + + - name: Download selfhost mobile artifact + uses: actions/download-artifact@v4 + with: + name: selfhost-mobile + path: ./packages/frontend/mobile/dist/selfhost - name: Download selfhost admin artifact uses: actions/download-artifact@v4 with: name: selfhost-admin - path: ./packages/frontend/admin/dist + path: ./packages/frontend/admin/dist/selfhost - name: Install Node.js dependencies run: | @@ -220,6 +365,17 @@ jobs: id: version uses: ./.github/actions/setup-version + - name: Build front Dockerfile + uses: docker/build-push-action@v6 + with: + context: . + push: true + pull: true + platforms: linux/amd64,linux/arm64 + provenance: true + file: .github/deployment/front/Dockerfile + tags: ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}} + - name: Build graphql Dockerfile uses: docker/build-push-action@v6 with: diff --git a/.github/workflows/build-selfhost-image.yml b/.github/workflows/build-selfhost-image.yml index b8fc83446d..c11ed41270 100644 --- a/.github/workflows/build-selfhost-image.yml +++ b/.github/workflows/build-selfhost-image.yml @@ -20,6 +20,6 @@ permissions: jobs: build-image: name: Build Image - uses: ./.github/workflows/build-server-image.yml + uses: ./.github/workflows/build-images.yml with: flavor: ${{ github.event.inputs.flavor }} diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 777a3c81d2..1e135b7e6c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -117,7 +117,7 @@ jobs: name: E2E Test runs-on: ubuntu-latest env: - DISTRIBUTION: browser + DISTRIBUTION: web IN_CI_TEST: true strategy: fail-fast: false @@ -177,7 +177,7 @@ jobs: name: E2E Migration Test runs-on: ubuntu-latest env: - DISTRIBUTION: browser + DISTRIBUTION: web steps: - uses: actions/checkout@v4 - name: Setup Node.js @@ -204,7 +204,7 @@ jobs: needs: - build-native env: - DISTRIBUTION: browser + DISTRIBUTION: web steps: - uses: actions/checkout@v4 - name: Setup Node.js @@ -311,7 +311,7 @@ jobs: # always skip cache because its fast, and cache configuration is always changing run: yarn nx build @affine/web --skip-nx-cache env: - DISTRIBUTION: 'desktop' + DISTRIBUTION: desktop - name: zip web run: tar -czf dist.tar.gz --directory=packages/frontend/electron/renderer/dist . - name: Upload web artifact @@ -327,7 +327,7 @@ jobs: needs: build-server-native env: NODE_ENV: test - DISTRIBUTION: browser + DISTRIBUTION: web services: postgres: image: postgres @@ -396,7 +396,7 @@ jobs: name: ${{ matrix.tests.name }} runs-on: ubuntu-latest env: - DISTRIBUTION: browser + DISTRIBUTION: web DATABASE_URL: postgresql://affine:affine@localhost:5432/affine IN_CI_TEST: true strategy: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e00a266489..374ecb0e63 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -62,171 +62,19 @@ jobs: echo "version=$prev_version" >> $GITHUB_OUTPUT echo "namesapce=$namespace" >> $GITHUB_OUTPUT - build-server-image: - name: Build Server Image - uses: ./.github/workflows/build-server-image.yml + build-images: + name: Build Images + uses: ./.github/workflows/build-images.yml + secrets: inherit with: flavor: ${{ github.event.inputs.flavor }} - build-web: - name: Build @affine/web - runs-on: ubuntu-latest - environment: ${{ github.event.inputs.flavor }} - steps: - - uses: actions/checkout@v4 - - name: Setup Version - id: version - uses: ./.github/actions/setup-version - - name: Setup Node.js - uses: ./.github/actions/setup-node - - name: Build Core - run: yarn nx build @affine/web --skip-nx-cache - env: - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - BUILD_TYPE: ${{ github.event.inputs.flavor }} - CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }} - SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine-web' - SENTRY_RELEASE: ${{ steps.version.outputs.APP_VERSION }} - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }} - MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }} - - name: Upload web artifact - uses: actions/upload-artifact@v4 - with: - name: web - path: ./packages/frontend/web/dist - if-no-files-found: error - - build-admin: - name: Build @affine/admin - runs-on: ubuntu-latest - environment: ${{ github.event.inputs.flavor }} - steps: - - uses: actions/checkout@v4 - - name: Setup Version - id: version - uses: ./.github/actions/setup-version - - name: Setup Node.js - uses: ./.github/actions/setup-node - - name: Build Admin - run: yarn nx build @affine/admin --skip-nx-cache - env: - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - BUILD_TYPE: ${{ github.event.inputs.flavor }} - CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }} - SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine-admin' - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }} - MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }} - - name: Upload admin artifact - uses: actions/upload-artifact@v4 - with: - name: admin - path: ./packages/frontend/admin/dist - if-no-files-found: error - - build-mobile: - name: Build @affine/mobile - runs-on: ubuntu-latest - environment: ${{ github.event.inputs.flavor }} - steps: - - uses: actions/checkout@v4 - - name: Setup Version - id: version - uses: ./.github/actions/setup-version - - name: Setup Node.js - uses: ./.github/actions/setup-node - - name: Build Mobile - run: yarn nx build @affine/mobile --skip-nx-cache - env: - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - BUILD_TYPE: ${{ github.event.inputs.flavor }} - CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }} - SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine-mobile' - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }} - MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }} - - name: Upload mobile artifact - uses: actions/upload-artifact@v4 - with: - name: mobile - path: ./packages/frontend/mobile/dist - if-no-files-found: error - - build-frontend-image: - name: Build Frontend Image - runs-on: ubuntu-latest - needs: - - build-web - - build-admin - - build-mobile - steps: - - uses: actions/checkout@v4 - - name: Download web artifact - uses: actions/download-artifact@v4 - with: - name: web - path: ./packages/frontend/web/dist - - name: Download admin artifact - uses: actions/download-artifact@v4 - with: - name: admin - path: ./packages/frontend/admin/dist - - name: Download mobile artifact - uses: actions/download-artifact@v4 - with: - name: mobile - path: ./packages/frontend/mobile/dist - - name: Setup env - run: | - echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" - if [ -z "${{ inputs.flavor }}" ] - then - echo "RELEASE_FLAVOR=canary" >> "$GITHUB_ENV" - else - echo "RELEASE_FLAVOR=${{ inputs.flavor }}" >> "$GITHUB_ENV" - fi - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - logout: false - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Build front Dockerfile - uses: docker/build-push-action@v6 - with: - context: . - push: true - pull: true - platforms: linux/amd64,linux/arm64 - provenance: true - file: .github/deployment/front/Dockerfile - tags: ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}} - deploy: name: Deploy to cluster if: ${{ github.event_name == 'workflow_dispatch' }} environment: ${{ github.event.inputs.flavor }} needs: - - build-frontend-image - - build-server-image + - build-images runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -276,11 +124,7 @@ jobs: deploy-done: needs: - output-prev-version - - build-web - - build-admin - - build-mobile - - build-frontend-image - - build-server-image + - build-images - deploy if: always() runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 17e0c91985..ee9de1d473 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,6 +58,7 @@ dependencies = [ "sha3", "tiktoken-rs", "tokio", + "v_htmlescape", "y-octo", ] @@ -2218,6 +2219,12 @@ dependencies = [ "serde", ] +[[package]] +name = "v_htmlescape" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c" + [[package]] name = "valuable" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5f29f879c6..d8e9323521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,26 +3,27 @@ members = ["./packages/backend/native", "./packages/frontend/native", "./packag resolver = "2" [workspace.dependencies] -anyhow = "1" -chrono = "0.4" -dotenv = "0.15" -file-format = { version = "0.25", features = ["reader"] } -mimalloc = "0.1" -napi = { version = "3.0.0-alpha.1", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] } -napi-build = { version = "2" } -napi-derive = { version = "3.0.0-alpha.1" } -notify = { version = "6", features = ["serde"] } -once_cell = "1" -parking_lot = "0.12" -rand = "0.8" -serde = "1" -serde_json = "1" -sha3 = "0.10" -sqlx = { version = "0.8", default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] } -tiktoken-rs = "0.5" -tokio = "1.37" -uuid = "1.8" -y-octo = { git = "https://github.com/y-crdt/y-octo.git", branch = "main" } +anyhow = "1" +chrono = "0.4" +dotenv = "0.15" +file-format = { version = "0.25", features = ["reader"] } +mimalloc = "0.1" +napi = { version = "3.0.0-alpha.1", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] } +napi-build = { version = "2" } +napi-derive = { version = "3.0.0-alpha.1" } +notify = { version = "6", features = ["serde"] } +once_cell = "1" +parking_lot = "0.12" +rand = "0.8" +serde = "1" +serde_json = "1" +sha3 = "0.10" +sqlx = { version = "0.8", default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] } +tiktoken-rs = "0.5" +tokio = "1.37" +uuid = "1.8" +v_htmlescape = "0.15" +y-octo = { git = "https://github.com/y-crdt/y-octo.git", branch = "main" } [profile.dev.package.sqlx-macros] opt-level = 3 diff --git a/packages/backend/native/Cargo.toml b/packages/backend/native/Cargo.toml index 948b2e66f1..f63096d47b 100644 --- a/packages/backend/native/Cargo.toml +++ b/packages/backend/native/Cargo.toml @@ -7,14 +7,15 @@ version = "1.0.0" crate-type = ["cdylib"] [dependencies] -chrono = { workspace = true } -file-format = { workspace = true } -napi = { workspace = true } -napi-derive = { workspace = true } -rand = { workspace = true } -sha3 = { workspace = true } -tiktoken-rs = { workspace = true } -y-octo = { workspace = true } +chrono = { workspace = true } +file-format = { workspace = true } +napi = { workspace = true } +napi-derive = { workspace = true } +rand = { workspace = true } +sha3 = { workspace = true } +tiktoken-rs = { workspace = true } +v_htmlescape = { workspace = true } +y-octo = { workspace = true } [target.'cfg(not(target_os = "linux"))'.dependencies] mimalloc = { workspace = true } diff --git a/packages/backend/native/index.d.ts b/packages/backend/native/index.d.ts index 747d410c04..de691213ec 100644 --- a/packages/backend/native/index.d.ts +++ b/packages/backend/native/index.d.ts @@ -8,6 +8,8 @@ export declare function fromModelName(modelName: string): Tokenizer | null export declare function getMime(input: Uint8Array): string +export declare function htmlSanitize(input: string): string + /** * Merge updates in form like `Y.applyUpdate(doc, update)` way and return the * result binary. diff --git a/packages/backend/native/index.js b/packages/backend/native/index.js index 3f7991b018..5c4b150f0e 100644 --- a/packages/backend/native/index.js +++ b/packages/backend/native/index.js @@ -11,3 +11,4 @@ export const mintChallengeResponse = binding.mintChallengeResponse; export const getMime = binding.getMime; export const Tokenizer = binding.Tokenizer; export const fromModelName = binding.fromModelName; +export const htmlSanitize = binding.htmlSanitize; diff --git a/packages/backend/native/src/html_sanitize.rs b/packages/backend/native/src/html_sanitize.rs new file mode 100644 index 0000000000..4dced7aec6 --- /dev/null +++ b/packages/backend/native/src/html_sanitize.rs @@ -0,0 +1,4 @@ +#[napi] +pub fn html_sanitize(input: String) -> String { + v_htmlescape::escape(&input).to_string() +} diff --git a/packages/backend/native/src/lib.rs b/packages/backend/native/src/lib.rs index 0c3ac6ddd7..8e18135c50 100644 --- a/packages/backend/native/src/lib.rs +++ b/packages/backend/native/src/lib.rs @@ -2,6 +2,7 @@ pub mod file_type; pub mod hashcash; +pub mod html_sanitize; pub mod tiktoken; use std::fmt::{Debug, Display}; diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index 7ca0190421..578b4de62f 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -70,6 +70,7 @@ "graphql-upload": "^16.0.2", "html-validate": "^8.20.1", "ioredis": "^5.3.2", + "is-mobile": "^4.0.0", "keyv": "^5.0.0", "lodash-es": "^4.17.21", "mixpanel": "^0.18.0", @@ -94,7 +95,6 @@ "ts-node": "^10.9.2", "typescript": "^5.4.5", "ws": "^8.16.0", - "xss": "^1.0.15", "yjs": "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch", "zod": "^3.22.4" }, diff --git a/packages/backend/server/src/core/doc-renderer/controller.ts b/packages/backend/server/src/core/doc-renderer/controller.ts index adbbcccd94..9d69ae14ba 100644 --- a/packages/backend/server/src/core/doc-renderer/controller.ts +++ b/packages/backend/server/src/core/doc-renderer/controller.ts @@ -1,93 +1,191 @@ -import { Controller, Get, Param, Res } from '@nestjs/common'; -import type { Response } from 'express'; -import xss from 'xss'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; -import { DocNotFound } from '../../fundamentals'; +import { Controller, Get, Logger, Param, Req, Res } from '@nestjs/common'; +import type { Request, Response } from 'express'; +import isMobile from 'is-mobile'; + +import { Config, metrics, URLHelper } from '../../fundamentals'; +import { htmlSanitize } from '../../native'; +import { Public } from '../auth'; import { PermissionService } from '../permission'; -import { PageDocContent } from '../utils/blocksuite'; import { DocContentService } from './service'; interface RenderOptions { - og: boolean; - content: boolean; + title: string; + summary: string; + avatar?: string; } +interface HtmlAssets { + css: string[]; + js: string[]; + publicPath: string; + gitHash: string; + description: string; +} + +const defaultAssets: HtmlAssets = { + css: [], + js: [], + publicPath: '/', + gitHash: '', + description: '', +}; + @Controller('/workspace/:workspaceId/:docId') export class DocRendererController { + private readonly logger = new Logger(DocRendererController.name); + private readonly webAssets: HtmlAssets = defaultAssets; + private readonly mobileAssets: HtmlAssets = defaultAssets; + constructor( private readonly doc: DocContentService, - private readonly permission: PermissionService - ) {} + private readonly permission: PermissionService, + private readonly config: Config, + private readonly url: URLHelper + ) { + try { + const webConfigMapsPath = join( + this.config.projectRoot, + this.config.isSelfhosted ? 'static/selfhost' : 'static', + 'assets-manifest.json' + ); + const mobileConfigMapsPath = join( + this.config.projectRoot, + this.config.isSelfhosted ? 'static/mobile/selfhost' : 'static/mobile', + 'assets-manifest.json' + ); + this.webAssets = JSON.parse(readFileSync(webConfigMapsPath, 'utf-8')); + this.mobileAssets = JSON.parse( + readFileSync(mobileConfigMapsPath, 'utf-8') + ); + } catch (e) { + if (this.config.node.prod) { + throw e; + } + } + } + @Public() @Get() async render( + @Req() req: Request, @Res() res: Response, @Param('workspaceId') workspaceId: string, @Param('docId') docId: string ) { - if (workspaceId === docId) { - throw new DocNotFound({ spaceId: workspaceId, docId }); - } + const assets: HtmlAssets = + this.config.affine.canary && + isMobile({ + ua: req.headers['user-agent'] ?? undefined, + }) + ? this.mobileAssets + : this.webAssets; - // if page is public, show all - // if page is private, but workspace public og is on, show og but not content - const opts: RenderOptions = { - og: false, - content: false, - }; - const isPagePublic = await this.permission.isPublicPage(workspaceId, docId); - - if (isPagePublic) { - opts.og = true; - opts.content = true; - } else { - const allowPreview = await this.permission.allowUrlPreview(workspaceId); - - if (allowPreview) { - opts.og = true; - } - } - - let docContent = opts.og - ? await this.doc.getPageContent(workspaceId, docId) - : null; - if (!docContent) { - docContent = { title: 'untitled', summary: '' }; + let opts: RenderOptions | null = null; + try { + opts = + workspaceId === docId + ? await this.renderWorkspace(workspaceId) + : await this.getPageContent(workspaceId, docId); + metrics.doc.counter('render').add(1); + } catch (e) { + this.logger.error('failed to render page', e); } res.setHeader('Content-Type', 'text/html'); - if (!opts.og) { + if (!opts) { res.setHeader('X-Robots-Tag', 'noindex'); } - res.send(this._render(docContent, opts)); + + res.send(this._render(opts, assets)); } - _render(doc: PageDocContent, { og }: RenderOptions): string { - const title = xss(doc.title); - const summary = xss(doc.summary); + private async getPageContent( + workspaceId: string, + docId: string + ): Promise { + let allowUrlPreview = await this.permission.isPublicPage( + workspaceId, + docId + ); + + if (!allowUrlPreview) { + // if page is private, but workspace url preview is on + allowUrlPreview = await this.permission.allowUrlPreview(workspaceId); + } + + if (allowUrlPreview) { + return this.doc.getPageContent(workspaceId, docId); + } + + return null; + } + + private async renderWorkspace( + workspaceId: string + ): Promise { + const allowUrlPreview = await this.permission.allowUrlPreview(workspaceId); + + if (allowUrlPreview) { + const workspaceContent = await this.doc.getWorkspaceContent(workspaceId); + + if (workspaceContent) { + return { + title: workspaceContent.name, + summary: '', + avatar: workspaceContent.avatarKey + ? this.url.link( + `/api/workspaces/${workspaceId}/blobs/${workspaceContent.avatarKey}` + ) + : undefined, + }; + } + } + + return null; + } + + _render(opts: RenderOptions | null, assets: HtmlAssets): string { + const title = opts?.title + ? htmlSanitize(`${opts.title} | AFFiNE`) + : 'AFFiNE'; + const summary = opts ? htmlSanitize(opts.summary) : assets.description; + const image = opts?.avatar ?? 'https://affine.pro/og.jpeg'; return ` - ${title} | AFFiNE + + + ${title} + - ${!og ? '' : ''} + + ${!opts ? '' : ''} - + - + - + + ${assets.css.map(url => ``).join('\n')} +
+ ${assets.js.map(url => ``).join('\n')} `; diff --git a/packages/backend/server/src/core/doc-renderer/service.ts b/packages/backend/server/src/core/doc-renderer/service.ts index d07180e63c..30946ca616 100644 --- a/packages/backend/server/src/core/doc-renderer/service.ts +++ b/packages/backend/server/src/core/doc-renderer/service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { applyUpdate, Doc } from 'yjs'; -import { Cache } from '../../fundamentals'; +import { Cache, type EventPayload, OnEvent } from '../../fundamentals'; import { PgWorkspaceDocStorageAdapter } from '../doc'; import { type PageDocContent, @@ -78,11 +78,15 @@ export class DocContentService { return content; } - async markDocContentCacheStale(workspaceId: string, guid: string) { + @OnEvent('snapshot.updated') + async markDocContentCacheStale({ + workspaceId, + id, + }: EventPayload<'snapshot.updated'>) { const key = - workspaceId === guid + workspaceId === id ? `workspace:${workspaceId}:content` - : `workspace:${workspaceId}:doc:${guid}:content`; + : `workspace:${workspaceId}:doc:${id}:content`; await this.cache.delete(key); } } diff --git a/packages/backend/server/src/core/doc/adapters/workspace.ts b/packages/backend/server/src/core/doc/adapters/workspace.ts index bd2e98127b..1f0c8a2333 100644 --- a/packages/backend/server/src/core/doc/adapters/workspace.ts +++ b/packages/backend/server/src/core/doc/adapters/workspace.ts @@ -6,6 +6,7 @@ import { Cache, DocHistoryNotFound, DocNotFound, + EventEmitter, FailedToSaveUpdates, FailedToUpsertSnapshot, metrics, @@ -30,6 +31,7 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter { private readonly db: PrismaClient, private readonly mutex: Mutex, private readonly cache: Cache, + private readonly event: EventEmitter, protected override readonly options: DocStorageOptions ) { super(options); @@ -97,7 +99,6 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter { metrics.doc.counter('doc_update_insert_failed').add(1); throw new FailedToSaveUpdates(); } - return timestamp; } @@ -463,6 +464,14 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter { // the updates has been applied to current `doc` must have been seen by the other process as well. // The `updatedSnapshot` will be `undefined` in this case. const updatedSnapshot = result.at(0); + + if (updatedSnapshot) { + this.event.emit('snapshot.updated', { + workspaceId: snapshot.spaceId, + id: snapshot.docId, + }); + } + return !!updatedSnapshot; } catch (e) { metrics.doc.counter('snapshot_upsert_failed').add(1); diff --git a/packages/backend/server/src/core/selfhost/index.ts b/packages/backend/server/src/core/selfhost/index.ts index 0c559767e8..cd0b1a072c 100644 --- a/packages/backend/server/src/core/selfhost/index.ts +++ b/packages/backend/server/src/core/selfhost/index.ts @@ -22,7 +22,6 @@ export class SetupMiddleware implements NestMiddleware { use = (req: Request, res: Response, next: (error?: Error | any) => void) => { // never throw - // eslint-disable-next-line @typescript-eslint/no-floating-promises this.server .initialized() .then(initialized => { @@ -59,6 +58,10 @@ export class SelfhostModule implements OnModuleInit { ) {} onModuleInit() { + // selfhost static file location + // web => 'static/selfhost' + // admin => 'static/admin/selfhost' + // mobile => 'static/mobile/selfhost' const staticPath = join(this.config.projectRoot, 'static'); // in command line mode if (!this.adapterHost.httpAdapter) { @@ -73,7 +76,7 @@ export class SelfhostModule implements OnModuleInit { }); app.use( basePath + '/admin', - serveStatic(join(staticPath, 'admin'), { + serveStatic(join(staticPath, 'admin', 'selfhost'), { redirect: false, index: false, }) @@ -83,7 +86,7 @@ export class SelfhostModule implements OnModuleInit { [basePath + '/admin', basePath + '/admin/*'], this.check.use, (_req, res) => { - res.sendFile(join(staticPath, 'admin', 'index.html')); + res.sendFile(join(staticPath, 'admin', 'selfhost', 'index.html')); } ); @@ -92,13 +95,13 @@ export class SelfhostModule implements OnModuleInit { }); app.use( basePath, - serveStatic(staticPath, { + serveStatic(join(staticPath, 'selfhost'), { redirect: false, index: false, }) ); app.get('*', this.check.use, (_req, res) => { - res.sendFile(join(staticPath, 'index.html')); + res.sendFile(join(staticPath, 'selfhost', 'index.html')); }); } } diff --git a/packages/backend/server/src/fundamentals/event/def.ts b/packages/backend/server/src/fundamentals/event/def.ts index 3e4641e97c..ff1081463f 100644 --- a/packages/backend/server/src/fundamentals/event/def.ts +++ b/packages/backend/server/src/fundamentals/event/def.ts @@ -13,12 +13,8 @@ export interface WorkspaceEvents { } export interface DocEvents { - updated: Payload< - Pick & { - previous: Pick; - } - >; deleted: Payload>; + updated: Payload>; } export interface UserEvents { diff --git a/packages/backend/server/src/native.ts b/packages/backend/server/src/native.ts index c512214052..968eefdf28 100644 --- a/packages/backend/server/src/native.ts +++ b/packages/backend/server/src/native.ts @@ -32,3 +32,4 @@ export const mintChallengeResponse = async (resource: string, bits: number) => { export const getMime = serverNativeModule.getMime; export const Tokenizer = serverNativeModule.Tokenizer; export const fromModelName = serverNativeModule.fromModelName; +export const htmlSanitize = serverNativeModule.htmlSanitize; diff --git a/packages/backend/server/tests/app/selfhost.e2e.ts b/packages/backend/server/tests/app/selfhost.e2e.ts index 935066d8d1..d51f839fc9 100644 --- a/packages/backend/server/tests/app/selfhost.e2e.ts +++ b/packages/backend/server/tests/app/selfhost.e2e.ts @@ -18,16 +18,17 @@ const test = ava as TestFn<{ }>; function initTestStaticFiles(staticPath: string) { - mkdirSync(path.join(staticPath, 'admin'), { recursive: true }); const files = { - 'index.html': `AFFiNE