Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-01 17:50:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: init renderer server (#8088)

Browse Source
This commit is contained in:
Brooooooklyn
2024-09-10 04:03:58 +00:00
parent 0add8917f9
commit fe1eefdbb2
52 changed files with 827 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/deploy/deploy.mjs
+7 -4
View File
@@ -90,9 +90,14 @@ const createHelmCommand = ({ isDryRun }) => {
const deployCommand = [
`helm upgrade --install affine .github/helm/affine`,
`--namespace ${namespace}`,
`--set-string global.app.buildType="${buildType}"`,
`--set global.ingress.enabled=true`,
`--set-json global.ingress.annotations=\"{ \\"kubernetes.io/ingress.class\\": \\"gce\\", \\"kubernetes.io/ingress.allow-http\\": \\"true\\", \\"kubernetes.io/ingress.global-static-ip-name\\": \\"${STATIC_IP_NAME}\\" }\"`,
`--set-string global.ingress.host="${host}"`,
`--set global.objectStorage.r2.enabled=true`,
`--set-string global.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`,
`--set-string global.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`,
`--set-string global.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`,
`--set-string global.version="${APP_VERSION}"`,
...redisAndPostgres,
`--set web.replicaCount=${webReplicaCount}`,
@@ -106,10 +111,6 @@ const createHelmCommand = ({ isDryRun }) => {
`--set-string graphql.app.copilot.openai.key="${COPILOT_OPENAI_API_KEY}"`,
`--set-string graphql.app.copilot.fal.key="${COPILOT_FAL_API_KEY}"`,
`--set-string graphql.app.copilot.unsplash.key="${COPILOT_UNSPLASH_API_KEY}"`,
`--set graphql.app.objectStorage.r2.enabled=true`,
`--set-string graphql.app.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`,
`--set-string graphql.app.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`,
`--set-string graphql.app.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`,
`--set-string graphql.app.mailer.sender="${MAILER_SENDER}"`,
`--set-string graphql.app.mailer.user="${MAILER_USER}"`,
`--set-string graphql.app.mailer.password="${MAILER_PASSWORD}"`,
@@ -125,6 +126,8 @@ const createHelmCommand = ({ isDryRun }) => {
`--set graphql.app.features.syncClientVersionCheck=true`,
`--set sync.replicaCount=${syncReplicaCount}`,
`--set-string sync.image.tag="${imageTag}"`,
`--set-string renderer.image.tag="${imageTag}"`,
`--set renderer.app.host=${host}`,
...serviceAnnotations,
`--timeout 10m`,
flag,
.github/deployment/front/affine.nginx.conf
+5 -5
View File
@@ -6,11 +6,6 @@ server {
try_files $uri/index.html $uri/ $uri /admin/index.html;
}
location ~ ^/(_plugin|assets|imgs|js|plugins|static)/ {
root /app/dist/;
try_files $uri $uri/ =404;
}
set $app_root_path /app/dist/;
set $mobile_root /app/dist/;
set_by_lua $affine_env 'return os.getenv("AFFINE_ENV")';
@@ -28,6 +23,11 @@ server {
set $app_root_path $mobile_root;
}
location ~ ^/(_plugin|assets|imgs|js|plugins|static)/ {
root $app_root_path;
try_files $uri $uri/ =404;
}
location / {
root $app_root_path;
index index.html;
.github/deployment/node/Dockerfile
+1
View File
@@ -3,6 +3,7 @@ FROM node:20-bookworm-slim
COPY ./packages/backend/server /app
COPY ./packages/frontend/web/dist /app/static
COPY ./packages/frontend/admin/dist /app/static/admin
COPY ./packages/frontend/mobile/dist /app/static/mobile
WORKDIR /app
RUN apt-get update && \
.github/helm/affine/charts/graphql/templates/deployment.yaml
+5 -5
View File
@@ -76,7 +76,7 @@ spec:
- name: AFFINE_SERVER_HTTPS
value: "{{ .Values.app.https }}"
- name: ENABLE_R2_OBJECT_STORAGE
value: "{{ .Values.app.objectStorage.r2.enabled }}"
value: "{{ .Values.global.objectStorage.r2.enabled }}"
- name: FEATURES_EARLY_ACCESS_PREVIEW
value: "{{ .Values.app.features.earlyAccessPreview }}"
- name: FEATURES_SYNC_CLIENT_VERSION_CHECK
@@ -122,21 +122,21 @@ spec:
- name: DOC_MERGE_USE_JWST_CODEC
value: "true"
{{ end }}
{{ if .Values.app.objectStorage.r2.enabled }}
{{ if .Values.global.objectStorage.r2.enabled }}
- name: R2_OBJECT_STORAGE_ACCOUNT_ID
valueFrom:
secretKeyRef:
name: "{{ .Values.app.objectStorage.r2.secretName }}"
name: "{{ .Values.global.objectStorage.r2.secretName }}"
key: accountId
- name: R2_OBJECT_STORAGE_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: "{{ .Values.app.objectStorage.r2.secretName }}"
name: "{{ .Values.global.objectStorage.r2.secretName }}"
key: accessKeyId
- name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: "{{ .Values.app.objectStorage.r2.secretName }}"
name: "{{ .Values.global.objectStorage.r2.secretName }}"
key: secretAccessKey
{{ end }}
{{ if .Values.app.captcha.enabled }}
.github/helm/affine/charts/graphql/templates/migration.yaml
+4 -4
View File
@@ -37,21 +37,21 @@ spec:
- name: DATABASE_URL
value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.gcloud.cloudSqlInternal }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
{{ end }}
{{ if .Values.app.objectStorage.r2.enabled }}
{{ if .Values.global.objectStorage.r2.enabled }}
- name: R2_OBJECT_STORAGE_ACCOUNT_ID
valueFrom:
secretKeyRef:
name: "{{ .Values.app.objectStorage.r2.secretName }}"
name: "{{ .Values.global.objectStorage.r2.secretName }}"
key: accountId
- name: R2_OBJECT_STORAGE_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: "{{ .Values.app.objectStorage.r2.secretName }}"
name: "{{ .Values.global.objectStorage.r2.secretName }}"
key: accessKeyId
- name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: "{{ .Values.app.objectStorage.r2.secretName }}"
name: "{{ .Values.global.objectStorage.r2.secretName }}"
key: secretAccessKey
{{ end }}
resources:
.github/helm/affine/charts/graphql/templates/r2-secret.yaml
-11
View File
@@ -1,11 +0,0 @@
{{- if .Values.app.objectStorage.r2.enabled -}}
apiVersion: v1
kind: Secret
metadata:
name: "{{ .Values.app.objectStorage.r2.secretName }}"
type: Opaque
data:
accountId: {{ .Values.app.objectStorage.r2.accountId | b64enc }}
accessKeyId: {{ .Values.app.objectStorage.r2.accessKeyId | b64enc }}
secretAccessKey: {{ .Values.app.objectStorage.r2.secretAccessKey | b64enc }}
{{- end }}
.github/helm/affine/charts/graphql/values.yaml
+1 -8
View File
@@ -29,14 +29,7 @@ app:
secretName: copilot
openai:
key: ''
objectStorage:
r2:
enabled: false
secretName: r2
accountId: ''
accessKeyId: ''
secretAccessKey: ''
oauth:
oauth:
google:
enabled: false
secretName: oauth-google
.github/helm/affine/charts/renderer/Chart.yaml
+11
View File
@@ -0,0 +1,11 @@
apiVersion: v2
name: renderer
description: AFFiNE renderer server
type: application
version: 0.0.0
appVersion: "0.16.0"
dependencies:
- name: gcloud-sql-proxy
version: 0.0.0
repository: "file://../gcloud-sql-proxy"
condition: .global.database.gcloud.enabled
.github/helm/affine/charts/renderer/templates/NOTES.txt
+16
View File
@@ -0,0 +1,16 @@
1. Get the application URL by running these commands:
{{- if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "renderer.fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "renderer.fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "renderer.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "renderer.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
.github/helm/affine/charts/renderer/templates/_helpers.tpl
+63
View File
@@ -0,0 +1,63 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "renderer.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "renderer.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "renderer.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "renderer.labels" -}}
helm.sh/chart: {{ include "renderer.chart" . }}
{{ include "renderer.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
monitoring: enabled
{{- end }}
{{/*
Selector labels
*/}}
{{- define "renderer.selectorLabels" -}}
app.kubernetes.io/name: {{ include "renderer.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "renderer.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "renderer.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
.github/helm/affine/charts/renderer/templates/deployment.yaml
+124
View File
@@ -0,0 +1,124 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "renderer.fullname" . }}
labels:
{{- include "renderer.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
{{- include "renderer.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "renderer.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "renderer.serviceAccountName" . }}
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
- name: AFFINE_PRIVATE_KEY
valueFrom:
secretKeyRef:
name: "{{ .Values.global.secret.secretName }}"
key: key
- name: NODE_ENV
value: "{{ .Values.env }}"
- name: NODE_OPTIONS
value: "--max-old-space-size=4096"
- name: NO_COLOR
value: "1"
- name: DEPLOYMENT_TYPE
value: "affine"
- name: SERVER_FLAVOR
value: "renderer"
- name: AFFINE_ENV
value: "{{ .Release.Namespace }}"
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: pg-postgresql
key: postgres-password
- name: DATABASE_URL
value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.url }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
- name: REDIS_SERVER_ENABLED
value: "true"
- name: REDIS_SERVER_HOST
value: "{{ .Values.global.redis.host }}"
- name: REDIS_SERVER_PORT
value: "{{ .Values.global.redis.port }}"
- name: REDIS_SERVER_USER
value: "{{ .Values.global.redis.username }}"
- name: REDIS_SERVER_PASSWORD
valueFrom:
secretKeyRef:
name: redis
key: redis-password
- name: REDIS_SERVER_DATABASE
value: "{{ .Values.global.redis.database }}"
- name: AFFINE_SERVER_PORT
value: "{{ .Values.service.port }}"
- name: AFFINE_SERVER_SUB_PATH
value: "{{ .Values.app.path }}"
- name: AFFINE_SERVER_HOST
value: "{{ .Values.app.host }}"
- name: AFFINE_SERVER_HTTPS
value: "{{ .Values.app.https }}"
- name: ENABLE_R2_OBJECT_STORAGE
value: "{{ .Values.global.objectStorage.r2.enabled }}"
{{ if .Values.global.objectStorage.r2.enabled }}
- name: R2_OBJECT_STORAGE_ACCOUNT_ID
valueFrom:
secretKeyRef:
name: "{{ .Values.global.objectStorage.r2.secretName }}"
key: accountId
- name: R2_OBJECT_STORAGE_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: "{{ .Values.global.objectStorage.r2.secretName }}"
key: accessKeyId
- name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: "{{ .Values.global.objectStorage.r2.secretName }}"
key: secretAccessKey
{{ end }}
ports:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
livenessProbe:
httpGet:
path: /info
port: http
initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }}
readinessProbe:
httpGet:
path: /info
port: http
initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
.github/helm/affine/charts/renderer/templates/service.yaml
+19
View File
@@ -0,0 +1,19 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "graphql.fullname" . }}
labels:
{{- include "graphql.labels" . | nindent 4 }}
{{- with .Values.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "graphql.selectorLabels" . | nindent 4 }}
.github/helm/affine/charts/renderer/templates/serviceaccount.yaml
+12
View File
@@ -0,0 +1,12 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "graphql.serviceAccountName" . }}
labels:
{{- include "graphql.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
.github/helm/affine/charts/renderer/templates/tests/test-connection.yaml
+15
View File
@@ -0,0 +1,15 @@
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "renderer.fullname" . }}-test-connection"
labels:
{{- include "renderer.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: wget
image: busybox
command: ['wget']
args: ['{{ include "renderer.fullname" . }}:{{ .Values.service.port }}']
restartPolicy: Never
.github/helm/affine/charts/renderer/values.yaml
+38
View File
@@ -0,0 +1,38 @@
replicaCount: 1
image:
repository: ghcr.io/toeverything/affine-graphql
pullPolicy: IfNotPresent
tag: ''
imagePullSecrets: []
nameOverride: ''
fullnameOverride: ''
# map to NODE_ENV environment variable
env: 'production'
app:
# AFFINE_SERVER_SUB_PATH
path: ''
# AFFINE_SERVER_HOST
host: '0.0.0.0'
https: true
serviceAccount:
create: true
annotations: {}
name: 'affine-renderer'
podAnnotations: {}
podSecurityContext:
fsGroup: 2000
resources:
requests:
cpu: '4'
memory: 4Gi
probe:
initialDelaySeconds: 20
nodeSelector: {}
tolerations: []
affinity: {}
.github/helm/affine/templates/configmap.yaml
+9
View File
@@ -0,0 +1,9 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Release.Name }}-runtime-config
data:
web-assets-manifest: |-
{{ .Files.Get "web-assets-manifest.json" | nindent 4 }}
mobile-assets-manifest: |-
{{ .Files.Get "mobile-assets-manifest.json" | nindent 4 }}
.github/helm/affine/templates/ingress.yaml
+9
View File
@@ -60,6 +60,15 @@ spec:
name: affine-graphql
port:
number: {{ .Values.graphql.service.port }}
{{- if eq .Values.global.app.buildType "canary" }}
- path: /workspace
pathType: Prefix
backend:
service:
name: affine-renderer
port:
number: {{ .Values.graphql.service.port }}
{{- end }}
- path: /
pathType: Prefix
backend:
.github/helm/affine/charts/graphql/templates/pg-secret.yaml → .github/helm/affine/templates/pg-secret.yaml
View File
.github/helm/affine/templates/r2-secret.yaml
+11
View File
@@ -0,0 +1,11 @@
{{- if .Values.global.objectStorage.r2.enabled -}}
apiVersion: v1
kind: Secret
metadata:
name: "{{ .Values.global.objectStorage.r2.secretName }}"
type: Opaque
data:
accountId: {{ .Values.global.objectStorage.r2.accountId | b64enc }}
accessKeyId: {{ .Values.global.objectStorage.r2.accessKeyId | b64enc }}
secretAccessKey: {{ .Values.global.objectStorage.r2.secretAccessKey | b64enc }}
{{- end }}
.github/helm/affine/charts/graphql/templates/redis-secret.yaml → .github/helm/affine/templates/redis-secret.yaml
View File
.github/helm/affine/values.yaml
+16
View File
@@ -1,4 +1,6 @@
global:
app:
buildType: 'stable'
ingress:
enabled: false
className: ''
@@ -28,6 +30,13 @@ global:
username: ''
password: ''
database: 0
objectStorage:
r2:
enabled: false
secretName: r2
accountId: ''
accessKeyId: ''
secretAccessKey: ''
gke:
enabled: true
@@ -45,6 +54,13 @@ sync:
annotations:
cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}'
renderer:
service:
type: ClusterIP
port: 3000
annotations:
cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}'
web:
service:
type: ClusterIP
.github/workflows/build-server-image.yml → .github/workflows/build-images.yml
+166 -10
View File
@@ -6,11 +6,6 @@ on:
flavor:
type: string
required: true
workflow_dispatch:
inputs:
flavor:
type: string
required: false
env:
NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
@@ -43,6 +38,103 @@ jobs:
path: ./packages/backend/server/dist
if-no-files-found: error
build-web:
name: Build @affine/web
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.flavor }}
steps:
- uses: actions/checkout@v4
- name: Setup Version
id: version
uses: ./.github/actions/setup-version
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Build Core
run: yarn nx build @affine/web --skip-nx-cache
env:
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
BUILD_TYPE: ${{ github.event.inputs.flavor }}
CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: 'affine-web'
SENTRY_RELEASE: ${{ steps.version.outputs.APP_VERSION }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
- name: Upload web artifact
uses: actions/upload-artifact@v4
with:
name: web
path: ./packages/frontend/web/dist
if-no-files-found: error
build-admin:
name: Build @affine/admin
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.flavor }}
steps:
- uses: actions/checkout@v4
- name: Setup Version
id: version
uses: ./.github/actions/setup-version
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Build Admin
run: yarn nx build @affine/admin --skip-nx-cache
env:
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
BUILD_TYPE: ${{ github.event.inputs.flavor }}
CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: 'affine-admin'
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
- name: Upload admin artifact
uses: actions/upload-artifact@v4
with:
name: admin
path: ./packages/frontend/admin/dist
if-no-files-found: error
build-mobile:
name: Build @affine/mobile
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.flavor }}
steps:
- uses: actions/checkout@v4
- name: Setup Version
id: version
uses: ./.github/actions/setup-version
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Build Mobile
run: yarn nx build @affine/mobile --skip-nx-cache
env:
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
BUILD_TYPE: ${{ github.event.inputs.flavor }}
CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: 'affine-mobile'
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
- name: Upload mobile artifact
uses: actions/upload-artifact@v4
with:
name: mobile
path: ./packages/frontend/mobile/dist
if-no-files-found: error
build-web-selfhost:
name: Build @affine/web selfhost
runs-on: ubuntu-latest
@@ -70,6 +162,31 @@ jobs:
path: ./packages/frontend/web/dist
if-no-files-found: error
build-mobile-selfhost:
name: Build @affine/mobile selfhost
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.flavor }}
steps:
- uses: actions/checkout@v4
- name: Setup Version
id: version
uses: ./.github/actions/setup-version
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Build Mobile
run: yarn nx build @affine/mobile --skip-nx-cache
env:
BUILD_TYPE: ${{ github.event.inputs.flavor }}
PUBLIC_PATH: '/'
SELF_HOSTED: true
MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
- name: Upload mobile artifact
uses: actions/upload-artifact@v4
with:
name: selfhost-mobile
path: ./packages/frontend/mobile/dist
if-no-files-found: error
build-admin-selfhost:
name: Build @affine/admin selfhost
runs-on: ubuntu-latest
@@ -81,7 +198,7 @@ jobs:
uses: ./.github/actions/setup-version
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Build Core
- name: Build admin
run: yarn nx build @affine/admin --skip-nx-cache
env:
BUILD_TYPE: ${{ github.event.inputs.flavor }}
@@ -131,12 +248,16 @@ jobs:
path: ./packages/backend/native/server-native.node
if-no-files-found: error
build-docker:
name: Build Docker
build-images:
name: Build Images
runs-on: ubuntu-latest
needs:
- build-server
- build-web
- build-mobile
- build-admin
- build-web-selfhost
- build-mobile-selfhost
- build-admin-selfhost
- build-server-native
steps:
@@ -195,17 +316,41 @@ jobs:
registry-url: https://npm.pkg.github.com
scope: '@toeverything'
- name: Download web artifact
uses: actions/download-artifact@v4
with:
name: web
path: ./packages/frontend/web/dist
- name: Download mobile artifact
uses: actions/download-artifact@v4
with:
name: mobile
path: ./packages/frontend/mobile/dist
- name: Download admin artifact
uses: actions/download-artifact@v4
with:
name: admin
path: ./packages/frontend/admin/dist
- name: Download selfhost web artifact
uses: actions/download-artifact@v4
with:
name: selfhost-web
path: ./packages/frontend/web/dist
path: ./packages/frontend/web/dist/selfhost
- name: Download selfhost mobile artifact
uses: actions/download-artifact@v4
with:
name: selfhost-mobile
path: ./packages/frontend/mobile/dist/selfhost
- name: Download selfhost admin artifact
uses: actions/download-artifact@v4
with:
name: selfhost-admin
path: ./packages/frontend/admin/dist
path: ./packages/frontend/admin/dist/selfhost
- name: Install Node.js dependencies
run: |
@@ -220,6 +365,17 @@ jobs:
id: version
uses: ./.github/actions/setup-version
- name: Build front Dockerfile
uses: docker/build-push-action@v6
with:
context: .
push: true
pull: true
platforms: linux/amd64,linux/arm64
provenance: true
file: .github/deployment/front/Dockerfile
tags: ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}
- name: Build graphql Dockerfile
uses: docker/build-push-action@v6
with:
.github/workflows/build-selfhost-image.yml
+1 -1
View File
@@ -20,6 +20,6 @@ permissions:
jobs:
build-image:
name: Build Image
uses: ./.github/workflows/build-server-image.yml
uses: ./.github/workflows/build-images.yml
with:
flavor: ${{ github.event.inputs.flavor }}
.github/workflows/build-test.yml
+6 -6
View File
@@ -117,7 +117,7 @@ jobs:
name: E2E Test
runs-on: ubuntu-latest
env:
DISTRIBUTION: browser
DISTRIBUTION: web
IN_CI_TEST: true
strategy:
fail-fast: false
@@ -177,7 +177,7 @@ jobs:
name: E2E Migration Test
runs-on: ubuntu-latest
env:
DISTRIBUTION: browser
DISTRIBUTION: web
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
@@ -204,7 +204,7 @@ jobs:
needs:
- build-native
env:
DISTRIBUTION: browser
DISTRIBUTION: web
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
@@ -311,7 +311,7 @@ jobs:
# always skip cache because its fast, and cache configuration is always changing
run: yarn nx build @affine/web --skip-nx-cache
env:
DISTRIBUTION: 'desktop'
DISTRIBUTION: desktop
- name: zip web
run: tar -czf dist.tar.gz --directory=packages/frontend/electron/renderer/dist .
- name: Upload web artifact
@@ -327,7 +327,7 @@ jobs:
needs: build-server-native
env:
NODE_ENV: test
DISTRIBUTION: browser
DISTRIBUTION: web
services:
postgres:
image: postgres
@@ -396,7 +396,7 @@ jobs:
name: ${{ matrix.tests.name }}
runs-on: ubuntu-latest
env:
DISTRIBUTION: browser
DISTRIBUTION: web
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
IN_CI_TEST: true
strategy:
.github/workflows/deploy.yml
+6 -162
View File
@@ -62,171 +62,19 @@ jobs:
echo "version=$prev_version" >> $GITHUB_OUTPUT
echo "namesapce=$namespace" >> $GITHUB_OUTPUT
build-server-image:
name: Build Server Image
uses: ./.github/workflows/build-server-image.yml
build-images:
name: Build Images
uses: ./.github/workflows/build-images.yml
secrets: inherit
with:
flavor: ${{ github.event.inputs.flavor }}
build-web:
name: Build @affine/web
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.flavor }}
steps:
- uses: actions/checkout@v4
- name: Setup Version
id: version
uses: ./.github/actions/setup-version
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Build Core
run: yarn nx build @affine/web --skip-nx-cache
env:
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
BUILD_TYPE: ${{ github.event.inputs.flavor }}
CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: 'affine-web'
SENTRY_RELEASE: ${{ steps.version.outputs.APP_VERSION }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
- name: Upload web artifact
uses: actions/upload-artifact@v4
with:
name: web
path: ./packages/frontend/web/dist
if-no-files-found: error
build-admin:
name: Build @affine/admin
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.flavor }}
steps:
- uses: actions/checkout@v4
- name: Setup Version
id: version
uses: ./.github/actions/setup-version
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Build Admin
run: yarn nx build @affine/admin --skip-nx-cache
env:
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
BUILD_TYPE: ${{ github.event.inputs.flavor }}
CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: 'affine-admin'
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
- name: Upload admin artifact
uses: actions/upload-artifact@v4
with:
name: admin
path: ./packages/frontend/admin/dist
if-no-files-found: error
build-mobile:
name: Build @affine/mobile
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.flavor }}
steps:
- uses: actions/checkout@v4
- name: Setup Version
id: version
uses: ./.github/actions/setup-version
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Build Mobile
run: yarn nx build @affine/mobile --skip-nx-cache
env:
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
BUILD_TYPE: ${{ github.event.inputs.flavor }}
CAPTCHA_SITE_KEY: ${{ secrets.CAPTCHA_SITE_KEY }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: 'affine-mobile'
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }}
MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
- name: Upload mobile artifact
uses: actions/upload-artifact@v4
with:
name: mobile
path: ./packages/frontend/mobile/dist
if-no-files-found: error
build-frontend-image:
name: Build Frontend Image
runs-on: ubuntu-latest
needs:
- build-web
- build-admin
- build-mobile
steps:
- uses: actions/checkout@v4
- name: Download web artifact
uses: actions/download-artifact@v4
with:
name: web
path: ./packages/frontend/web/dist
- name: Download admin artifact
uses: actions/download-artifact@v4
with:
name: admin
path: ./packages/frontend/admin/dist
- name: Download mobile artifact
uses: actions/download-artifact@v4
with:
name: mobile
path: ./packages/frontend/mobile/dist
- name: Setup env
run: |
echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
if [ -z "${{ inputs.flavor }}" ]
then
echo "RELEASE_FLAVOR=canary" >> "$GITHUB_ENV"
else
echo "RELEASE_FLAVOR=${{ inputs.flavor }}" >> "$GITHUB_ENV"
fi
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
logout: false
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build front Dockerfile
uses: docker/build-push-action@v6
with:
context: .
push: true
pull: true
platforms: linux/amd64,linux/arm64
provenance: true
file: .github/deployment/front/Dockerfile
tags: ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}-${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-front:${{env.RELEASE_FLAVOR}}
deploy:
name: Deploy to cluster
if: ${{ github.event_name == 'workflow_dispatch' }}
environment: ${{ github.event.inputs.flavor }}
needs:
- build-frontend-image
- build-server-image
- build-images
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -276,11 +124,7 @@ jobs:
deploy-done:
needs:
- output-prev-version
- build-web
- build-admin
- build-mobile
- build-frontend-image
- build-server-image
- build-images
- deploy
if: always()
runs-on: ubuntu-latest
Cargo.lock
Generated
+7
View File
@@ -58,6 +58,7 @@ dependencies = [
"sha3",
"tiktoken-rs",
"tokio",
"v_htmlescape",
"y-octo",
]
@@ -2218,6 +2219,12 @@ dependencies = [
"serde",
]
[[package]]
name = "v_htmlescape"
version = "0.15.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c"
[[package]]
name = "valuable"
version = "0.1.0"
Cargo.toml
+21 -20
View File
@@ -3,26 +3,27 @@ members = ["./packages/backend/native", "./packages/frontend/native", "./packag
resolver = "2"
[workspace.dependencies]
anyhow = "1"
chrono = "0.4"
dotenv = "0.15"
file-format = { version = "0.25", features = ["reader"] }
mimalloc = "0.1"
napi = { version = "3.0.0-alpha.1", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] }
napi-build = { version = "2" }
napi-derive = { version = "3.0.0-alpha.1" }
notify = { version = "6", features = ["serde"] }
once_cell = "1"
parking_lot = "0.12"
rand = "0.8"
serde = "1"
serde_json = "1"
sha3 = "0.10"
sqlx = { version = "0.8", default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] }
tiktoken-rs = "0.5"
tokio = "1.37"
uuid = "1.8"
y-octo = { git = "https://github.com/y-crdt/y-octo.git", branch = "main" }
anyhow = "1"
chrono = "0.4"
dotenv = "0.15"
file-format = { version = "0.25", features = ["reader"] }
mimalloc = "0.1"
napi = { version = "3.0.0-alpha.1", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] }
napi-build = { version = "2" }
napi-derive = { version = "3.0.0-alpha.1" }
notify = { version = "6", features = ["serde"] }
once_cell = "1"
parking_lot = "0.12"
rand = "0.8"
serde = "1"
serde_json = "1"
sha3 = "0.10"
sqlx = { version = "0.8", default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] }
tiktoken-rs = "0.5"
tokio = "1.37"
uuid = "1.8"
v_htmlescape = "0.15"
y-octo = { git = "https://github.com/y-crdt/y-octo.git", branch = "main" }
[profile.dev.package.sqlx-macros]
opt-level = 3
packages/backend/native/Cargo.toml
+9 -8
View File
@@ -7,14 +7,15 @@ version = "1.0.0"
crate-type = ["cdylib"]
[dependencies]
chrono = { workspace = true }
file-format = { workspace = true }
napi = { workspace = true }
napi-derive = { workspace = true }
rand = { workspace = true }
sha3 = { workspace = true }
tiktoken-rs = { workspace = true }
y-octo = { workspace = true }
chrono = { workspace = true }
file-format = { workspace = true }
napi = { workspace = true }
napi-derive = { workspace = true }
rand = { workspace = true }
sha3 = { workspace = true }
tiktoken-rs = { workspace = true }
v_htmlescape = { workspace = true }
y-octo = { workspace = true }
[target.'cfg(not(target_os = "linux"))'.dependencies]
mimalloc = { workspace = true }
packages/backend/native/index.d.ts
Vendored
+2
View File
@@ -8,6 +8,8 @@ export declare function fromModelName(modelName: string): Tokenizer | null
export declare function getMime(input: Uint8Array): string
export declare function htmlSanitize(input: string): string
/**
* Merge updates in form like `Y.applyUpdate(doc, update)` way and return the
* result binary.
packages/backend/native/index.js
+1
View File
@@ -11,3 +11,4 @@ export const mintChallengeResponse = binding.mintChallengeResponse;
export const getMime = binding.getMime;
export const Tokenizer = binding.Tokenizer;
export const fromModelName = binding.fromModelName;
export const htmlSanitize = binding.htmlSanitize;
packages/backend/native/src/html_sanitize.rs
+4
View File
@@ -0,0 +1,4 @@
#[napi]
pub fn html_sanitize(input: String) -> String {
v_htmlescape::escape(&input).to_string()
}
packages/backend/native/src/lib.rs
+1
View File
@@ -2,6 +2,7 @@
pub mod file_type;
pub mod hashcash;
pub mod html_sanitize;
pub mod tiktoken;
use std::fmt::{Debug, Display};
packages/backend/server/package.json
+1 -1
View File
@@ -70,6 +70,7 @@
"graphql-upload": "^16.0.2",
"html-validate": "^8.20.1",
"ioredis": "^5.3.2",
"is-mobile": "^4.0.0",
"keyv": "^5.0.0",
"lodash-es": "^4.17.21",
"mixpanel": "^0.18.0",
@@ -94,7 +95,6 @@
"ts-node": "^10.9.2",
"typescript": "^5.4.5",
"ws": "^8.16.0",
"xss": "^1.0.15",
"yjs": "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch",
"zod": "^3.22.4"
},
packages/backend/server/src/core/doc-renderer/controller.ts
+145 -47
View File
@@ -1,93 +1,191 @@
import { Controller, Get, Param, Res } from '@nestjs/common';
import type { Response } from 'express';
import xss from 'xss';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { DocNotFound } from '../../fundamentals';
import { Controller, Get, Logger, Param, Req, Res } from '@nestjs/common';
import type { Request, Response } from 'express';
import isMobile from 'is-mobile';
import { Config, metrics, URLHelper } from '../../fundamentals';
import { htmlSanitize } from '../../native';
import { Public } from '../auth';
import { PermissionService } from '../permission';
import { PageDocContent } from '../utils/blocksuite';
import { DocContentService } from './service';
interface RenderOptions {
og: boolean;
content: boolean;
title: string;
summary: string;
avatar?: string;
}
interface HtmlAssets {
css: string[];
js: string[];
publicPath: string;
gitHash: string;
description: string;
}
const defaultAssets: HtmlAssets = {
css: [],
js: [],
publicPath: '/',
gitHash: '',
description: '',
};
@Controller('/workspace/:workspaceId/:docId')
export class DocRendererController {
private readonly logger = new Logger(DocRendererController.name);
private readonly webAssets: HtmlAssets = defaultAssets;
private readonly mobileAssets: HtmlAssets = defaultAssets;
constructor(
private readonly doc: DocContentService,
private readonly permission: PermissionService
) {}
private readonly permission: PermissionService,
private readonly config: Config,
private readonly url: URLHelper
) {
try {
const webConfigMapsPath = join(
this.config.projectRoot,
this.config.isSelfhosted ? 'static/selfhost' : 'static',
'assets-manifest.json'
);
const mobileConfigMapsPath = join(
this.config.projectRoot,
this.config.isSelfhosted ? 'static/mobile/selfhost' : 'static/mobile',
'assets-manifest.json'
);
this.webAssets = JSON.parse(readFileSync(webConfigMapsPath, 'utf-8'));
this.mobileAssets = JSON.parse(
readFileSync(mobileConfigMapsPath, 'utf-8')
);
} catch (e) {
if (this.config.node.prod) {
throw e;
}
}
}
@Public()
@Get()
async render(
@Req() req: Request,
@Res() res: Response,
@Param('workspaceId') workspaceId: string,
@Param('docId') docId: string
) {
if (workspaceId === docId) {
throw new DocNotFound({ spaceId: workspaceId, docId });
}
const assets: HtmlAssets =
this.config.affine.canary &&
isMobile({
ua: req.headers['user-agent'] ?? undefined,
})
? this.mobileAssets
: this.webAssets;
// if page is public, show all
// if page is private, but workspace public og is on, show og but not content
const opts: RenderOptions = {
og: false,
content: false,
};
const isPagePublic = await this.permission.isPublicPage(workspaceId, docId);
if (isPagePublic) {
opts.og = true;
opts.content = true;
} else {
const allowPreview = await this.permission.allowUrlPreview(workspaceId);
if (allowPreview) {
opts.og = true;
}
}
let docContent = opts.og
? await this.doc.getPageContent(workspaceId, docId)
: null;
if (!docContent) {
docContent = { title: 'untitled', summary: '' };
let opts: RenderOptions | null = null;
try {
opts =
workspaceId === docId
? await this.renderWorkspace(workspaceId)
: await this.getPageContent(workspaceId, docId);
metrics.doc.counter('render').add(1);
} catch (e) {
this.logger.error('failed to render page', e);
}
res.setHeader('Content-Type', 'text/html');
if (!opts.og) {
if (!opts) {
res.setHeader('X-Robots-Tag', 'noindex');
}
res.send(this._render(docContent, opts));
res.send(this._render(opts, assets));
}
_render(doc: PageDocContent, { og }: RenderOptions): string {
const title = xss(doc.title);
const summary = xss(doc.summary);
private async getPageContent(
workspaceId: string,
docId: string
): Promise<RenderOptions | null> {
let allowUrlPreview = await this.permission.isPublicPage(
workspaceId,
docId
);
if (!allowUrlPreview) {
// if page is private, but workspace url preview is on
allowUrlPreview = await this.permission.allowUrlPreview(workspaceId);
}
if (allowUrlPreview) {
return this.doc.getPageContent(workspaceId, docId);
}
return null;
}
private async renderWorkspace(
workspaceId: string
): Promise<RenderOptions | null> {
const allowUrlPreview = await this.permission.allowUrlPreview(workspaceId);
if (allowUrlPreview) {
const workspaceContent = await this.doc.getWorkspaceContent(workspaceId);
if (workspaceContent) {
return {
title: workspaceContent.name,
summary: '',
avatar: workspaceContent.avatarKey
? this.url.link(
`/api/workspaces/${workspaceId}/blobs/${workspaceContent.avatarKey}`
)
: undefined,
};
}
}
return null;
}
_render(opts: RenderOptions | null, assets: HtmlAssets): string {
const title = opts?.title
? htmlSanitize(`${opts.title} | AFFiNE`)
: 'AFFiNE';
const summary = opts ? htmlSanitize(opts.summary) : assets.description;
const image = opts?.avatar ?? 'https://affine.pro/og.jpeg';
return `
<!DOCTYPE html>
<html>
<head>
<title>${title} | AFFiNE</title>
<meta charset="utf-8" />
<meta
name="viewport"
content="width=device-width, initial-scale=1, maximum-scale=1"
/>
<title>${title}</title>
<meta name="theme-color" content="#fafafa" />
<link rel="preconnect" href="${assets.publicPath}">
<link rel="manifest" href="/manifest.json" />
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
<link rel="icon" sizes="192x192" href="/favicon-192.png" />
${!og ? '<meta name="robots" content="noindex, nofollow" />' : ''}
<meta name="emotion-insertion-point" content="" />
${!opts ? '<meta name="robots" content="noindex, nofollow" />' : ''}
<meta
name="twitter:title"
content="AFFiNE: There can be more than Notion and Miro."
content="${title}"
/>
<meta name="twitter:description" content="${title}" />
<meta name="twitter:description" content="${summary}" />
<meta name="twitter:site" content="@AffineOfficial" />
<meta name="twitter:image" content="https://affine.pro/og.jpeg" />
<meta name="twitter:image" content="${image}" />
<meta property="og:title" content="${title}" />
<meta property="og:description" content="${summary}" />
<meta property="og:image" content="https://affine.pro/og.jpeg" />
<meta property="og:image" content="${image}" />
${assets.css.map(url => `<link rel="stylesheet" href="${url}" />`).join('\n')}
</head>
<body>
<div id="app" data-version="${assets.gitHash}"></div>
${assets.js.map(url => `<script type="module" src="${url}"></script>`).join('\n')}
</body>
</html>
`;
packages/backend/server/src/core/doc-renderer/service.ts
+8 -4
View File
@@ -1,7 +1,7 @@
import { Injectable } from '@nestjs/common';
import { applyUpdate, Doc } from 'yjs';
import { Cache } from '../../fundamentals';
import { Cache, type EventPayload, OnEvent } from '../../fundamentals';
import { PgWorkspaceDocStorageAdapter } from '../doc';
import {
type PageDocContent,
@@ -78,11 +78,15 @@ export class DocContentService {
return content;
}
async markDocContentCacheStale(workspaceId: string, guid: string) {
@OnEvent('snapshot.updated')
async markDocContentCacheStale({
workspaceId,
id,
}: EventPayload<'snapshot.updated'>) {
const key =
workspaceId === guid
workspaceId === id
? `workspace:${workspaceId}:content`
: `workspace:${workspaceId}:doc:${guid}:content`;
: `workspace:${workspaceId}:doc:${id}:content`;
await this.cache.delete(key);
}
}
packages/backend/server/src/core/doc/adapters/workspace.ts
+10 -1
View File
@@ -6,6 +6,7 @@ import {
Cache,
DocHistoryNotFound,
DocNotFound,
EventEmitter,
FailedToSaveUpdates,
FailedToUpsertSnapshot,
metrics,
@@ -30,6 +31,7 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
private readonly db: PrismaClient,
private readonly mutex: Mutex,
private readonly cache: Cache,
private readonly event: EventEmitter,
protected override readonly options: DocStorageOptions
) {
super(options);
@@ -97,7 +99,6 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
metrics.doc.counter('doc_update_insert_failed').add(1);
throw new FailedToSaveUpdates();
}
return timestamp;
}
@@ -463,6 +464,14 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
// the updates has been applied to current `doc` must have been seen by the other process as well.
// The `updatedSnapshot` will be `undefined` in this case.
const updatedSnapshot = result.at(0);
if (updatedSnapshot) {
this.event.emit('snapshot.updated', {
workspaceId: snapshot.spaceId,
id: snapshot.docId,
});
}
return !!updatedSnapshot;
} catch (e) {
metrics.doc.counter('snapshot_upsert_failed').add(1);
packages/backend/server/src/core/selfhost/index.ts
+8 -5
View File
@@ -22,7 +22,6 @@ export class SetupMiddleware implements NestMiddleware {
use = (req: Request, res: Response, next: (error?: Error | any) => void) => {
// never throw
// eslint-disable-next-line @typescript-eslint/no-floating-promises
this.server
.initialized()
.then(initialized => {
@@ -59,6 +58,10 @@ export class SelfhostModule implements OnModuleInit {
) {}
onModuleInit() {
// selfhost static file location
// web => 'static/selfhost'
// admin => 'static/admin/selfhost'
// mobile => 'static/mobile/selfhost'
const staticPath = join(this.config.projectRoot, 'static');
// in command line mode
if (!this.adapterHost.httpAdapter) {
@@ -73,7 +76,7 @@ export class SelfhostModule implements OnModuleInit {
});
app.use(
basePath + '/admin',
serveStatic(join(staticPath, 'admin'), {
serveStatic(join(staticPath, 'admin', 'selfhost'), {
redirect: false,
index: false,
})
@@ -83,7 +86,7 @@ export class SelfhostModule implements OnModuleInit {
[basePath + '/admin', basePath + '/admin/*'],
this.check.use,
(_req, res) => {
res.sendFile(join(staticPath, 'admin', 'index.html'));
res.sendFile(join(staticPath, 'admin', 'selfhost', 'index.html'));
}
);
@@ -92,13 +95,13 @@ export class SelfhostModule implements OnModuleInit {
});
app.use(
basePath,
serveStatic(staticPath, {
serveStatic(join(staticPath, 'selfhost'), {
redirect: false,
index: false,
})
);
app.get('*', this.check.use, (_req, res) => {
res.sendFile(join(staticPath, 'index.html'));
res.sendFile(join(staticPath, 'selfhost', 'index.html'));
});
}
}
packages/backend/server/src/fundamentals/event/def.ts
+1 -5
View File
@@ -13,12 +13,8 @@ export interface WorkspaceEvents {
}
export interface DocEvents {
updated: Payload<
Pick<Snapshot, 'id' | 'workspaceId'> & {
previous: Pick<Snapshot, 'blob' | 'state' | 'updatedAt'>;
}
>;
deleted: Payload<Pick<Snapshot, 'id' | 'workspaceId'>>;
updated: Payload<Pick<Snapshot, 'id' | 'workspaceId'>>;
}
export interface UserEvents {
packages/backend/server/src/native.ts
+1
View File
@@ -32,3 +32,4 @@ export const mintChallengeResponse = async (resource: string, bits: number) => {
export const getMime = serverNativeModule.getMime;
export const Tokenizer = serverNativeModule.Tokenizer;
export const fromModelName = serverNativeModule.fromModelName;
export const htmlSanitize = serverNativeModule.htmlSanitize;
packages/backend/server/tests/app/selfhost.e2e.ts
+7 -6
View File
@@ -18,16 +18,17 @@ const test = ava as TestFn<{
}>;
function initTestStaticFiles(staticPath: string) {
mkdirSync(path.join(staticPath, 'admin'), { recursive: true });
const files = {
'index.html': `<!DOCTYPE html><html><body>AFFiNE</body><script src="main.js"/></html>`,
'main.js': `const name = 'affine'`,
'admin/index.html': `<!DOCTYPE html><html><body>AFFiNE Admin</body><script src="/admin/main.js"/></html>`,
'admin/main.js': `const name = 'affine-admin'`,
'selfhost/index.html': `<!DOCTYPE html><html><body>AFFiNE</body><script src="main.js"/></html>`,
'selfhost/main.js': `const name = 'affine'`,
'admin/selfhost/index.html': `<!DOCTYPE html><html><body>AFFiNE Admin</body><script src="/admin/main.js"/></html>`,
'admin/selfhost/main.js': `const name = 'affine-admin'`,
};
for (const [filename, content] of Object.entries(files)) {
writeFileSync(path.join(staticPath, filename), content);
const filePath = path.join(staticPath, filename);
mkdirSync(path.dirname(filePath), { recursive: true });
writeFileSync(filePath, content);
}
}
packages/common/env/src/global.ts
Vendored
+2 -2
View File
@@ -10,7 +10,7 @@ export const runtimeFlagsSchema = z.object({
serverUrlPrefix: z.string(),
appVersion: z.string(),
editorVersion: z.string(),
distribution: z.enum(['browser', 'desktop', 'admin', 'mobile']),
distribution: z.enum(['web', 'desktop', 'admin', 'mobile']),
appBuildType: z.union([
z.literal('stable'),
z.literal('beta'),
@@ -104,7 +104,7 @@ export function setupGlobal() {
environment = {
isDesktopEdition: runtimeConfig.distribution !== 'mobile',
isMobileEdition: runtimeConfig.distribution === 'mobile',
isDesktopWeb: runtimeConfig.distribution === 'browser',
isDesktopWeb: runtimeConfig.distribution === 'web',
isMobileWeb: runtimeConfig.distribution === 'mobile',
isElectron,
isDebug,
packages/frontend/component/.storybook/main.ts
+2 -1
View File
@@ -55,9 +55,10 @@ export default {
define: {
'process.env.CAPTCHA_SITE_KEY': `"${process.env.CAPTCHA_SITE_KEY}"`,
runtimeConfig: getRuntimeConfig({
distribution: 'browser',
distribution: 'web',
mode: 'development',
channel: 'canary',
static: false,
coverage: false,
static: false,
}),
packages/frontend/graphql/src/schema.ts
+4
View File
@@ -1074,6 +1074,8 @@ export interface UpdateUserInput {
}
export interface UpdateWorkspaceInput {
/** Enable url previous when sharing */
enableUrlPreview: InputMaybe<Scalars['Boolean']['input']>;
id: Scalars['ID']['input'];
/** is Public workspace */
public: InputMaybe<Scalars['Boolean']['input']>;
@@ -1217,6 +1219,8 @@ export interface WorkspaceType {
blobsSize: Scalars['Int']['output'];
/** Workspace created date */
createdAt: Scalars['DateTime']['output'];
/** Enable url previous when sharing */
enableUrlPreview: Scalars['Boolean']['output'];
/** Enabled features of workspace */
features: Array<FeatureType>;
histories: Array<DocHistoryType>;
packages/frontend/web/package.json
+1 -1
View File
@@ -5,7 +5,7 @@
"private": true,
"browser": "src/index.tsx",
"scripts": {
"build": "cross-env DISTRIBUTION=browser yarn workspace @affine/cli build",
"build": "cross-env DISTRIBUTION=web yarn workspace @affine/cli build",
"dev": "yarn workspace @affine/cli dev",
"static-server": "yarn workspace @affine/cli dev --static"
},
scripts/setup/global.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import { setupGlobal } from '@affine/env/global';
process.env.RUNTIME_CONFIG = JSON.stringify(
getRuntimeConfig({
distribution: 'browser',
distribution: 'web',
mode: 'development',
channel: 'canary',
static: false,
tools/cli/src/bin/build.ts
+1 -1
View File
@@ -31,7 +31,7 @@ const getChannel = () => {
let entry: BuildFlags['entry'];
const { DISTRIBUTION } = process.env;
const { DISTRIBUTION = 'web' } = process.env;
const cwd = getCwdFromDistribution(DISTRIBUTION);
tools/cli/src/bin/dev.ts
+3 -3
View File
@@ -13,7 +13,7 @@ import { createWebpackConfig } from '../webpack/webpack.config.js';
const flags: BuildFlags = {
distribution:
(process.env.DISTRIBUTION as BuildFlags['distribution']) ?? 'browser',
(process.env.DISTRIBUTION as BuildFlags['distribution']) ?? 'web',
mode: 'development',
static: false,
channel: 'canary',
@@ -42,7 +42,7 @@ const buildFlags = process.argv.includes('--static')
message: 'Distribution',
options: [
{
value: 'browser',
value: 'web',
},
{
value: 'desktop',
@@ -54,7 +54,7 @@ const buildFlags = process.argv.includes('--static')
value: 'mobile',
},
],
initialValue: 'browser',
initialValue: 'web',
}),
mode: () =>
p.select({
tools/cli/src/config/cwd.cjs
+4 -2
View File
@@ -15,7 +15,7 @@ module.exports.getCwdFromDistribution = function getCwdFromDistribution(
distribution
) {
switch (distribution) {
case 'browser':
case 'web':
case undefined:
case null:
return join(projectRoot, 'packages/frontend/web');
@@ -26,7 +26,9 @@ module.exports.getCwdFromDistribution = function getCwdFromDistribution(
case 'mobile':
return join(projectRoot, 'packages/frontend/mobile');
default: {
throw new Error('DISTRIBUTION must be one of browser, desktop');
throw new Error(
'DISTRIBUTION must be one of web, desktop, admin, mobile'
);
}
}
};
tools/cli/src/config/index.ts
+1 -1
View File
@@ -1,5 +1,5 @@
export type BuildFlags = {
distribution: 'browser' | 'desktop' | 'admin' | 'mobile';
distribution: 'web' | 'desktop' | 'admin' | 'mobile';
mode: 'development' | 'production';
channel: 'stable' | 'beta' | 'canary' | 'internal';
static: boolean;
tools/cli/src/webpack/template.html
+1
View File
@@ -8,6 +8,7 @@
/>
<title>AFFiNE</title>
<meta name="theme-color" content="#fafafa" />
<link rel="preconnect" href="<%= PUBLIC_PATH %>" />
<link rel="manifest" href="/manifest.json" />
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
<link rel="icon" sizes="192x192" href="/favicon-192.png" />
tools/cli/src/webpack/webpack.config.ts
+27 -3
View File
@@ -5,6 +5,7 @@ import type { BuildFlags } from '@affine/cli/config';
import { Repository } from '@napi-rs/simple-git';
import HTMLPlugin from 'html-webpack-plugin';
import { once } from 'lodash-es';
import webpack from 'webpack';
import { merge } from 'webpack-merge';
import { createConfiguration, rootPath, workspaceRoot } from './config.js';
@@ -48,9 +49,32 @@ export function createWebpackConfig(cwd: string, flags: BuildFlags) {
minify: false,
chunks: [entryName],
filename: `${entryName === 'app' ? 'index' : entryName}.html`, // main entry should take name index.html
templateParameters: {
GIT_SHORT_SHA: gitShortHash(),
DESCRIPTION,
templateParameters: (compilation, assets) => {
if (entryName === 'app') {
// emit assets manifest for ssr
compilation.emitAsset(
`assets-manifest.json`,
new webpack.sources.RawSource(
JSON.stringify(
{
...assets,
gitHash: gitShortHash(),
description: DESCRIPTION,
},
null,
2
)
),
{
immutable: true,
}
);
}
return {
GIT_SHORT_SHA: gitShortHash(),
DESCRIPTION,
PUBLIC_PATH: config.output?.publicPath,
};
},
});
};
yarn.lock
+9 -2
View File
@@ -869,6 +869,7 @@ __metadata:
graphql-upload: "npm:^16.0.2"
html-validate: "npm:^8.20.1"
ioredis: "npm:^5.3.2"
is-mobile: "npm:^4.0.0"
keyv: "npm:^5.0.0"
lodash-es: "npm:^4.17.21"
mixpanel: "npm:^0.18.0"
@@ -896,7 +897,6 @@ __metadata:
ts-node: "npm:^10.9.2"
typescript: "npm:^5.4.5"
ws: "npm:^8.16.0"
xss: "npm:^1.0.15"
yjs: "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch"
zod: "npm:^3.22.4"
bin:
@@ -24792,6 +24792,13 @@ __metadata:
languageName: node
linkType: hard
"is-mobile@npm:^4.0.0":
version: 4.0.0
resolution: "is-mobile@npm:4.0.0"
checksum: 10/1c4f32ab030ac6c203d63b547ef23933eacfebe81fd9d800c86739d5a73afad7983aea4c5e832c3d9c0a63d1e68cd318637490e6406bdda1cbadc8f701d5d557
languageName: node
linkType: hard
"is-my-ip-valid@npm:^1.0.0":
version: 1.0.1
resolution: "is-my-ip-valid@npm:1.0.1"
@@ -36501,7 +36508,7 @@ __metadata:
languageName: node
linkType: hard
"xss@npm:^1.0.15, xss@npm:^1.0.8":
"xss@npm:^1.0.8":
version: 1.0.15
resolution: "xss@npm:1.0.15"
dependencies: