Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-08-24 05:07:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(server): indexer & worker & sync perf (#15504)

Browse Source
This commit is contained in:
DarkSky
2026-08-21 08:11:37 +08:00
committed by GitHub
parent 8c9aad9a9b
commit c57004ea2c
259 changed files with 16530 additions and 17793 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/server-test-env/action.yml
+27 -3
View File
@@ -1,6 +1,12 @@
name: 'Prepare Server Test Environment'
description: 'Prepare Server Test Environment'
inputs:
indexer-provider:
description: 'Search provider used by the server test runtime'
required: false
default: embedded
runs:
using: 'composite'
steps:
@@ -19,12 +25,30 @@ runs:
NODE_ENV: test
run: |
yarn affine @affine/server prisma generate
yarn affine @affine/server data-migration admit-legacy-context-blobs
yarn affine @affine/server prisma migrate deploy
yarn affine @affine/server data-migration run
- name: Import config
shell: bash
env:
DEFAULT_CONFIG: '{}'
run: |
printf '%s\n' "${SERVER_CONFIG:-$DEFAULT_CONFIG}" > ./packages/backend/server/config.json
case '${{ inputs.indexer-provider }}' in
embedded)
default_indexer='{"enabled":true,"provider":{"type":"embedded","endpoint":""}}'
;;
elasticsearch)
default_indexer='{"enabled":true,"provider":{"type":"elasticsearch","endpoint":"http://localhost:9200"}}'
;;
manticoresearch)
default_indexer='{"enabled":true,"provider":{"type":"manticoresearch","endpoint":"http://localhost:9308"}}'
;;
*)
echo "Unsupported test indexer provider: ${{ inputs.indexer-provider }}" >&2
exit 1
;;
esac
if [[ -n "${SERVER_CONFIG:-}" ]]; then
jq --argjson indexer "$default_indexer" '.indexer = $indexer' <<<"$SERVER_CONFIG" > ./packages/backend/server/config.json
else
jq -n --argjson indexer "$default_indexer" '{indexer: $indexer}' > ./packages/backend/server/config.json
fi
.github/helm/affine/charts/doc/Chart.yaml
-11
View File
@@ -1,11 +0,0 @@
apiVersion: v2
name: doc
description: AFFiNE doc server
type: application
version: 0.0.0
appVersion: "0.27.0"
dependencies:
- name: gcloud-sql-proxy
version: 0.0.0
repository: "file://../gcloud-sql-proxy"
condition: .global.database.gcloud.enabled
.github/helm/affine/charts/doc/values.yaml
-46
View File
@@ -1,46 +0,0 @@
replicaCount: 1
image:
repository: ghcr.io/toeverything/affine
pullPolicy: IfNotPresent
tag: ''
imagePullSecrets: []
nameOverride: ''
fullnameOverride: ''
# map to NODE_ENV environment variable
env: 'production'
app:
# AFFINE_SERVER_SUB_PATH
path: ''
# AFFINE_SERVER_HOST
host: '0.0.0.0'
https: true
copilot:
enabled: false
secretName: copilot
openai:
key: ''
serviceAccount:
create: true
annotations: {}
podAnnotations: {}
podSecurityContext:
fsGroup: 2000
resources:
limits:
cpu: '1'
memory: 4Gi
requests:
cpu: '1'
memory: 2Gi
probe:
initialDelaySeconds: 20
timeoutSeconds: 5
nodeSelector: {}
tolerations: []
affinity: {}
.github/helm/affine/charts/front/templates/deployment.yaml
+11 -10
View File
@@ -11,8 +11,9 @@ spec:
{{- include "front.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
@@ -87,15 +88,6 @@ spec:
key: redis-password
- name: REDIS_SERVER_DATABASE
value: "{{ .Values.global.redis.database }}"
- name: AFFINE_INDEXER_SEARCH_PROVIDER
value: "{{ .Values.global.indexer.provider }}"
- name: AFFINE_INDEXER_SEARCH_ENDPOINT
value: "{{ .Values.global.indexer.endpoint }}"
- name: AFFINE_INDEXER_SEARCH_API_KEY
valueFrom:
secretKeyRef:
name: indexer
key: indexer-apiKey
- name: AFFINE_SERVER_PORT
value: "{{ .Values.app.port }}"
- name: AFFINE_SERVER_SUB_PATH
@@ -128,6 +120,15 @@ spec:
successThreshold: {{ default .Values.probe.successThreshold .Values.probe.readiness.successThreshold }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: indexer-config
mountPath: /app/config.json
subPath: config.json
readOnly: true
volumes:
- name: indexer-config
secret:
secretName: indexer
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
.github/helm/affine/charts/front/templates/service-doc.yaml
-19
View File
@@ -1,19 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: {{ .Values.global.docService.name }}
labels:
{{- include "front.labels" . | nindent 4 }}
{{- with .Values.services.doc.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.services.doc.type }}
ports:
- port: {{ .Values.global.docService.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "front.selectorLabels" . | nindent 4 }}
.github/helm/affine/charts/front/values.yaml
-4
View File
@@ -65,10 +65,6 @@ services:
type: ClusterIP
port: 8080
annotations: {}
doc:
type: ClusterIP
annotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
.github/helm/affine/charts/graphql/templates/deployment.yaml
+11 -12
View File
@@ -11,8 +11,9 @@ spec:
{{- include "graphql.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
@@ -83,15 +84,6 @@ spec:
key: redis-password
- name: REDIS_SERVER_DATABASE
value: "{{ .Values.global.redis.database }}"
- name: AFFINE_INDEXER_SEARCH_PROVIDER
value: "{{ .Values.global.indexer.provider }}"
- name: AFFINE_INDEXER_SEARCH_ENDPOINT
value: "{{ .Values.global.indexer.endpoint }}"
- name: AFFINE_INDEXER_SEARCH_API_KEY
valueFrom:
secretKeyRef:
name: indexer
key: indexer-apiKey
- name: AFFINE_SERVER_PORT
value: "{{ .Values.service.port }}"
- name: AFFINE_SERVER_SUB_PATH
@@ -100,8 +92,6 @@ spec:
value: "{{ .Values.app.host }}"
- name: AFFINE_SERVER_HTTPS
value: "{{ .Values.app.https }}"
- name: DOC_SERVICE_ENDPOINT
value: "http://{{ .Values.global.docService.name }}:{{ .Values.global.docService.port }}"
ports:
- name: http
containerPort: {{ .Values.service.port }}
@@ -126,6 +116,15 @@ spec:
successThreshold: {{ default .Values.probe.successThreshold .Values.probe.readiness.successThreshold }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: indexer-config
mountPath: /app/config.json
subPath: config.json
readOnly: true
volumes:
- name: indexer-config
secret:
secretName: indexer
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
.github/helm/affine/charts/graphql/templates/migration.yaml
+9 -9
View File
@@ -60,18 +60,18 @@ spec:
secretKeyRef:
name: redis
key: redis-password
- name: AFFINE_INDEXER_SEARCH_PROVIDER
value: "{{ .Values.global.indexer.provider }}"
- name: AFFINE_INDEXER_SEARCH_ENDPOINT
value: "{{ .Values.global.indexer.endpoint }}"
- name: AFFINE_INDEXER_SEARCH_API_KEY
valueFrom:
secretKeyRef:
name: indexer
key: indexer-apiKey
volumeMounts:
- name: indexer-config
mountPath: /app/config.json
subPath: config.json
readOnly: true
resources:
requests:
cpu: '100m'
memory: '200Mi'
volumes:
- name: indexer-config
secret:
secretName: indexer
restartPolicy: Never
backoffLimit: 1
.github/helm/affine/templates/_helpers.tpl
+7
View File
@@ -60,3 +60,10 @@ Create the name of the service account to use
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
{{/*
Render the indexer runtime configuration shared by the Secret and Pod checksums.
*/}}
{{- define "affine.indexerConfig" -}}
{{- dict "indexer" (dict "enabled" .Values.global.indexer.enabled "provider" (dict "type" .Values.global.indexer.provider "endpoint" .Values.global.indexer.endpoint "apiKey" .Values.global.indexer.apiKey "username" .Values.global.indexer.username "password" .Values.global.indexer.password)) | toJson -}}
{{- end }}
.github/helm/affine/templates/indexer-secret.yaml
+1 -3
View File
@@ -1,4 +1,3 @@
{{- if .Values.global.indexer.apiKey -}}
apiVersion: v1
kind: Secret
metadata:
@@ -9,5 +8,4 @@ metadata:
"helm.sh/hook-delete-policy": before-hook-creation
type: Opaque
data:
indexer-apiKey: {{ .Values.global.indexer.apiKey | b64enc }}
{{- end }}
config.json: {{ include "affine.indexerConfig" . | b64enc | quote }}
.github/helm/affine/templates/worker-deployment.yaml
+149
View File
@@ -0,0 +1,149 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: affine-worker
labels:
app.kubernetes.io/name: affine-worker
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
monitoring: enabled
spec:
replicas: {{ .Values.worker.replicaCount }}
selector:
matchLabels:
app.kubernetes.io/name: affine-worker
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
annotations:
checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }}
{{- with .Values.worker.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
app.kubernetes.io/name: affine-worker
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
serviceAccountName: {{ .Values.worker.serviceAccount.name }}
securityContext:
{{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
{{- with .Values.global.database.gcloud }}
{{- if .enabled }}
initContainers:
- name: wait-for-cloud-sql-proxy
image: busybox:1.36.1
imagePullPolicy: IfNotPresent
command:
- /bin/sh
- -ec
- |
until wget -q -T 2 -O /dev/null "http://{{ $.Values.global.database.host }}:9801/startup"; do
echo "waiting for cloud sql proxy to become ready"
sleep 2
done
{{- end }}
{{- end }}
containers:
- name: worker
image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Values.graphql.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: IfNotPresent
securityContext:
{{- toYaml .Values.worker.securityContext | nindent 12 }}
env:
- name: AFFINE_PRIVATE_KEY
valueFrom:
secretKeyRef:
name: "{{ .Values.global.secret.secretName }}"
key: key
- name: NODE_ENV
value: "{{ .Values.worker.env }}"
- name: NODE_OPTIONS
value: "{{ .Values.worker.nodeOptions }}"
- name: NO_COLOR
value: "1"
- name: DEPLOYMENT_TYPE
value: "{{ .Values.global.deployment.type }}"
- name: DEPLOYMENT_PLATFORM
value: "{{ .Values.global.deployment.platform }}"
- name: SERVER_FLAVOR
value: "worker"
- name: AFFINE_ENV
value: "{{ .Release.Namespace }}"
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: pg-postgresql
key: postgres-password
- name: DATABASE_URL
value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.host }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
- name: REDIS_SERVER_ENABLED
value: "true"
- name: REDIS_SERVER_HOST
value: "{{ .Values.global.redis.host }}"
- name: REDIS_SERVER_PORT
value: "{{ .Values.global.redis.port }}"
- name: REDIS_SERVER_USER
value: "{{ .Values.global.redis.username }}"
- name: REDIS_SERVER_PASSWORD
valueFrom:
secretKeyRef:
name: redis
key: redis-password
- name: REDIS_SERVER_DATABASE
value: "{{ .Values.global.redis.database }}"
- name: AFFINE_SERVER_PORT
value: "{{ .Values.worker.app.port }}"
- name: AFFINE_SERVER_HOST
value: "{{ .Values.worker.app.host }}"
- name: AFFINE_SERVER_HTTPS
value: "{{ .Values.worker.app.https }}"
ports:
- name: http
containerPort: {{ .Values.worker.app.port }}
protocol: TCP
livenessProbe:
httpGet:
path: /info
port: http
initialDelaySeconds: {{ default .Values.worker.probe.initialDelaySeconds .Values.worker.probe.liveness.initialDelaySeconds }}
timeoutSeconds: {{ default .Values.worker.probe.timeoutSeconds .Values.worker.probe.liveness.timeoutSeconds }}
periodSeconds: {{ default .Values.worker.probe.periodSeconds .Values.worker.probe.liveness.periodSeconds }}
failureThreshold: {{ default .Values.worker.probe.failureThreshold .Values.worker.probe.liveness.failureThreshold }}
successThreshold: {{ default .Values.worker.probe.successThreshold .Values.worker.probe.liveness.successThreshold }}
readinessProbe:
httpGet:
path: /info
port: http
initialDelaySeconds: {{ default .Values.worker.probe.initialDelaySeconds .Values.worker.probe.readiness.initialDelaySeconds }}
timeoutSeconds: {{ default .Values.worker.probe.timeoutSeconds .Values.worker.probe.readiness.timeoutSeconds }}
periodSeconds: {{ default .Values.worker.probe.periodSeconds .Values.worker.probe.readiness.periodSeconds }}
failureThreshold: {{ default .Values.worker.probe.failureThreshold .Values.worker.probe.readiness.failureThreshold }}
successThreshold: {{ default .Values.worker.probe.successThreshold .Values.worker.probe.readiness.successThreshold }}
resources:
{{- toYaml .Values.worker.resources | nindent 12 }}
volumeMounts:
- name: indexer-config
mountPath: /app/config.json
subPath: config.json
readOnly: true
- name: tmp
mountPath: /tmp
volumes:
- name: indexer-config
secret:
secretName: indexer
- name: tmp
emptyDir: {}
{{- with .Values.worker.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.worker.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.worker.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
.github/helm/affine/templates/worker-serviceaccount.yaml
+14
View File
@@ -0,0 +1,14 @@
{{- if .Values.worker.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ .Values.worker.serviceAccount.name }}
labels:
app.kubernetes.io/name: affine-worker
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- with .Values.worker.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
.github/helm/affine/values.yaml
+48 -8
View File
@@ -28,13 +28,12 @@ global:
password: ''
database: 0
indexer:
provider: ''
enabled: false
provider: embedded
endpoint: ''
apiKey: ''
username: ''
password: ''
docService:
name: 'affine-doc'
port: 3020
deployment:
# change to 'selfhosted' and 'unknown' if this chart is ready to be used for selfhosted deployment
type: 'affine'
@@ -65,7 +64,48 @@ front:
name: affine-web
type: ClusterIP
port: 8080
doc:
type: ClusterIP
annotations:
cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}'
worker:
replicaCount: 1
image:
repository: ghcr.io/toeverything/affine
tag: ''
env: 'production'
nodeOptions: '--max-old-space-size=3072'
app:
port: 3002
host: '0.0.0.0'
https: true
podAnnotations: {}
podSecurityContext:
fsGroup: 2000
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
serviceAccount:
create: true
annotations: {}
name: 'affine-worker'
resources:
limits:
cpu: '1'
memory: 4Gi
requests:
cpu: '1'
memory: 2Gi
probe:
initialDelaySeconds: 20
timeoutSeconds: 5
periodSeconds: 10
failureThreshold: 6
successThreshold: 1
liveness:
initialDelaySeconds: 60
failureThreshold: 12
readiness: {}
nodeSelector: {}
tolerations: []
affinity: {}
.github/workflows/build-test.yml
+94 -24
View File
@@ -21,7 +21,6 @@ env:
COVERAGE: true
MACOSX_DEPLOYMENT_TARGET: '11.6'
DEPLOYMENT_TYPE: affine
AFFINE_INDEXER_ENABLED: true
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -890,10 +889,6 @@ jobs:
ports:
- 1025:1025
- 8025:8025
indexer:
image: manticoresearch/manticore:10.1.0
ports:
- 9308:9308
steps:
- uses: actions/checkout@v7
@@ -938,8 +933,8 @@ jobs:
NODE_ENV: test
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
REDIS_SERVER_HOST: localhost
AFFINE_INDEXER_SEARCH_PROVIDER: elasticsearch
AFFINE_INDEXER_SEARCH_ENDPOINT: http://localhost:9200
SERVER_CONFIG: >-
{"indexer":{"enabled":true,"provider":{"type":"elasticsearch","endpoint":"http://localhost:9200"}}}
services:
postgres:
image: pgvector/pgvector:pg16
@@ -992,10 +987,12 @@ jobs:
path: ./packages/backend/native
- name: Prepare Server Test Environment
with:
indexer-provider: elasticsearch
uses: ./.github/actions/server-test-env
- name: Run server tests with elasticsearch only
run: yarn affine @affine/server test:coverage "**/*/*elasticsearch.spec.ts" --forbid-only
run: yarn affine @affine/server e2e:coverage src/__tests__/e2e/indexer/search-docs.spec.ts --forbid-only
env:
CARGO_TARGET_DIR: '${{ github.workspace }}/target'
@@ -1008,6 +1005,93 @@ jobs:
name: affine
fail_ci_if_error: false
server-test-manticore:
name: Server Test with Manticore
runs-on: ubuntu-latest
needs:
- build-server-native
env:
NODE_ENV: test
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
REDIS_SERVER_HOST: localhost
SEARCH_MS_URL: http://localhost:9308
SERVER_CONFIG: >-
{"indexer":{"enabled":true,"provider":{"type":"manticoresearch","endpoint":"http://localhost:9308"}}}
services:
postgres:
image: pgvector/pgvector:pg16
env:
POSTGRES_PASSWORD: affine
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
redis:
image: redis
ports:
- 6379:6379
mailer:
image: mailhog/mailhog
ports:
- 1025:1025
- 8025:8025
indexer:
image: manticoresearch/manticore:10.1.0
options: >-
--health-cmd "wget -q -O /dev/null http://localhost:9308/"
--health-interval 5s
--health-timeout 3s
--health-retries 12
ports:
- 9308:9308
steps:
- uses: actions/checkout@v7
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
extra-flags: workspaces focus @affine/monorepo @affine/server
electron-install: false
full-cache: true
- name: Setup Rust
uses: ./.github/actions/build-rust
with:
target: x86_64-unknown-linux-gnu
package: '@affine/server-native'
no-build: 'true'
- name: Download server-native.node
uses: actions/download-artifact@v4
with:
name: server-native.node
path: ./packages/backend/native
- name: Prepare Server Test Environment
uses: ./.github/actions/server-test-env
with:
indexer-provider: manticoresearch
- name: Run server tests with Manticore
run: yarn affine @affine/server e2e:coverage src/__tests__/e2e/indexer/search-docs.spec.ts --forbid-only
- name: Run Manticore provider contract
run: >-
cargo test --manifest-path packages/backend/native/Cargo.toml
remote_providers_apply_search_and_delete_the_same_contract --lib
- name: Upload server test coverage results
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./packages/backend/server/.coverage/lcov.info
flags: server-test
name: affine
fail_ci_if_error: false
server-e2e-test:
# the new version of server e2e test should be super fast, so sharding testing is not needed
name: Server E2E Test
@@ -1034,10 +1118,6 @@ jobs:
image: redis
ports:
- 6379:6379
indexer:
image: manticoresearch/manticore:10.1.0
ports:
- 9308:9308
steps:
- uses: actions/checkout@v7
@@ -1165,10 +1245,6 @@ jobs:
ports:
- 1025:1025
- 8025:8025
indexer:
image: manticoresearch/manticore:10.1.0
ports:
- 9308:9308
steps:
- uses: actions/checkout@v7
@@ -1238,10 +1314,6 @@ jobs:
image: redis
ports:
- 6379:6379
indexer:
image: manticoresearch/manticore:10.1.0
ports:
- 9308:9308
steps:
- uses: actions/checkout@v7
@@ -1321,10 +1393,6 @@ jobs:
ports:
- 1025:1025
- 8025:8025
indexer:
image: manticoresearch/manticore:10.1.0
ports:
- 9308:9308
steps:
- uses: actions/checkout@v7
@@ -1528,6 +1596,8 @@ jobs:
- build-electron-renderer
- native-unit-test
- server-test
- server-test-elasticsearch
- server-test-manticore
- server-e2e-test
- rust-test
- rust-test-filter
.github/workflows/copilot-test.yml
-8
View File
@@ -61,10 +61,6 @@ jobs:
ports:
- 1025:1025
- 8025:8025
indexer:
image: manticoresearch/manticore:10.1.0
ports:
- 9308:9308
steps:
- uses: actions/checkout@v7
@@ -131,10 +127,6 @@ jobs:
image: redis
ports:
- 6379:6379
indexer:
image: manticoresearch/manticore:10.1.0
ports:
- 9308:9308
steps:
- uses: actions/checkout@v7