diff --git a/.docker/selfhost/compose.yml b/.docker/selfhost/compose.yml index f9b8376e4e..d600b811bf 100644 --- a/.docker/selfhost/compose.yml +++ b/.docker/selfhost/compose.yml @@ -18,7 +18,6 @@ services: environment: - REDIS_SERVER_HOST=redis - DATABASE_URL=postgresql://affine@postgres:5432/affine - - AFFINE_INDEXER_ENABLED=false restart: unless-stopped affine_migration: @@ -31,7 +30,6 @@ services: environment: - REDIS_SERVER_HOST=redis - DATABASE_URL=postgresql://affine@postgres:5432/affine - - AFFINE_INDEXER_ENABLED=false depends_on: postgres: condition: service_healthy diff --git a/.docker/selfhost/config.example.json b/.docker/selfhost/config.example.json index b728fd7cf2..38b447ba9b 100644 --- a/.docker/selfhost/config.example.json +++ b/.docker/selfhost/config.example.json @@ -2,5 +2,9 @@ "$schema": "https://github.com/toeverything/affine/releases/latest/download/config.schema.json", "server": { "name": "AFFiNE Self Hosted Server" + }, + "indexer": { + "enabled": true, + "provider.type": "embedded" } -} \ No newline at end of file +} diff --git a/.docker/selfhost/config.json.example b/.docker/selfhost/config.json.example index e3ed7ac32b..eeec280496 100644 --- a/.docker/selfhost/config.json.example +++ b/.docker/selfhost/config.json.example @@ -10,5 +10,9 @@ "enabled": true, "allowCustomEndpoint": true } + }, + "indexer": { + "enabled": true, + "provider.type": "embedded" } -} \ No newline at end of file +} diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index 6664d01481..6f00d1a3e9 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -209,6 +209,15 @@ "description": "Whether request abuse source facts should trust Cloudflare headers from the origin edge.\n@default false", "default": false }, + "signInRateLimit": { + "type": "object", + "description": "Limits for sign-in attempts shared through Redis by source IP and email. ttl is measured in milliseconds.\n@default {\"ttl\":60000,\"ipLimit\":20,\"emailLimit\":5}", + "default": { + "ttl": 60000, + "ipLimit": 20, + "emailLimit": 5 + } + }, "inviteQuotaShadowMode": { "type": "boolean", "description": "Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions.\n@default false", @@ -971,17 +980,6 @@ } } }, - "docService": { - "type": "object", - "description": "Configuration for docService module", - "properties": { - "endpoint": { - "type": "string", - "description": "The endpoint of the doc service.\n@default \"\"\n@environment `DOC_SERVICE_ENDPOINT`", - "default": "" - } - } - }, "telemetry": { "type": "object", "description": "Configuration for telemetry module", @@ -1495,13 +1493,13 @@ }, "provider.type": { "type": "string", - "description": "Indexer search service provider name\n@default \"manticoresearch\"\n@environment `AFFINE_INDEXER_SEARCH_PROVIDER`", - "default": "manticoresearch" + "description": "Indexer search provider. Self-hosted uses the embedded provider by default; remote providers require an endpoint.\n@default \"embedded\"\n@environment `AFFINE_INDEXER_SEARCH_PROVIDER`", + "default": "embedded" }, "provider.endpoint": { "type": "string", - "description": "Indexer search service endpoint\n@default \"http://localhost:9308\"\n@environment `AFFINE_INDEXER_SEARCH_ENDPOINT`", - "default": "http://localhost:9308" + "description": "Remote indexer endpoint. Not used by the embedded provider.\n@default \"\"\n@environment `AFFINE_INDEXER_SEARCH_ENDPOINT`", + "default": "" }, "provider.apiKey": { "type": "string", diff --git a/.github/actions/server-test-env/action.yml b/.github/actions/server-test-env/action.yml index 50b3a6bc2b..9ce243289b 100644 --- a/.github/actions/server-test-env/action.yml +++ b/.github/actions/server-test-env/action.yml @@ -1,6 +1,12 @@ name: 'Prepare Server Test Environment' description: 'Prepare Server Test Environment' +inputs: + indexer-provider: + description: 'Search provider used by the server test runtime' + required: false + default: embedded + runs: using: 'composite' steps: @@ -19,12 +25,30 @@ runs: NODE_ENV: test run: | yarn affine @affine/server prisma generate + yarn affine @affine/server data-migration admit-legacy-context-blobs yarn affine @affine/server prisma migrate deploy yarn affine @affine/server data-migration run - name: Import config shell: bash - env: - DEFAULT_CONFIG: '{}' run: | - printf '%s\n' "${SERVER_CONFIG:-$DEFAULT_CONFIG}" > ./packages/backend/server/config.json + case '${{ inputs.indexer-provider }}' in + embedded) + default_indexer='{"enabled":true,"provider":{"type":"embedded","endpoint":""}}' + ;; + elasticsearch) + default_indexer='{"enabled":true,"provider":{"type":"elasticsearch","endpoint":"http://localhost:9200"}}' + ;; + manticoresearch) + default_indexer='{"enabled":true,"provider":{"type":"manticoresearch","endpoint":"http://localhost:9308"}}' + ;; + *) + echo "Unsupported test indexer provider: ${{ inputs.indexer-provider }}" >&2 + exit 1 + ;; + esac + if [[ -n "${SERVER_CONFIG:-}" ]]; then + jq --argjson indexer "$default_indexer" '.indexer = $indexer' <<<"$SERVER_CONFIG" > ./packages/backend/server/config.json + else + jq -n --argjson indexer "$default_indexer" '{indexer: $indexer}' > ./packages/backend/server/config.json + fi diff --git a/.github/helm/affine/charts/doc/Chart.yaml b/.github/helm/affine/charts/doc/Chart.yaml deleted file mode 100644 index 18d7b9c470..0000000000 --- a/.github/helm/affine/charts/doc/Chart.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v2 -name: doc -description: AFFiNE doc server -type: application -version: 0.0.0 -appVersion: "0.27.0" -dependencies: - - name: gcloud-sql-proxy - version: 0.0.0 - repository: "file://../gcloud-sql-proxy" - condition: .global.database.gcloud.enabled diff --git a/.github/helm/affine/charts/doc/values.yaml b/.github/helm/affine/charts/doc/values.yaml deleted file mode 100644 index e0a30a4183..0000000000 --- a/.github/helm/affine/charts/doc/values.yaml +++ /dev/null @@ -1,46 +0,0 @@ -replicaCount: 1 -image: - repository: ghcr.io/toeverything/affine - pullPolicy: IfNotPresent - tag: '' - -imagePullSecrets: [] -nameOverride: '' -fullnameOverride: '' -# map to NODE_ENV environment variable -env: 'production' -app: - # AFFINE_SERVER_SUB_PATH - path: '' - # AFFINE_SERVER_HOST - host: '0.0.0.0' - https: true - copilot: - enabled: false - secretName: copilot - openai: - key: '' -serviceAccount: - create: true - annotations: {} - -podAnnotations: {} - -podSecurityContext: - fsGroup: 2000 - -resources: - limits: - cpu: '1' - memory: 4Gi - requests: - cpu: '1' - memory: 2Gi - -probe: - initialDelaySeconds: 20 - timeoutSeconds: 5 - -nodeSelector: {} -tolerations: [] -affinity: {} diff --git a/.github/helm/affine/charts/front/templates/deployment.yaml b/.github/helm/affine/charts/front/templates/deployment.yaml index e36e10daf3..ee82343c61 100644 --- a/.github/helm/affine/charts/front/templates/deployment.yaml +++ b/.github/helm/affine/charts/front/templates/deployment.yaml @@ -11,8 +11,9 @@ spec: {{- include "front.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} annotations: + checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }} + {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} labels: @@ -87,15 +88,6 @@ spec: key: redis-password - name: REDIS_SERVER_DATABASE value: "{{ .Values.global.redis.database }}" - - name: AFFINE_INDEXER_SEARCH_PROVIDER - value: "{{ .Values.global.indexer.provider }}" - - name: AFFINE_INDEXER_SEARCH_ENDPOINT - value: "{{ .Values.global.indexer.endpoint }}" - - name: AFFINE_INDEXER_SEARCH_API_KEY - valueFrom: - secretKeyRef: - name: indexer - key: indexer-apiKey - name: AFFINE_SERVER_PORT value: "{{ .Values.app.port }}" - name: AFFINE_SERVER_SUB_PATH @@ -128,6 +120,15 @@ spec: successThreshold: {{ default .Values.probe.successThreshold .Values.probe.readiness.successThreshold }} resources: {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: indexer-config + mountPath: /app/config.json + subPath: config.json + readOnly: true + volumes: + - name: indexer-config + secret: + secretName: indexer {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/.github/helm/affine/charts/front/templates/service-doc.yaml b/.github/helm/affine/charts/front/templates/service-doc.yaml deleted file mode 100644 index 16ccd93f29..0000000000 --- a/.github/helm/affine/charts/front/templates/service-doc.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.global.docService.name }} - labels: - {{- include "front.labels" . | nindent 4 }} - {{- with .Values.services.doc.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.services.doc.type }} - ports: - - port: {{ .Values.global.docService.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "front.selectorLabels" . | nindent 4 }} diff --git a/.github/helm/affine/charts/front/values.yaml b/.github/helm/affine/charts/front/values.yaml index cc4ac4b6bb..4b4ba7cc83 100644 --- a/.github/helm/affine/charts/front/values.yaml +++ b/.github/helm/affine/charts/front/values.yaml @@ -65,10 +65,6 @@ services: type: ClusterIP port: 8080 annotations: {} - doc: - type: ClusterIP - annotations: {} - nodeSelector: {} tolerations: [] affinity: {} diff --git a/.github/helm/affine/charts/graphql/templates/deployment.yaml b/.github/helm/affine/charts/graphql/templates/deployment.yaml index da332b07d0..6b622df079 100644 --- a/.github/helm/affine/charts/graphql/templates/deployment.yaml +++ b/.github/helm/affine/charts/graphql/templates/deployment.yaml @@ -11,8 +11,9 @@ spec: {{- include "graphql.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} annotations: + checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }} + {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} labels: @@ -83,15 +84,6 @@ spec: key: redis-password - name: REDIS_SERVER_DATABASE value: "{{ .Values.global.redis.database }}" - - name: AFFINE_INDEXER_SEARCH_PROVIDER - value: "{{ .Values.global.indexer.provider }}" - - name: AFFINE_INDEXER_SEARCH_ENDPOINT - value: "{{ .Values.global.indexer.endpoint }}" - - name: AFFINE_INDEXER_SEARCH_API_KEY - valueFrom: - secretKeyRef: - name: indexer - key: indexer-apiKey - name: AFFINE_SERVER_PORT value: "{{ .Values.service.port }}" - name: AFFINE_SERVER_SUB_PATH @@ -100,8 +92,6 @@ spec: value: "{{ .Values.app.host }}" - name: AFFINE_SERVER_HTTPS value: "{{ .Values.app.https }}" - - name: DOC_SERVICE_ENDPOINT - value: "http://{{ .Values.global.docService.name }}:{{ .Values.global.docService.port }}" ports: - name: http containerPort: {{ .Values.service.port }} @@ -126,6 +116,15 @@ spec: successThreshold: {{ default .Values.probe.successThreshold .Values.probe.readiness.successThreshold }} resources: {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: indexer-config + mountPath: /app/config.json + subPath: config.json + readOnly: true + volumes: + - name: indexer-config + secret: + secretName: indexer {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/.github/helm/affine/charts/graphql/templates/migration.yaml b/.github/helm/affine/charts/graphql/templates/migration.yaml index 0fc60a80cd..0446305552 100644 --- a/.github/helm/affine/charts/graphql/templates/migration.yaml +++ b/.github/helm/affine/charts/graphql/templates/migration.yaml @@ -60,18 +60,18 @@ spec: secretKeyRef: name: redis key: redis-password - - name: AFFINE_INDEXER_SEARCH_PROVIDER - value: "{{ .Values.global.indexer.provider }}" - - name: AFFINE_INDEXER_SEARCH_ENDPOINT - value: "{{ .Values.global.indexer.endpoint }}" - - name: AFFINE_INDEXER_SEARCH_API_KEY - valueFrom: - secretKeyRef: - name: indexer - key: indexer-apiKey + volumeMounts: + - name: indexer-config + mountPath: /app/config.json + subPath: config.json + readOnly: true resources: requests: cpu: '100m' memory: '200Mi' + volumes: + - name: indexer-config + secret: + secretName: indexer restartPolicy: Never backoffLimit: 1 diff --git a/.github/helm/affine/templates/_helpers.tpl b/.github/helm/affine/templates/_helpers.tpl index 005cc9bf56..c6caa58c64 100644 --- a/.github/helm/affine/templates/_helpers.tpl +++ b/.github/helm/affine/templates/_helpers.tpl @@ -60,3 +60,10 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Render the indexer runtime configuration shared by the Secret and Pod checksums. +*/}} +{{- define "affine.indexerConfig" -}} +{{- dict "indexer" (dict "enabled" .Values.global.indexer.enabled "provider" (dict "type" .Values.global.indexer.provider "endpoint" .Values.global.indexer.endpoint "apiKey" .Values.global.indexer.apiKey "username" .Values.global.indexer.username "password" .Values.global.indexer.password)) | toJson -}} +{{- end }} diff --git a/.github/helm/affine/templates/indexer-secret.yaml b/.github/helm/affine/templates/indexer-secret.yaml index 711e683510..5cbc87223c 100644 --- a/.github/helm/affine/templates/indexer-secret.yaml +++ b/.github/helm/affine/templates/indexer-secret.yaml @@ -1,4 +1,3 @@ -{{- if .Values.global.indexer.apiKey -}} apiVersion: v1 kind: Secret metadata: @@ -9,5 +8,4 @@ metadata: "helm.sh/hook-delete-policy": before-hook-creation type: Opaque data: - indexer-apiKey: {{ .Values.global.indexer.apiKey | b64enc }} -{{- end }} + config.json: {{ include "affine.indexerConfig" . | b64enc | quote }} diff --git a/.github/helm/affine/templates/worker-deployment.yaml b/.github/helm/affine/templates/worker-deployment.yaml new file mode 100644 index 0000000000..77f35d44bf --- /dev/null +++ b/.github/helm/affine/templates/worker-deployment.yaml @@ -0,0 +1,149 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: affine-worker + labels: + app.kubernetes.io/name: affine-worker + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + monitoring: enabled +spec: + replicas: {{ .Values.worker.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: affine-worker + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + annotations: + checksum/indexer-config: {{ include "affine.indexerConfig" . | sha256sum }} + {{- with .Values.worker.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app.kubernetes.io/name: affine-worker + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + serviceAccountName: {{ .Values.worker.serviceAccount.name }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + {{- with .Values.global.database.gcloud }} + {{- if .enabled }} + initContainers: + - name: wait-for-cloud-sql-proxy + image: busybox:1.36.1 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -ec + - | + until wget -q -T 2 -O /dev/null "http://{{ $.Values.global.database.host }}:9801/startup"; do + echo "waiting for cloud sql proxy to become ready" + sleep 2 + done + {{- end }} + {{- end }} + containers: + - name: worker + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Values.graphql.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: IfNotPresent + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + env: + - name: AFFINE_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: "{{ .Values.global.secret.secretName }}" + key: key + - name: NODE_ENV + value: "{{ .Values.worker.env }}" + - name: NODE_OPTIONS + value: "{{ .Values.worker.nodeOptions }}" + - name: NO_COLOR + value: "1" + - name: DEPLOYMENT_TYPE + value: "{{ .Values.global.deployment.type }}" + - name: DEPLOYMENT_PLATFORM + value: "{{ .Values.global.deployment.platform }}" + - name: SERVER_FLAVOR + value: "worker" + - name: AFFINE_ENV + value: "{{ .Release.Namespace }}" + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: pg-postgresql + key: postgres-password + - name: DATABASE_URL + value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.host }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }} + - name: REDIS_SERVER_ENABLED + value: "true" + - name: REDIS_SERVER_HOST + value: "{{ .Values.global.redis.host }}" + - name: REDIS_SERVER_PORT + value: "{{ .Values.global.redis.port }}" + - name: REDIS_SERVER_USER + value: "{{ .Values.global.redis.username }}" + - name: REDIS_SERVER_PASSWORD + valueFrom: + secretKeyRef: + name: redis + key: redis-password + - name: REDIS_SERVER_DATABASE + value: "{{ .Values.global.redis.database }}" + - name: AFFINE_SERVER_PORT + value: "{{ .Values.worker.app.port }}" + - name: AFFINE_SERVER_HOST + value: "{{ .Values.worker.app.host }}" + - name: AFFINE_SERVER_HTTPS + value: "{{ .Values.worker.app.https }}" + ports: + - name: http + containerPort: {{ .Values.worker.app.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /info + port: http + initialDelaySeconds: {{ default .Values.worker.probe.initialDelaySeconds .Values.worker.probe.liveness.initialDelaySeconds }} + timeoutSeconds: {{ default .Values.worker.probe.timeoutSeconds .Values.worker.probe.liveness.timeoutSeconds }} + periodSeconds: {{ default .Values.worker.probe.periodSeconds .Values.worker.probe.liveness.periodSeconds }} + failureThreshold: {{ default .Values.worker.probe.failureThreshold .Values.worker.probe.liveness.failureThreshold }} + successThreshold: {{ default .Values.worker.probe.successThreshold .Values.worker.probe.liveness.successThreshold }} + readinessProbe: + httpGet: + path: /info + port: http + initialDelaySeconds: {{ default .Values.worker.probe.initialDelaySeconds .Values.worker.probe.readiness.initialDelaySeconds }} + timeoutSeconds: {{ default .Values.worker.probe.timeoutSeconds .Values.worker.probe.readiness.timeoutSeconds }} + periodSeconds: {{ default .Values.worker.probe.periodSeconds .Values.worker.probe.readiness.periodSeconds }} + failureThreshold: {{ default .Values.worker.probe.failureThreshold .Values.worker.probe.readiness.failureThreshold }} + successThreshold: {{ default .Values.worker.probe.successThreshold .Values.worker.probe.readiness.successThreshold }} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + volumeMounts: + - name: indexer-config + mountPath: /app/config.json + subPath: config.json + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: indexer-config + secret: + secretName: indexer + - name: tmp + emptyDir: {} + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/.github/helm/affine/templates/worker-serviceaccount.yaml b/.github/helm/affine/templates/worker-serviceaccount.yaml new file mode 100644 index 0000000000..05c8447a6d --- /dev/null +++ b/.github/helm/affine/templates/worker-serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if .Values.worker.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.worker.serviceAccount.name }} + labels: + app.kubernetes.io/name: affine-worker + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- with .Values.worker.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/.github/helm/affine/values.yaml b/.github/helm/affine/values.yaml index 9fa4b6d12a..a04234e12e 100644 --- a/.github/helm/affine/values.yaml +++ b/.github/helm/affine/values.yaml @@ -28,13 +28,12 @@ global: password: '' database: 0 indexer: - provider: '' + enabled: false + provider: embedded endpoint: '' + apiKey: '' username: '' password: '' - docService: - name: 'affine-doc' - port: 3020 deployment: # change to 'selfhosted' and 'unknown' if this chart is ready to be used for selfhosted deployment type: 'affine' @@ -65,7 +64,48 @@ front: name: affine-web type: ClusterIP port: 8080 - doc: - type: ClusterIP - annotations: - cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}' + +worker: + replicaCount: 1 + image: + repository: ghcr.io/toeverything/affine + tag: '' + env: 'production' + nodeOptions: '--max-old-space-size=3072' + app: + port: 3002 + host: '0.0.0.0' + https: true + podAnnotations: {} + podSecurityContext: + fsGroup: 2000 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + serviceAccount: + create: true + annotations: {} + name: 'affine-worker' + resources: + limits: + cpu: '1' + memory: 4Gi + requests: + cpu: '1' + memory: 2Gi + probe: + initialDelaySeconds: 20 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 6 + successThreshold: 1 + liveness: + initialDelaySeconds: 60 + failureThreshold: 12 + readiness: {} + nodeSelector: {} + tolerations: [] + affinity: {} diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 1266d8f748..7de46085f6 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -21,7 +21,6 @@ env: COVERAGE: true MACOSX_DEPLOYMENT_TARGET: '11.6' DEPLOYMENT_TYPE: affine - AFFINE_INDEXER_ENABLED: true concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -890,10 +889,6 @@ jobs: ports: - 1025:1025 - 8025:8025 - indexer: - image: manticoresearch/manticore:10.1.0 - ports: - - 9308:9308 steps: - uses: actions/checkout@v7 @@ -938,8 +933,8 @@ jobs: NODE_ENV: test DATABASE_URL: postgresql://affine:affine@localhost:5432/affine REDIS_SERVER_HOST: localhost - AFFINE_INDEXER_SEARCH_PROVIDER: elasticsearch - AFFINE_INDEXER_SEARCH_ENDPOINT: http://localhost:9200 + SERVER_CONFIG: >- + {"indexer":{"enabled":true,"provider":{"type":"elasticsearch","endpoint":"http://localhost:9200"}}} services: postgres: image: pgvector/pgvector:pg16 @@ -992,10 +987,12 @@ jobs: path: ./packages/backend/native - name: Prepare Server Test Environment + with: + indexer-provider: elasticsearch uses: ./.github/actions/server-test-env - name: Run server tests with elasticsearch only - run: yarn affine @affine/server test:coverage "**/*/*elasticsearch.spec.ts" --forbid-only + run: yarn affine @affine/server e2e:coverage src/__tests__/e2e/indexer/search-docs.spec.ts --forbid-only env: CARGO_TARGET_DIR: '${{ github.workspace }}/target' @@ -1008,6 +1005,93 @@ jobs: name: affine fail_ci_if_error: false + server-test-manticore: + name: Server Test with Manticore + runs-on: ubuntu-latest + needs: + - build-server-native + env: + NODE_ENV: test + DATABASE_URL: postgresql://affine:affine@localhost:5432/affine + REDIS_SERVER_HOST: localhost + SEARCH_MS_URL: http://localhost:9308 + SERVER_CONFIG: >- + {"indexer":{"enabled":true,"provider":{"type":"manticoresearch","endpoint":"http://localhost:9308"}}} + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_PASSWORD: affine + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + redis: + image: redis + ports: + - 6379:6379 + mailer: + image: mailhog/mailhog + ports: + - 1025:1025 + - 8025:8025 + indexer: + image: manticoresearch/manticore:10.1.0 + options: >- + --health-cmd "wget -q -O /dev/null http://localhost:9308/" + --health-interval 5s + --health-timeout 3s + --health-retries 12 + ports: + - 9308:9308 + steps: + - uses: actions/checkout@v7 + + - name: Setup Node.js + uses: ./.github/actions/setup-node + with: + extra-flags: workspaces focus @affine/monorepo @affine/server + electron-install: false + full-cache: true + + - name: Setup Rust + uses: ./.github/actions/build-rust + with: + target: x86_64-unknown-linux-gnu + package: '@affine/server-native' + no-build: 'true' + + - name: Download server-native.node + uses: actions/download-artifact@v4 + with: + name: server-native.node + path: ./packages/backend/native + + - name: Prepare Server Test Environment + uses: ./.github/actions/server-test-env + with: + indexer-provider: manticoresearch + + - name: Run server tests with Manticore + run: yarn affine @affine/server e2e:coverage src/__tests__/e2e/indexer/search-docs.spec.ts --forbid-only + + - name: Run Manticore provider contract + run: >- + cargo test --manifest-path packages/backend/native/Cargo.toml + remote_providers_apply_search_and_delete_the_same_contract --lib + + - name: Upload server test coverage results + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./packages/backend/server/.coverage/lcov.info + flags: server-test + name: affine + fail_ci_if_error: false + server-e2e-test: # the new version of server e2e test should be super fast, so sharding testing is not needed name: Server E2E Test @@ -1034,10 +1118,6 @@ jobs: image: redis ports: - 6379:6379 - indexer: - image: manticoresearch/manticore:10.1.0 - ports: - - 9308:9308 steps: - uses: actions/checkout@v7 @@ -1165,10 +1245,6 @@ jobs: ports: - 1025:1025 - 8025:8025 - indexer: - image: manticoresearch/manticore:10.1.0 - ports: - - 9308:9308 steps: - uses: actions/checkout@v7 @@ -1238,10 +1314,6 @@ jobs: image: redis ports: - 6379:6379 - indexer: - image: manticoresearch/manticore:10.1.0 - ports: - - 9308:9308 steps: - uses: actions/checkout@v7 @@ -1321,10 +1393,6 @@ jobs: ports: - 1025:1025 - 8025:8025 - indexer: - image: manticoresearch/manticore:10.1.0 - ports: - - 9308:9308 steps: - uses: actions/checkout@v7 @@ -1528,6 +1596,8 @@ jobs: - build-electron-renderer - native-unit-test - server-test + - server-test-elasticsearch + - server-test-manticore - server-e2e-test - rust-test - rust-test-filter diff --git a/.github/workflows/copilot-test.yml b/.github/workflows/copilot-test.yml index f176e1f074..e15b1f6ead 100644 --- a/.github/workflows/copilot-test.yml +++ b/.github/workflows/copilot-test.yml @@ -61,10 +61,6 @@ jobs: ports: - 1025:1025 - 8025:8025 - indexer: - image: manticoresearch/manticore:10.1.0 - ports: - - 9308:9308 steps: - uses: actions/checkout@v7 @@ -131,10 +127,6 @@ jobs: image: redis ports: - 6379:6379 - indexer: - image: manticoresearch/manticore:10.1.0 - ports: - - 9308:9308 steps: - uses: actions/checkout@v7 diff --git a/Cargo.lock b/Cargo.lock index 8a306342ff..ed70708113 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -151,6 +151,7 @@ dependencies = [ "lru 0.16.3", "objc2", "objc2-foundation", + "serde_json", "thiserror 2.0.18", "tokio", "uniffi", @@ -264,6 +265,7 @@ dependencies = [ "llm_adapter", "llm_runtime", "matroska", + "memory-indexer", "mimalloc", "mp4parse", "napi", @@ -5036,19 +5038,24 @@ dependencies = [ [[package]] name = "memory-indexer" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a349dd43a5b5efb437a192d7fe37e58feb94f094839d303482dc8bcc8d1043" +checksum = "ff94e8411d369f3197d3bf8b57414651cbfe01353959f91dbc3bfb1f903ef9d3" dependencies = [ + "bincode 2.0.1", + "crc32fast", "jieba-rs", "once_cell", "pinyin", + "roaring", "serde", "serde_json", + "sha2 0.10.9", "smol_str", "strsim 0.11.1", "unicode-normalization", "unicode-script", + "zstd", ] [[package]] @@ -6981,6 +6988,17 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" +[[package]] +name = "roaring" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" +dependencies = [ + "bytemuck", + "byteorder", + "serde", +] + [[package]] name = "roman-numerals-rs" version = "3.1.0" diff --git a/Cargo.toml b/Cargo.toml index 715472ad81..23c282ab51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ resolver = "3" llm_runtime = { version = "0.2", default-features = false } lru = "0.16" matroska = "0.30" - memory-indexer = "0.3.1" + memory-indexer = "0.4.0" mimalloc = "0.1" mp4parse = "0.17" nanoid = "0.4" diff --git a/packages/backend/native/Cargo.toml b/packages/backend/native/Cargo.toml index 7af627693a..5b8e815813 100644 --- a/packages/backend/native/Cargo.toml +++ b/packages/backend/native/Cargo.toml @@ -34,6 +34,7 @@ little_exif = { workspace = true } llm_adapter = { workspace = true, features = ["schema", "ureq-client"] } llm_runtime = { workspace = true, features = ["schema", "ureq-client"] } matroska = { workspace = true } +memory-indexer = { workspace = true } mp4parse = { workspace = true } napi = { workspace = true, features = ["async", "serde-json"] } napi-derive = { workspace = true } diff --git a/packages/backend/native/index.d.ts b/packages/backend/native/index.d.ts index c4e278ab2c..e080782cac 100644 --- a/packages/backend/native/index.d.ts +++ b/packages/backend/native/index.d.ts @@ -65,6 +65,14 @@ export declare class BackendRuntime { reloadConfig(privateKey?: string | undefined | null): Promise health(): Promise runMigrations(): Promise + searchAuthorized(actorUserId: string, workspaceId: string, request: RuntimeSearchRequest): Promise + aggregateAuthorized(actorUserId: string, workspaceId: string, request: RuntimeAggregateRequest): Promise + indexSearchDocument(workspaceId: string, docId: string): Promise + deleteSearchDocument(workspaceId: string, docId: string): Promise + reconcileSearchWorkspace(workspaceId: string): Promise + deleteSearchWorkspace(workspaceId: string): Promise + filterReadableDocs(actorUserId: string, workspaceId: string, docIds: Array): Promise> + searchStatus(): Promise embeddingHealth(): Promise syncEmbeddingState(input: SyncEmbeddingStateInput): Promise embeddingQueueCounts(): Promise @@ -100,7 +108,7 @@ export declare class StorageRuntime { cleanupExpiredPendingBlobs(cutoffMs: number, limit: number): Promise releaseDeletedBlobs(workspaceId: string, limit: number): Promise backfillMissingBlobMetadata(workspaceId: string | undefined | null, limit: number): Promise - rebuildDocBlobRefs(workspaceId: string, docId: string): Promise + rebuildDocBlobRefs(workspaceId: string, docId: string, sourceRevision: number): Promise rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number): Promise reconcileWorkspaceDocuments(workspaceId: string): Promise executeDocumentCleanupCandidates(workspaceId: string | undefined | null, gracePeriodDays: number, limit: number): Promise @@ -152,6 +160,17 @@ export const AFFINE_PRO_LICENSE_AES_KEY: string | undefined | null export const AFFINE_PRO_PUBLIC_KEY: string | undefined | null +export interface AggregateHitsOptions { + fields: Array + highlights: Array + pagination: SearchPagination +} + +export interface AggregateOptions { + hits: AggregateHitsOptions + pagination: SearchPagination +} + export interface AppConfigDescriptor { key: string description: string @@ -1159,6 +1178,14 @@ export interface RotateByokCredentialInput { actorUserId: string } +export interface RuntimeAggregateRequest { + table: SearchTable + queries: Array + rootQuery: number + field: string + options: AggregateOptions +} + export interface RuntimeBlobCleanupExecuteResult { scannedCandidates: number deletedObjects: number @@ -1435,6 +1462,23 @@ export interface RuntimeRetrievalScope { preferredSourceIds: Array } +export interface RuntimeSearchQuery { + queryType: string + field?: string + matchValue?: string + query?: number + queries?: Array + occur?: string + boost?: number +} + +export interface RuntimeSearchRequest { + table: SearchTable + queries: Array + rootQuery: number + options: SearchOptions +} + export interface RuntimeTurnScopeSnapshot { version: number resolvedAt: string @@ -1565,6 +1609,33 @@ export interface ScopeSelectorInput { source: string } +export interface SearchHighlight { + field: string + before: string + end: string +} + +export interface SearchOperationOutput { + ok: boolean + value?: any + errorCode?: string +} + +export interface SearchOptions { + fields: Array + highlights: Array + pagination: SearchPagination +} + +export interface SearchPagination { + limit?: number + skip?: number + cursor?: string +} + +export type SearchTable = 'doc'| +'block'; + export declare function signAuthSessionAccessToken(userId: string, authSessionId: string, keyId: string, secret: Buffer, issuedAt: number, expiresAt: number): string export interface StorageProviderCapabilities { diff --git a/packages/backend/native/src/entitlement.rs b/packages/backend/native/src/entitlement.rs index 0aeccaa206..2a567aceb0 100644 --- a/packages/backend/native/src/entitlement.rs +++ b/packages/backend/native/src/entitlement.rs @@ -28,6 +28,22 @@ const ONE_GB: i64 = 1024 * ONE_MB; const ONE_DAY_SECONDS: i64 = 24 * 60 * 60; const MAX_SEAT_QUANTITY: i32 = 100_000; +pub(crate) fn entitlement_priority(status: &str, plan: &str) -> i32 { + let status = match status { + "active" => 200, + "grace" => 100, + _ => 0, + }; + let plan = match plan { + "team" | "selfhost_team" => 40, + "lifetime_pro" => 30, + "pro" => 20, + "ai" => 10, + _ => 0, + }; + status + plan +} + #[napi(object)] pub struct ResolveEntitlementInput { pub deployment_type: String, diff --git a/packages/backend/native/src/lib.rs b/packages/backend/native/src/lib.rs index 0255357a5b..6faec57e0b 100644 --- a/packages/backend/native/src/lib.rs +++ b/packages/backend/native/src/lib.rs @@ -14,6 +14,7 @@ pub mod llm; pub mod permission; pub mod runtime; pub mod safe_fetch; +pub(crate) mod search_index; pub mod tiktoken; mod userdata_acl; mod utils; diff --git a/packages/backend/native/src/permission/candidates.rs b/packages/backend/native/src/permission/candidates.rs index 78353778ba..156504ff50 100644 --- a/packages/backend/native/src/permission/candidates.rs +++ b/packages/backend/native/src/permission/candidates.rs @@ -23,7 +23,7 @@ pub(super) fn parse_workspace_role(role: &str) -> anyhow::Result } } -fn parse_doc_role(role: &str) -> anyhow::Result { +pub(super) fn parse_doc_role(role: &str) -> anyhow::Result { match role { "none" => Ok(DocRole::None), "external" => Ok(DocRole::External), diff --git a/packages/backend/native/src/permission/mod.rs b/packages/backend/native/src/permission/mod.rs index f85805b615..c491790a97 100644 --- a/packages/backend/native/src/permission/mod.rs +++ b/packages/backend/native/src/permission/mod.rs @@ -10,6 +10,11 @@ use napi_derive::napi; use serde_json::Value; pub use types::*; +pub(crate) fn doc_role_allows(role: &str, action: &str) -> anyhow::Result { + let role = candidates::parse_doc_role(role)?; + Ok(actions::doc_actions_for_role(role).contains(action)) +} + #[napi] pub fn evaluate_permission_v1(input: Value) -> Result { let input = serde_json::from_value::(input) diff --git a/packages/backend/native/src/runtime/backend_runtime/byok/profile.rs b/packages/backend/native/src/runtime/backend_runtime/byok/profile.rs index 479f49f31d..45a9c3eba0 100644 --- a/packages/backend/native/src/runtime/backend_runtime/byok/profile.rs +++ b/packages/backend/native/src/runtime/backend_runtime/byok/profile.rs @@ -47,10 +47,7 @@ pub(in super::super) async fn list(pool: &PgPool, workspace_id: &str) -> Runtime .fetch_all(pool) .await .map_err(|error| RuntimeError::database("list BYOK profiles failed", error))?; - // Rows written by the previous release while it shares the database carry - // only the database-default definition and fail to parse; skip them until - // that release is retired. - Ok(rows.into_iter().filter_map(|row| profile_output(row).ok()).collect()) + rows.into_iter().map(profile_output).collect() } pub(in super::super) async fn create( @@ -103,21 +100,7 @@ pub(in super::super) async fn create( definition, sort_order, enabled, created_by, updated_by, created_at, updated_at ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $10, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - ON CONFLICT (workspace_id, provider, name) DO UPDATE - SET id = EXCLUDED.id, - description = EXCLUDED.description, - encrypted_api_key = EXCLUDED.encrypted_api_key, - definition = EXCLUDED.definition, - sort_order = EXCLUDED.sort_order, - enabled = EXCLUDED.enabled, - revision = 1, - credential_generation = 1, - validation = NULL, - created_by = EXCLUDED.created_by, - updated_by = EXCLUDED.updated_by, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at - WHERE ai_workspace_byok_configs.definition = '{}'::jsonb + ON CONFLICT (workspace_id, provider, name) DO NOTHING RETURNING id, workspace_id, provider, name, description, encrypted_api_key, definition, sort_order, enabled, revision, credential_generation, validation "#, @@ -591,100 +574,3 @@ pub(super) fn require_text(value: &str, field: &'static str) -> RuntimeResult<() Ok(()) } } - -#[cfg(test)] -mod tests { - use super::{ByokPolicy, PgPool, Uuid, create, list}; - use crate::{ - llm::{ - ByokCapabilityInput, ByokEndpointInput, ByokModelDeclarationInput, ByokProfileDefinitionInput, - CreateByokProfileInput, Deployment, - }, - runtime::config::CopilotByokRuntimeConfig, - }; - - #[tokio::test] - async fn list_skips_rows_with_unparseable_legacy_definition() { - let Ok(database_url) = std::env::var("DATABASE_URL") else { - return; - }; - let pool = PgPool::connect(&database_url).await.unwrap(); - let workspace_id = format!("byok-legacy-{}", Uuid::new_v4()); - sqlx::query("INSERT INTO workspaces (id) VALUES ($1)") - .bind(&workspace_id) - .execute(&pool) - .await - .unwrap(); - // a row as written by the previous release while it shares the database: - // definition is left at the database default and cannot be parsed - for (id, name, definition) in [ - (Uuid::new_v4().to_string(), "legacy", "{}"), - ( - Uuid::new_v4().to_string(), - "valid", - r#"{"endpoint":{"kind":"provider_default"},"models":[]}"#, - ), - ] { - sqlx::query( - "INSERT INTO ai_workspace_byok_configs (id, workspace_id, provider, name, encrypted_api_key, definition, \ - created_at, updated_at) VALUES ($1, $2, 'openai', $3, 'x', $4::jsonb, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)", - ) - .bind(&id) - .bind(&workspace_id) - .bind(name) - .bind(definition) - .execute(&pool) - .await - .unwrap(); - } - - let profiles = list(&pool, &workspace_id).await.unwrap(); - assert_eq!(profiles.len(), 1); - assert_eq!(profiles[0].name, "valid"); - - let input = || CreateByokProfileInput { - workspace_id: workspace_id.clone(), - provider: "openai".to_string(), - name: "legacy".to_string(), - description: None, - credential: "replacement-key".to_string(), - definition: ByokProfileDefinitionInput { - endpoint: ByokEndpointInput { - kind: "provider_default".to_string(), - url: None, - dialect: None, - }, - models: vec![ByokModelDeclarationInput { - model_id: "gpt-4o-mini".to_string(), - enabled: true, - capabilities: vec![ByokCapabilityInput { - input: vec!["text".to_string()], - output: vec!["text".to_string()], - features: vec![], - attachment_kinds: vec![], - attachment_sources: vec![], - }], - }], - }, - enabled: true, - actor_user_id: "user-1".to_string(), - }; - let policy = ByokPolicy::from(Deployment::Cloud, &CopilotByokRuntimeConfig::default()); - create(&pool, &[7; 32], &policy, input()).await.unwrap(); - let profiles = list(&pool, &workspace_id).await.unwrap(); - assert_eq!(profiles.len(), 2); - assert!(profiles.iter().any(|profile| profile.name == "legacy")); - assert!(create(&pool, &[7; 32], &policy, input()).await.is_err()); - - sqlx::query("DELETE FROM ai_workspace_byok_configs WHERE workspace_id = $1") - .bind(&workspace_id) - .execute(&pool) - .await - .unwrap(); - sqlx::query("DELETE FROM workspaces WHERE id = $1") - .bind(&workspace_id) - .execute(&pool) - .await - .unwrap(); - } -} diff --git a/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs b/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs index bb3c06b42d..b63a115e15 100644 --- a/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs +++ b/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs @@ -80,16 +80,11 @@ async fn load_server_profiles( .map_err(|error| RuntimeError::database("load authorized BYOK profiles failed", error))?; rows .into_iter() - .filter_map(|row| { - // Rows written by the previous release while it shares the database - // carry only the database-default definition; skip them until that - // release is retired instead of failing the whole profile load. - let definition = match serde_json::from_value::(row.definition) { - Ok(definition) => definition, - Err(_) => return None, - }; + .map(|row| { + let definition = serde_json::from_value::(row.definition) + .map_err(|error| RuntimeError::json("invalid stored BYOK definition", error))?; if !policy.allows(&row.provider, &definition.endpoint) { - return None; + return Ok(None); } let aad = server_aad( &row.workspace_id, @@ -97,7 +92,7 @@ async fn load_server_profiles( &row.provider, definition.endpoint_identity(), ); - Some(Ok(authorized_byok_profile( + Ok(Some(authorized_byok_profile( row.id, ProfileSource::Server, row.provider, @@ -110,7 +105,8 @@ async fn load_server_profiles( }, ))) }) - .collect() + .collect::>>() + .map(|profiles| profiles.into_iter().flatten().collect()) } async fn load_local_profiles( diff --git a/packages/backend/native/src/runtime/backend_runtime/embedding/index.rs b/packages/backend/native/src/runtime/backend_runtime/embedding/index.rs index 1da8e5ba0f..c0a058b9fc 100644 --- a/packages/backend/native/src/runtime/backend_runtime/embedding/index.rs +++ b/packages/backend/native/src/runtime/backend_runtime/embedding/index.rs @@ -160,7 +160,7 @@ mod tests { .unwrap(); assert_eq!(first.active_index_id, repeated.active_index_id); assert_eq!(first.index_epoch, repeated.index_epoch); - let failed_probe = super::super::store::claim_index_probe(&pool, "probe-a") + let failed_probe = super::super::store::claim_index_probe_for_workspace(&pool, "probe-a", &workspace_id) .await .unwrap() .unwrap(); @@ -178,7 +178,7 @@ mod tests { .execute(&pool) .await .unwrap(); - let recovered_probe = super::super::store::claim_index_probe(&pool, "probe-b") + let recovered_probe = super::super::store::claim_index_probe_for_workspace(&pool, "probe-b", &workspace_id) .await .unwrap() .unwrap(); diff --git a/packages/backend/native/src/runtime/backend_runtime/embedding/mod.rs b/packages/backend/native/src/runtime/backend_runtime/embedding/mod.rs index fe2ca3c8c1..81742447a5 100644 --- a/packages/backend/native/src/runtime/backend_runtime/embedding/mod.rs +++ b/packages/backend/native/src/runtime/backend_runtime/embedding/mod.rs @@ -12,7 +12,7 @@ use std::{ }; use sqlx::PgPool; -use tokio::sync::{Mutex, Notify}; +use tokio::sync::Notify; pub(super) use types::EmbeddingTarget; use types::*; @@ -36,10 +36,31 @@ pub(super) struct EmbeddingService { object_storage: RwLock>, provider: BackgroundEmbeddingProvider, wake: Notify, - worker: Mutex>, candidate_cancellations: StdMutex>>>, } +pub(super) struct EmbeddingWorker { + handle: Option, +} + +impl EmbeddingWorker { + pub(super) fn start(service: Arc) -> Self { + Self { + handle: Some(worker::start(service)), + } + } + + pub(super) async fn stop(mut self) { + if let Some(handle) = self.handle.take() { + handle.stop().await; + } + } + + pub(super) fn is_running(&self) -> bool { + self.handle.is_some() + } +} + impl EmbeddingService { pub(super) fn new( pool: PgPool, @@ -51,28 +72,10 @@ impl EmbeddingService { object_storage: RwLock::new(object_storage), provider, wake: Notify::new(), - worker: Mutex::new(None), candidate_cancellations: StdMutex::new(HashMap::new()), }) } - pub(super) async fn start(self: &Arc) { - let mut worker = self.worker.lock().await; - if worker.is_none() { - *worker = Some(worker::start(Arc::clone(self))); - } - } - - pub(super) async fn stop(&self) { - if let Some(worker) = self.worker.lock().await.take() { - worker.stop().await; - } - } - - pub(super) async fn is_running(&self) -> bool { - self.worker.lock().await.is_some() - } - fn wake(&self) { self.wake.notify_one(); } @@ -237,6 +240,13 @@ pub(in crate::runtime::backend_runtime) async fn register_artifact_source( pool: &PgPool, artifact: &crate::runtime::types::RuntimeWorkspaceArtifact, ) -> RuntimeResult<()> { + let schema_ready: bool = sqlx::query_scalar("SELECT to_regclass('embedding_sources') IS NOT NULL") + .fetch_one(pool) + .await + .map_err(|error| RuntimeError::database("Embedding source schema health check failed", error))?; + if !schema_ready { + return Ok(()); + } uuid::Uuid::parse_str(&artifact.id).map_err(|_| RuntimeError::invalid_input("artifact_id_invalid"))?; source::register_artifact(pool, artifact).await } diff --git a/packages/backend/native/src/runtime/backend_runtime/embedding/store.rs b/packages/backend/native/src/runtime/backend_runtime/embedding/store.rs index b466f49bd2..60331d43a4 100644 --- a/packages/backend/native/src/runtime/backend_runtime/embedding/store.rs +++ b/packages/backend/native/src/runtime/backend_runtime/embedding/store.rs @@ -106,6 +106,34 @@ pub(super) async fn claim_index_probe(pool: &PgPool, owner: &str) -> RuntimeResu .map_err(|error| RuntimeError::database("claim embedding index probe failed", error)) } +#[cfg(test)] +pub(super) async fn claim_index_probe_for_workspace( + pool: &PgPool, + owner: &str, + workspace_id: &str, +) -> RuntimeResult> { + sqlx::query_as( + r#"WITH candidate AS( + SELECT index_fact.id FROM embedding_indexes index_fact + JOIN embedding_workspace_states state ON state.active_index_id=index_fact.id + WHERE state.workspace_id=$2 AND state.runtime_state='active' AND( + index_fact.health_status='pending' + OR index_fact.health_status='retry_wait' AND index_fact.next_probe_at<=clock_timestamp() + OR index_fact.probe_lease_until<=clock_timestamp()) + ORDER BY index_fact.next_probe_at NULLS FIRST,index_fact.updated_at + FOR UPDATE OF index_fact SKIP LOCKED LIMIT 1 + ) UPDATE embedding_indexes index_fact SET probe_lease_owner=$1, + probe_lease_until=clock_timestamp()+interval '2 minutes',updated_at=now() + FROM candidate WHERE index_fact.id=candidate.id + RETURNING index_fact.id,index_fact.workspace_id,index_fact.fingerprint,index_fact.probe_lease_owner"#, + ) + .bind(owner) + .bind(workspace_id) + .fetch_optional(pool) + .await + .map_err(|error| RuntimeError::database("claim embedding index probe for workspace failed", error)) +} + pub(super) async fn complete_index_probe(pool: &PgPool, claim: &IndexProbeClaim) -> RuntimeResult<()> { sqlx::query( "UPDATE embedding_indexes SET \ diff --git a/packages/backend/native/src/runtime/backend_runtime/mod.rs b/packages/backend/native/src/runtime/backend_runtime/mod.rs index 2f350c08e0..5b7ad24167 100644 --- a/packages/backend/native/src/runtime/backend_runtime/mod.rs +++ b/packages/backend/native/src/runtime/backend_runtime/mod.rs @@ -8,9 +8,12 @@ mod doc_storage; mod embedding; mod gate; mod housekeeping; +mod permission; +mod role; mod rolling_quota; mod runtime_state; mod scope_compiler; +mod search; #[cfg(test)] mod tests; mod workspace_stats; @@ -23,16 +26,21 @@ use byok::LocalLeasePayload; use copilot::{backend_provider, executable_protocol}; use embedding::register_artifact_source; use napi::{Result, bindgen_prelude::Buffer}; +use search::SearchRuntime; use sha2::{Digest, Sha256}; use sqlx::{PgPool, Row, postgres::PgPoolOptions}; use tokio::sync::Mutex; -use self::types::{BackendRuntimeHealth, EmbeddingHealth}; +use self::{ + role::ServerRole, + search::{RuntimeAggregateRequest, RuntimeSearchRequest}, + types::{BackendRuntimeHealth, EmbeddingHealth, SearchOperationOutput}, +}; use super::object_storage::ObjectStorageService; pub(crate) use super::types; pub(super) use super::{ BackendRuntimeConfig, ConfigSource, InviteQuotaConfig, RuntimeError, RuntimeResult, - migrations::{migrate_embedding_tables, migrate_runtime_tables}, + migrations::{embedding_schema_health, migrate_all_tables}, napi_error, to_napi_error, }; use crate::llm::{ @@ -45,15 +53,45 @@ pub(super) fn token_hash(token: &str) -> String { hex::encode(Sha256::digest(token.as_bytes())) } +fn search_operation_output(result: RuntimeResult) -> SearchOperationOutput { + match result { + Ok(value) => SearchOperationOutput { + ok: true, + value: Some(value), + error_code: None, + }, + Err(error) => SearchOperationOutput { + ok: false, + value: None, + error_code: Some( + match error { + RuntimeError::SearchWorkspaceDenied => "workspace_denied", + RuntimeError::SearchPermissionUnavailable => "permission_unavailable", + RuntimeError::SearchProviderUnavailable => "provider_unavailable", + RuntimeError::SearchUnsupportedQuery => "unsupported_query", + RuntimeError::SearchReplayGap => "provider_unavailable", + RuntimeError::InvalidInput(_) | RuntimeError::Json { .. } => "invalid_request", + _ => "internal", + } + .to_string(), + ), + }, + } +} + #[napi_derive::napi] pub struct BackendRuntime { config_source: ConfigSource, + role: ServerRole, + script_mode: bool, config: Arc>>, config_reload: Mutex<()>, pool: Mutex>, embedding_health: RwLock, object_storage: RwLock>, embedding: Mutex>>, + embedding_worker: Mutex>, + search: Mutex>>, managed_token_providers: Arc, } @@ -62,16 +100,21 @@ impl BackendRuntime { #[napi(constructor)] pub fn new(private_key: Option, config_paths: Option>) -> Result { let config_source = ConfigSource::new(config_paths); + let (role, script_mode) = ServerRole::from_environment().map_err(napi_error)?; let config = BackendRuntimeConfig::from_config_source(private_key, &config_source).map_err(to_napi_error)?; let object_storage = ObjectStorageService::from_config_source(&config_source).map_err(to_napi_error)?; Ok(Self { config_source, + role, + script_mode, config: Arc::new(RwLock::new(Arc::new(config))), config_reload: Mutex::new(()), pool: Mutex::new(None), embedding_health: RwLock::new(EmbeddingHealth::disabled("runtime_not_started", None)), object_storage: RwLock::new(Arc::new(object_storage)), embedding: Mutex::new(None), + embedding_worker: Mutex::new(None), + search: Mutex::new(None), managed_token_providers: Arc::new(Default::default()), }) } @@ -109,7 +152,30 @@ impl BackendRuntime { .write() .map_err(|_| RuntimeError::invalid_state("object storage service lock poisoned"))? = Arc::new(object_storage); - let mut embedding_health = migrate_embedding_tables(&pool).await; + let embedding_health = if self.script_mode { + EmbeddingHealth::disabled("script_runtime", None) + } else { + let config = self.config()?; + if config.search.enabled { + if config.search.provider == "embedded" && !self.role.allows_embedded_search() { + return Err(RuntimeError::config(format!( + "embedded search is only available for the allinone role (current role: {})", + self.role.as_str() + ))); + } + let search = Arc::new(SearchRuntime::new(pool.clone(), config.search.clone())?); + if self.role.owns_background() { + search.initialize().await?; + } + *self.search.lock().await = Some(search); + } else { + *self.search.lock().await = None; + } + embedding_schema_health(&pool).await? + }; + if self.script_mode { + *self.search.lock().await = None; + } if embedding_health.enabled { let provider = copilot::BackgroundEmbeddingProvider::new( pool.clone(), @@ -117,18 +183,30 @@ impl BackendRuntime { Arc::clone(&self.managed_token_providers), ); let embedding = embedding::EmbeddingService::new(pool.clone(), self.object_storage()?, provider); - if std::env::var("NODE_ENV").as_deref() != Ok("test") - || std::env::var("AFFINE_EMBEDDING_WORKER").as_deref() == Ok("1") + if self.role.owns_background() + && (std::env::var("NODE_ENV").as_deref() != Ok("test") + || std::env::var("AFFINE_EMBEDDING_WORKER").as_deref() == Ok("1")) { - embedding.start().await; + *self.embedding_worker.lock().await = Some(embedding::EmbeddingWorker::start(Arc::clone(&embedding))); } - embedding_health.worker_running = embedding.is_running().await; + let mut embedding_health = embedding_health; + embedding_health.worker_running = self + .embedding_worker + .lock() + .await + .as_ref() + .is_some_and(embedding::EmbeddingWorker::is_running); *self.embedding.lock().await = Some(embedding); + *self + .embedding_health + .write() + .map_err(|_| RuntimeError::invalid_state("embedding health lock poisoned"))? = embedding_health; + } else { + *self + .embedding_health + .write() + .map_err(|_| RuntimeError::invalid_state("embedding health lock poisoned"))? = embedding_health; } - *self - .embedding_health - .write() - .map_err(|_| RuntimeError::invalid_state("embedding health lock poisoned"))? = embedding_health; *guard = Some(pool); Ok(()) @@ -136,9 +214,11 @@ impl BackendRuntime { #[napi] pub async fn stop(&self) -> Result<()> { - if let Some(embedding) = self.embedding.lock().await.take() { - embedding.stop().await; + self.search.lock().await.take(); + if let Some(worker) = self.embedding_worker.lock().await.take() { + worker.stop().await; } + self.embedding.lock().await.take(); let pool = self.pool.lock().await.take(); if let Some(pool) = pool { pool.close().await; @@ -168,6 +248,26 @@ impl BackendRuntime { .await .map_err(to_napi_error)?; self.update_config(config).map_err(to_napi_error)?; + if !self.script_mode { + let config = self.config().map_err(to_napi_error)?; + if config.search.enabled { + if config.search.provider == "embedded" && !self.role.allows_embedded_search() { + return Err(napi_error(format!( + "embedded search is only available for the allinone role (current role: {})", + self.role.as_str() + ))); + } + let search = Arc::new(SearchRuntime::new(pool.clone(), config.search.clone()).map_err(to_napi_error)?); + if self.role.owns_background() { + search.initialize().await.map_err(to_napi_error)?; + } + *self.search.lock().await = Some(search); + } else { + *self.search.lock().await = None; + } + } else { + *self.search.lock().await = None; + } let object_storage = Arc::new(object_storage); *self .object_storage @@ -176,17 +276,19 @@ impl BackendRuntime { if let Some(embedding) = self.embedding.lock().await.as_ref() { embedding.reload_object_storage(object_storage).map_err(to_napi_error)?; } - let workspace_ids = sqlx::query_scalar::<_, String>("SELECT id FROM workspaces") - .fetch_all(&pool) - .await - .map_err(|error| { - to_napi_error(RuntimeError::database( - "load workspaces for embedding reconciliation failed", - error, - )) - })?; - for workspace_id in workspace_ids { - self.reconcile_embedding_workspace(&workspace_id).await?; + if self.role.owns_background() { + let workspace_ids = sqlx::query_scalar::<_, String>("SELECT id FROM workspaces") + .fetch_all(&pool) + .await + .map_err(|error| { + to_napi_error(RuntimeError::database( + "load workspaces for embedding reconciliation failed", + error, + )) + })?; + for workspace_id in workspace_ids { + self.reconcile_embedding_workspace(&workspace_id).await?; + } } Ok(()) } @@ -217,7 +319,106 @@ impl BackendRuntime { #[napi] pub async fn run_migrations(&self) -> Result<()> { let pool = self.pool().await?; - migrate_runtime_tables(&pool).await.map_err(to_napi_error) + let embedding_health = migrate_all_tables(&pool).await.map_err(to_napi_error)?; + *self + .embedding_health + .write() + .map_err(|_| napi_error("embedding health lock poisoned"))? = embedding_health; + Ok(()) + } + + #[napi] + pub async fn search_authorized( + &self, + actor_user_id: String, + workspace_id: String, + request: RuntimeSearchRequest, + ) -> Result { + let result = self + .search_runtime() + .await? + .search_authorized(&actor_user_id, &workspace_id, request) + .await; + Ok(search_operation_output(result)) + } + + #[napi] + pub async fn aggregate_authorized( + &self, + actor_user_id: String, + workspace_id: String, + request: RuntimeAggregateRequest, + ) -> Result { + let result = self + .search_runtime() + .await? + .aggregate_authorized(&actor_user_id, &workspace_id, request) + .await; + Ok(search_operation_output(result)) + } + + #[napi] + pub async fn index_search_document(&self, workspace_id: String, doc_id: String) -> Result<()> { + let search = self.search_runtime().await?; + let result = if self.role.owns_background() { + search.index_document(&workspace_id, &doc_id).await + } else { + search.project_document_only(&workspace_id, &doc_id).await + }; + result.map_err(to_napi_error) + } + + #[napi] + pub async fn delete_search_document(&self, workspace_id: String, doc_id: String) -> Result<()> { + let search = self.search_runtime().await?; + let result = if self.role.owns_background() { + search.delete_document(&workspace_id, &doc_id).await + } else { + search.delete_document_only(&workspace_id, &doc_id).await + }; + result.map_err(to_napi_error) + } + + #[napi] + pub async fn reconcile_search_workspace(&self, workspace_id: String) -> Result<()> { + self.require_background()?; + self + .search_runtime() + .await? + .reconcile_workspace(permission::SystemSearchCapability::ReconcileIndex, &workspace_id) + .await + .map_err(to_napi_error) + } + + #[napi] + pub async fn delete_search_workspace(&self, workspace_id: String) -> Result<()> { + self.require_background()?; + self + .search_runtime() + .await? + .delete_workspace(&workspace_id) + .await + .map_err(to_napi_error) + } + + #[napi] + pub async fn filter_readable_docs( + &self, + actor_user_id: String, + workspace_id: String, + doc_ids: Vec, + ) -> Result> { + let authorizer = permission::PermissionAuthorizer::new(self.pool().await?); + authorizer + .filter_readable_docs(&workspace_id, &actor_user_id, doc_ids) + .await + .map(|ids| ids.into_iter().collect()) + .map_err(to_napi_error) + } + + #[napi] + pub async fn search_status(&self) -> Result { + self.search_runtime().await?.status().await.map_err(to_napi_error) } #[napi] @@ -370,6 +571,7 @@ impl BackendRuntime { #[napi] pub async fn reconcile_embedding_workspaces(&self) -> Result { + self.require_background()?; let workspace_ids = sqlx::query_scalar::<_, String>("SELECT id FROM workspaces") .fetch_all(&self.pool().await?) .await @@ -405,6 +607,7 @@ impl BackendRuntime { #[napi] pub async fn cleanup_unreferenced_artifacts(&self, limit: i64) -> Result { + self.require_background()?; if limit <= 0 { return Err(napi_error("artifact cleanup limit must be positive")); } @@ -572,6 +775,27 @@ impl BackendRuntime { .ok_or_else(|| RuntimeError::invalid_state("BackendRuntime must be started before using postgres operations")) } + fn require_background(&self) -> Result<()> { + if self.role.owns_background() { + Ok(()) + } else { + Err(napi_error(format!( + "backend runtime role {} does not own background work", + self.role.as_str() + ))) + } + } + + async fn search_runtime(&self) -> Result> { + self + .search + .lock() + .await + .as_ref() + .cloned() + .ok_or_else(|| napi_error("search_provider_not_ready")) + } + async fn reconcile_embedding_workspace(&self, workspace_id: &str) -> Result<()> { let enabled = sqlx::query_scalar::<_, bool>("SELECT enable_doc_embedding FROM workspaces WHERE id=$1") .bind(workspace_id) diff --git a/packages/backend/native/src/runtime/backend_runtime/permission/authorizer.rs b/packages/backend/native/src/runtime/backend_runtime/permission/authorizer.rs new file mode 100644 index 0000000000..93d972f113 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/permission/authorizer.rs @@ -0,0 +1,90 @@ +use std::collections::BTreeSet; + +use sqlx::PgPool; + +use super::{ + store::PermissionStore, + types::{AclPredicate, AuthorizedSearchScope, DocAclCapability, DocReadScope, SearchActor}, +}; +use crate::{ + permission::evaluate_permission, + runtime::{RuntimeError, RuntimeResult}, +}; + +pub(in crate::runtime::backend_runtime) struct PermissionAuthorizer { + store: PermissionStore, +} + +impl PermissionAuthorizer { + pub(in crate::runtime::backend_runtime) fn new(pool: PgPool) -> Self { + Self { + store: PermissionStore::new(pool), + } + } + + pub(in crate::runtime::backend_runtime) async fn authorize_search( + &self, + actor: &SearchActor, + workspace_id: &str, + ) -> RuntimeResult { + match actor { + SearchActor::User { user_id } => { + let snapshot = self.store.search_snapshot(workspace_id, user_id).await?; + let owner_or_admin = matches!(snapshot.evaluation.workspace.role.as_deref(), Some("owner" | "admin")) + && snapshot.evaluation.workspace.member_state.as_deref() == Some("active"); + let decision = evaluate_permission(snapshot.evaluation) + .map_err(|_| RuntimeError::SearchPermissionUnavailable)? + .workspace + .decisions + .into_iter() + .find(|decision| decision.action == "Workspace.Read") + .ok_or(RuntimeError::SearchPermissionUnavailable)?; + if !decision.allowed { + return Err(RuntimeError::SearchWorkspaceDenied); + } + let docs = match snapshot.capability { + DocAclCapability::Disabled => DocReadScope::All, + DocAclCapability::Unknown => return Err(RuntimeError::SearchPermissionUnavailable), + DocAclCapability::Enabled if owner_or_admin => DocReadScope::All, + DocAclCapability::Enabled => DocReadScope::ProjectedAcl(AclPredicate { + actor_user_id: snapshot.actor_user_id, + active_member: snapshot.active_member, + sharing_enabled: snapshot.sharing_enabled, + }), + }; + Ok(AuthorizedSearchScope { + workspace_id: workspace_id.to_string(), + permission_revision: snapshot.revision, + docs, + }) + } + } + } + + pub(in crate::runtime::backend_runtime) async fn revision(&self, workspace_id: &str) -> RuntimeResult { + self.store.revision(workspace_id).await + } + + pub(in crate::runtime::backend_runtime) async fn filter_readable_docs( + &self, + workspace_id: &str, + user_id: &str, + doc_ids: Vec, + ) -> RuntimeResult> { + let snapshot = self.store.permission_snapshot(workspace_id, user_id, &doc_ids).await?; + let output = evaluate_permission(snapshot.evaluation).map_err(|_| RuntimeError::SearchPermissionUnavailable)?; + Ok( + output + .docs + .into_iter() + .filter(|doc| { + doc + .decisions + .iter() + .any(|decision| decision.action == "Doc.Read" && decision.allowed) + }) + .map(|doc| doc.doc_id) + .collect(), + ) + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/permission/mod.rs b/packages/backend/native/src/runtime/backend_runtime/permission/mod.rs new file mode 100644 index 0000000000..6441685822 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/permission/mod.rs @@ -0,0 +1,11 @@ +mod authorizer; +mod store; +mod types; + +pub(super) use authorizer::PermissionAuthorizer; +#[cfg(test)] +pub(super) use types::AclPredicate; +pub(super) use types::{AuthorizedSearchScope, DocReadScope, SearchActor, SystemSearchCapability}; + +#[cfg(test)] +mod tests; diff --git a/packages/backend/native/src/runtime/backend_runtime/permission/store.rs b/packages/backend/native/src/runtime/backend_runtime/permission/store.rs new file mode 100644 index 0000000000..5496cde9cd --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/permission/store.rs @@ -0,0 +1,225 @@ +use chrono::{DateTime, Utc}; +use sqlx::{PgPool, Row}; + +use super::types::{DocAclCapability, PermissionSnapshot}; +use crate::{ + entitlement::entitlement_priority, + permission::{ + PermissionDocInputV1, PermissionEvaluationInputV1, PermissionRuntimeInputV1, PermissionSubjectInputV1, + PermissionWorkspaceInputV1, + }, + runtime::{RuntimeError, RuntimeResult}, +}; + +pub(super) struct PermissionStore { + pool: PgPool, +} + +impl PermissionStore { + pub(super) fn new(pool: PgPool) -> Self { + Self { pool } + } + + pub(super) async fn search_snapshot(&self, workspace_id: &str, user_id: &str) -> RuntimeResult { + self.permission_snapshot(workspace_id, user_id, &[]).await + } + + pub(super) async fn permission_snapshot( + &self, + workspace_id: &str, + user_id: &str, + doc_ids: &[String], + ) -> RuntimeResult { + let mut transaction = self + .pool + .begin() + .await + .map_err(|error| RuntimeError::database("begin permission snapshot", error))?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("configure permission snapshot", error))?; + let row = sqlx::query( + r#"SELECT revision.revision, + policy.visibility, coalesce(policy.sharing_enabled, true) AS sharing_enabled, + coalesce(policy.member_default_doc_role, 'manager') AS member_default_doc_role, + member.role, member.state + FROM workspaces workspace + LEFT JOIN workspace_permission_revisions revision ON revision.workspace_id=workspace.id + LEFT JOIN workspace_access_policies policy ON policy.workspace_id=workspace.id + LEFT JOIN LATERAL ( + SELECT role,state FROM workspace_members + WHERE workspace_id=workspace.id AND user_id=$2 + ORDER BY (state='active') DESC, updated_at DESC LIMIT 1 + ) member ON true + WHERE workspace.id=$1"#, + ) + .bind(workspace_id) + .bind(user_id) + .fetch_optional(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("load workspace permission facts", error))? + .ok_or_else(|| RuntimeError::invalid_input("workspace_not_found"))?; + let revision = row + .try_get::, _>("revision") + .map_err(|error| RuntimeError::database("decode permission revision", error))? + .ok_or_else(|| RuntimeError::invalid_state("permission_state_unavailable"))?; + let sharing_enabled: bool = row + .try_get("sharing_enabled") + .map_err(|error| RuntimeError::database("decode workspace sharing", error))?; + let role: Option = row + .try_get("role") + .map_err(|error| RuntimeError::database("decode workspace member role", error))?; + let member_state: Option = row + .try_get("state") + .map_err(|error| RuntimeError::database("decode workspace member state", error))?; + let visibility: Option = row + .try_get("visibility") + .map_err(|error| RuntimeError::database("decode workspace visibility", error))?; + let member_default_doc_role: String = row + .try_get("member_default_doc_role") + .map_err(|error| RuntimeError::database("decode member default doc role", error))?; + let capability = load_doc_acl_capability(&mut transaction, workspace_id).await?; + let docs = if doc_ids.is_empty() { + Vec::new() + } else { + sqlx::query( + r#"SELECT candidate.doc_id, policy.visibility, policy.public_role, + coalesce(policy.member_default_role, $3) AS member_default_role, + grant_fact.role AS explicit_user_role + FROM unnest($4::text[]) candidate(doc_id) + LEFT JOIN doc_access_policies policy + ON policy.workspace_id=$1 AND policy.doc_id=candidate.doc_id + LEFT JOIN doc_grants grant_fact + ON grant_fact.workspace_id=$1 AND grant_fact.doc_id=candidate.doc_id + AND grant_fact.principal_type='user' AND grant_fact.principal_id=$2"#, + ) + .bind(workspace_id) + .bind(user_id) + .bind(&member_default_doc_role) + .bind(doc_ids) + .fetch_all(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("load document permission facts", error))? + .into_iter() + .map(|row| { + Ok(PermissionDocInputV1 { + doc_id: row + .try_get("doc_id") + .map_err(|error| RuntimeError::database("decode permission doc id", error))?, + actions: vec!["Doc.Read".to_string()], + explicit_user_role: row + .try_get("explicit_user_role") + .map_err(|error| RuntimeError::database("decode explicit doc role", error))?, + member_default_role: row + .try_get("member_default_role") + .map_err(|error| RuntimeError::database("decode member default role", error))?, + public_role: row + .try_get("public_role") + .map_err(|error| RuntimeError::database("decode public doc role", error))?, + visibility: row + .try_get("visibility") + .map_err(|error| RuntimeError::database("decode doc visibility", error))?, + sharing_enabled: Some(sharing_enabled), + ..Default::default() + }) + }) + .collect::>>()? + }; + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit permission snapshot", error))?; + let active_member = + member_state.as_deref() == Some("active") && matches!(role.as_deref(), Some("member" | "admin" | "owner")); + + Ok(PermissionSnapshot { + revision, + capability, + evaluation: PermissionEvaluationInputV1 { + version: 1, + legacy_compat_mode: false, + subject: PermissionSubjectInputV1 { + user_id: Some(user_id.to_string()), + ..Default::default() + }, + runtime: PermissionRuntimeInputV1 { + known: true, + sharing_enabled: Some(sharing_enabled), + ..Default::default() + }, + workspace: PermissionWorkspaceInputV1 { + role, + member_state, + public: visibility.as_deref() == Some("public"), + sharing_enabled: Some(sharing_enabled), + ..Default::default() + }, + workspace_actions: vec!["Workspace.Read".to_string()], + docs, + }, + actor_user_id: user_id.to_string(), + active_member, + sharing_enabled, + }) + } + + pub(super) async fn revision(&self, workspace_id: &str) -> RuntimeResult { + sqlx::query_scalar("SELECT revision FROM workspace_permission_revisions WHERE workspace_id=$1") + .bind(workspace_id) + .fetch_optional(&self.pool) + .await + .map_err(|error| RuntimeError::database("read permission revision", error))? + .ok_or_else(|| RuntimeError::invalid_state("permission_state_unavailable")) + } +} + +async fn load_doc_acl_capability( + transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, + workspace_id: &str, +) -> RuntimeResult { + let rows = sqlx::query( + r#"SELECT plan,status,expires_at,grace_until,validated_at,source,signed_payload + FROM entitlements + WHERE target_type='workspace' AND target_id=$1 + AND ((status='active' AND (expires_at IS NULL OR expires_at>now())) + OR (status='grace' AND grace_until>now()))"#, + ) + .bind(workspace_id) + .fetch_all(&mut **transaction) + .await + .map_err(|error| RuntimeError::database("load workspace entitlement facts", error))?; + let mut best: Option<(i32, String)> = None; + for row in rows { + let plan: String = row + .try_get("plan") + .map_err(|error| RuntimeError::database("decode entitlement plan", error))?; + let status: String = row + .try_get("status") + .map_err(|error| RuntimeError::database("decode entitlement status", error))?; + let source: String = row + .try_get("source") + .map_err(|error| RuntimeError::database("decode entitlement source", error))?; + let validated_at: Option> = row + .try_get("validated_at") + .map_err(|error| RuntimeError::database("decode entitlement validation", error))?; + let signed_payload: Option> = row + .try_get("signed_payload") + .map_err(|error| RuntimeError::database("decode entitlement payload", error))?; + if source == "selfhost_license" && (validated_at.is_none() || signed_payload.is_none()) { + continue; + } + let priority = entitlement_priority(&status, &plan); + if best.as_ref().is_none_or(|(current, _)| priority > *current) { + best = Some((priority, plan)); + } + } + match best.map(|(_, plan)| plan) { + Some(plan) if matches!(plan.as_str(), "team" | "selfhost_team") => Ok(DocAclCapability::Enabled), + Some(plan) if matches!(plan.as_str(), "free" | "pro" | "lifetime_pro" | "ai" | "selfhost_free") => { + Ok(DocAclCapability::Disabled) + } + Some(_) => Ok(DocAclCapability::Unknown), + None => Ok(DocAclCapability::Disabled), + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/permission/tests.rs b/packages/backend/native/src/runtime/backend_runtime/permission/tests.rs new file mode 100644 index 0000000000..b0424cabf8 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/permission/tests.rs @@ -0,0 +1,165 @@ +use sqlx::PgPool; + +use super::{DocReadScope, PermissionAuthorizer, SearchActor}; + +static PERMISSION_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +async fn setup() -> Option<(PgPool, String, String)> { + let database_url = std::env::var("DATABASE_URL").ok()?; + let pool = PgPool::connect(&database_url).await.unwrap(); + crate::runtime::migrations::migrate_search_tables(&pool).await.unwrap(); + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let user_id = format!("search-permission-user-{suffix}"); + let workspace_id = format!("search-permission-workspace-{suffix}"); + sqlx::query( + "INSERT INTO users(id,name,email,registered,email_verified,disabled) VALUES($1,'Search Permission \ + User',$2,true,now(),false)", + ) + .bind(&user_id) + .bind(format!("search-permission-{suffix}@example.com")) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO workspaces(id) VALUES($1)") + .bind(&workspace_id) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO workspace_access_policies(workspace_id) VALUES($1)") + .bind(&workspace_id) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO workspace_members(workspace_id,user_id,role,state) VALUES($1,$2,'member','active')") + .bind(&workspace_id) + .bind(&user_id) + .execute(&pool) + .await + .unwrap(); + Some((pool, workspace_id, user_id)) +} + +#[tokio::test] +async fn non_team_is_all_and_team_uses_projected_acl() { + let _guard = PERMISSION_TEST_LOCK.lock().await; + let Some((pool, workspace_id, user_id)) = setup().await else { + return; + }; + let authorizer = PermissionAuthorizer::new(pool.clone()); + let actor = SearchActor::User { + user_id: user_id.clone(), + }; + let free = authorizer.authorize_search(&actor, &workspace_id).await.unwrap(); + assert_eq!(free.docs, DocReadScope::All); + + sqlx::query( + "INSERT INTO entitlements(id,target_type,target_id,source,plan,status,validated_at) \ + VALUES($1,'workspace',$2,'admin_grant','team','active',now())", + ) + .bind(format!("search-permission-entitlement-{workspace_id}")) + .bind(&workspace_id) + .execute(&pool) + .await + .unwrap(); + let team = authorizer.authorize_search(&actor, &workspace_id).await.unwrap(); + let DocReadScope::ProjectedAcl(predicate) = team.docs else { + panic!("team member must use projected ACL"); + }; + assert_eq!(predicate.actor_user_id, user_id); + assert!(predicate.active_member); + assert!(team.permission_revision > free.permission_revision); +} + +#[tokio::test] +async fn inactive_member_is_denied_and_unknown_capability_fails_closed() { + let _guard = PERMISSION_TEST_LOCK.lock().await; + let Some((pool, workspace_id, user_id)) = setup().await else { + return; + }; + let authorizer = PermissionAuthorizer::new(pool.clone()); + let actor = SearchActor::User { + user_id: user_id.clone(), + }; + sqlx::query("UPDATE workspace_members SET state='suspended' WHERE workspace_id=$1 AND user_id=$2") + .bind(&workspace_id) + .bind(&user_id) + .execute(&pool) + .await + .unwrap(); + let error = authorizer.authorize_search(&actor, &workspace_id).await.unwrap_err(); + assert!(matches!(error, crate::runtime::RuntimeError::SearchWorkspaceDenied)); + + sqlx::query("UPDATE workspace_members SET state='active' WHERE workspace_id=$1 AND user_id=$2") + .bind(&workspace_id) + .bind(&user_id) + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO entitlements(id,target_type,target_id,source,plan,status,validated_at) \ + VALUES($1,'workspace',$2,'admin_grant','future_plan','active',now())", + ) + .bind(format!("search-permission-entitlement-{workspace_id}")) + .bind(&workspace_id) + .execute(&pool) + .await + .unwrap(); + let error = authorizer.authorize_search(&actor, &workspace_id).await.unwrap_err(); + assert!(matches!( + error, + crate::runtime::RuntimeError::SearchPermissionUnavailable + )); +} + +#[tokio::test] +async fn fact_changes_advance_revision_and_write_ordered_change() { + let _guard = PERMISSION_TEST_LOCK.lock().await; + let Some((pool, workspace_id, user_id)) = setup().await else { + return; + }; + let authorizer = PermissionAuthorizer::new(pool.clone()); + let before = authorizer.revision(&workspace_id).await.unwrap(); + sqlx::query( + "INSERT INTO doc_grants(workspace_id,doc_id,principal_type,principal_id,role) VALUES($1,'doc','user',$2,'reader')", + ) + .bind(&workspace_id) + .bind(&user_id) + .execute(&pool) + .await + .unwrap(); + let after = authorizer.revision(&workspace_id).await.unwrap(); + assert_eq!(after, before + 1); + let change: (i64, Option, String) = sqlx::query_as( + "SELECT revision,doc_id,scope FROM workspace_permission_changes WHERE workspace_id=$1 AND revision=$2", + ) + .bind(&workspace_id) + .bind(after) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(change, (after, Some("doc".to_string()), "doc_grant".to_string())); + + sqlx::query("UPDATE workspace_members SET updated_at=now() WHERE workspace_id=$1 AND user_id=$2") + .bind(&workspace_id) + .bind(&user_id) + .execute(&pool) + .await + .unwrap(); + assert_eq!(authorizer.revision(&workspace_id).await.unwrap(), after); + + let moved_workspace_id = format!("{workspace_id}-moved"); + sqlx::query("INSERT INTO workspaces(id) VALUES($1)") + .bind(&moved_workspace_id) + .execute(&pool) + .await + .unwrap(); + assert_eq!(authorizer.revision(&moved_workspace_id).await.unwrap(), 0); + sqlx::query("UPDATE doc_grants SET workspace_id=$1 WHERE workspace_id=$2 AND doc_id='doc'") + .bind(&moved_workspace_id) + .bind(&workspace_id) + .execute(&pool) + .await + .unwrap(); + assert_eq!(authorizer.revision(&workspace_id).await.unwrap(), after + 1); + assert_eq!(authorizer.revision(&moved_workspace_id).await.unwrap(), 1); +} diff --git a/packages/backend/native/src/runtime/backend_runtime/permission/types.rs b/packages/backend/native/src/runtime/backend_runtime/permission/types.rs new file mode 100644 index 0000000000..d758b4b121 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/permission/types.rs @@ -0,0 +1,47 @@ +use crate::permission::PermissionEvaluationInputV1; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::runtime::backend_runtime) enum SystemSearchCapability { + ReconcileIndex, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::runtime::backend_runtime) enum SearchActor { + User { user_id: String }, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum DocAclCapability { + Enabled, + Disabled, + Unknown, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::runtime::backend_runtime) struct AclPredicate { + pub(in crate::runtime::backend_runtime) actor_user_id: String, + pub(in crate::runtime::backend_runtime) active_member: bool, + pub(in crate::runtime::backend_runtime) sharing_enabled: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::runtime::backend_runtime) enum DocReadScope { + All, + ProjectedAcl(AclPredicate), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::runtime::backend_runtime) struct AuthorizedSearchScope { + pub(in crate::runtime::backend_runtime) workspace_id: String, + pub(in crate::runtime::backend_runtime) permission_revision: i64, + pub(in crate::runtime::backend_runtime) docs: DocReadScope, +} + +pub(super) struct PermissionSnapshot { + pub(super) revision: i64, + pub(super) capability: DocAclCapability, + pub(super) evaluation: PermissionEvaluationInputV1, + pub(super) actor_user_id: String, + pub(super) active_member: bool, + pub(super) sharing_enabled: bool, +} diff --git a/packages/backend/native/src/runtime/backend_runtime/role.rs b/packages/backend/native/src/runtime/backend_runtime/role.rs new file mode 100644 index 0000000000..d94d669951 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/role.rs @@ -0,0 +1,98 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum ServerRole { + Frontend, + Api, + Worker, + AllInOne, +} + +impl ServerRole { + pub(super) fn from_environment() -> Result<(Self, bool), String> { + let script_mode = matches!(std::env::var("SERVER_FLAVOR").as_deref(), Ok("script")); + if let Ok(value) = std::env::var("AFFINE_SERVER_ROLE") { + return Self::parse(&value).map(|role| (role, script_mode)); + } + + match std::env::var("SERVER_FLAVOR") { + Err(std::env::VarError::NotPresent) => Ok((Self::AllInOne, false)), + Ok(value) => Self::from_flavor(&value), + Err(std::env::VarError::NotUnicode(_)) => Err("backend runtime role source is not valid unicode".to_string()), + } + } + + fn from_flavor(value: &str) -> Result<(Self, bool), String> { + match value { + "allinone" => Ok((Self::AllInOne, false)), + "front" => Ok((Self::Frontend, false)), + "graphql" => Ok((Self::Api, false)), + "worker" => Ok((Self::Worker, false)), + "sync" | "renderer" => Ok((Self::Frontend, false)), + // The CLI uses the BackendRuntime only for database/object-storage work. + // It is not one of the four server roles and must not initialize search. + "script" => Ok((Self::Frontend, true)), + value => Err(format!("unsupported backend runtime role source value: {value}")), + } + } + + fn parse(value: &str) -> Result { + match value { + "frontend" => Ok(Self::Frontend), + "api" => Ok(Self::Api), + "worker" => Ok(Self::Worker), + "allinone" => Ok(Self::AllInOne), + _ => Err(format!( + "unsupported backend runtime role: {value}; expected frontend, api, worker, or allinone" + )), + } + } + + pub(super) fn owns_background(self) -> bool { + matches!(self, Self::Worker | Self::AllInOne) + } + + pub(super) fn allows_embedded_search(self) -> bool { + matches!(self, Self::AllInOne) + } + + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Frontend => "frontend", + Self::Api => "api", + Self::Worker => "worker", + Self::AllInOne => "allinone", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn role_parser_is_closed() { + assert_eq!(ServerRole::parse("frontend"), Ok(ServerRole::Frontend)); + assert_eq!(ServerRole::parse("api"), Ok(ServerRole::Api)); + assert_eq!(ServerRole::parse("worker"), Ok(ServerRole::Worker)); + assert_eq!(ServerRole::parse("allinone"), Ok(ServerRole::AllInOne)); + assert!(ServerRole::parse("graphql").is_err()); + assert!(ServerRole::from_flavor("doc").is_err()); + assert_eq!(ServerRole::from_flavor("script"), Ok((ServerRole::Frontend, true))); + assert_eq!(ServerRole::from_flavor("worker"), Ok((ServerRole::Worker, false))); + } + + #[test] + fn only_all_in_one_allows_embedded_search() { + assert!(!ServerRole::Frontend.allows_embedded_search()); + assert!(!ServerRole::Api.allows_embedded_search()); + assert!(!ServerRole::Worker.allows_embedded_search()); + assert!(ServerRole::AllInOne.allows_embedded_search()); + } + + #[test] + fn only_worker_compositions_own_background() { + assert!(!ServerRole::Frontend.owns_background()); + assert!(!ServerRole::Api.owns_background()); + assert!(ServerRole::Worker.owns_background()); + assert!(ServerRole::AllInOne.owns_background()); + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/scope_compiler.rs b/packages/backend/native/src/runtime/backend_runtime/scope_compiler.rs index 29f5ac7fe3..888808fd04 100644 --- a/packages/backend/native/src/runtime/backend_runtime/scope_compiler.rs +++ b/packages/backend/native/src/runtime/backend_runtime/scope_compiler.rs @@ -6,18 +6,22 @@ use affine_doc_loader::{ use chrono::Utc; use sqlx::{PgPool, Row}; -use super::{RuntimeError, RuntimeResult, types}; +use super::{RuntimeError, RuntimeResult, permission::PermissionAuthorizer, types}; use crate::{runtime::storage_runtime::load_current_doc, userdata_acl}; const REQUIRED_DOCUMENT_LIMIT: usize = 64; pub(super) struct ScopeCompiler { pool: PgPool, + authorizer: PermissionAuthorizer, } impl ScopeCompiler { pub(super) fn new(pool: PgPool) -> Self { - Self { pool } + Self { + authorizer: PermissionAuthorizer::new(pool.clone()), + pool, + } } pub(super) async fn compile( @@ -63,10 +67,11 @@ impl ScopeCompiler { .await?; let readable = self - .readable_doc_ids( + .authorizer + .filter_readable_docs( &input.workspace_id, &input.user_id, - facts.documents.iter().map(|doc| doc.id.as_str()), + facts.documents.iter().map(|doc| doc.id.clone()).collect(), ) .await?; let mut required_docs = BTreeSet::new(); @@ -154,46 +159,6 @@ impl ScopeCompiler { Ok(()) } - async fn readable_doc_ids<'a>( - &self, - workspace_id: &str, - user_id: &str, - doc_ids: impl Iterator, - ) -> RuntimeResult> { - let doc_ids = doc_ids.map(str::to_string).collect::>(); - let rows = sqlx::query( - r#"SELECT candidate.doc_id FROM unnest($3::text[]) candidate(doc_id) - WHERE EXISTS ( - SELECT 1 FROM workspace_access_policies workspace_policy - LEFT JOIN doc_access_policies doc_policy - ON doc_policy.workspace_id=workspace_policy.workspace_id AND doc_policy.doc_id=candidate.doc_id - LEFT JOIN workspace_members member - ON member.workspace_id=workspace_policy.workspace_id AND member.user_id=$2 AND member.state='active' - LEFT JOIN doc_grants grant_fact - ON grant_fact.workspace_id=workspace_policy.workspace_id AND grant_fact.doc_id=candidate.doc_id - AND grant_fact.principal_type='user' AND grant_fact.principal_id=$2 - WHERE workspace_policy.workspace_id=$1 AND ( - member.id IS NOT NULL AND grant_fact.role=ANY(ARRAY['owner','manager','editor','commenter','reader']::text[]) - OR member.id IS NULL AND workspace_policy.sharing_enabled - AND grant_fact.role=ANY(ARRAY['owner','manager','editor','commenter','reader']::text[]) - OR member.role=ANY(ARRAY['owner','admin']::text[]) - OR member.id IS NOT NULL AND grant_fact.principal_id IS NULL - AND coalesce(doc_policy.member_default_role,workspace_policy.member_default_doc_role) - =ANY(ARRAY['owner','manager','editor','commenter','reader']::text[]) - OR workspace_policy.sharing_enabled AND doc_policy.visibility='public' - AND doc_policy.public_role=ANY(ARRAY['owner','manager','editor','commenter','reader','external']::text[]) - ) - )"#, - ) - .bind(workspace_id) - .bind(user_id) - .bind(doc_ids) - .fetch_all(&self.pool) - .await - .map_err(|error| RuntimeError::database("filter scope document permissions failed", error))?; - Ok(rows.into_iter().map(|row| row.get("doc_id")).collect()) - } - async fn artifact_is_readable(&self, workspace_id: &str, user_id: &str, artifact_id: &str) -> RuntimeResult { let id = artifact_id .parse::() @@ -285,6 +250,7 @@ mod tests { }; let _guard = crate::runtime::migrations::EMBEDDING_TEST_LOCK.lock().await; let pool = PgPool::connect(&database_url).await.unwrap(); + crate::runtime::migrations::migrate_search_tables(&pool).await.unwrap(); let suffix = uuid::Uuid::new_v4().simple().to_string(); let user_id = format!("scope-user-{suffix}"); let collaborator_id = format!("scope-collaborator-{suffix}"); diff --git a/packages/backend/native/src/runtime/backend_runtime/search/checkpoint.rs b/packages/backend/native/src/runtime/backend_runtime/search/checkpoint.rs new file mode 100644 index 0000000000..4e8863df94 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/checkpoint.rs @@ -0,0 +1,193 @@ +use napi::bindgen_prelude::Buffer; +use sha2::{Digest, Sha256}; +use sqlx::{PgPool, Row}; + +use super::{SCHEMA_FINGERPRINT, store::SearchTable}; +use crate::{ + runtime::{RuntimeError, RuntimeResult}, + search_index::EmbeddedSearchIndex, +}; + +const DIRTY_CHANGE_THRESHOLD: i64 = 1_000; +const RETAINED_CHANGES: i64 = 10_000; +const MAX_CHECKPOINT_AGE_SECONDS: i64 = 300; + +pub(super) async fn restore( + pool: &PgPool, + embedded: &EmbeddedSearchIndex, + table: SearchTable, +) -> RuntimeResult> { + let row = sqlx::query( + "SELECT source_cursor, checkpoint_blob, checksum FROM search_runtime_checkpoints WHERE table_key=$1 AND \ + schema_fingerprint=$2", + ) + .bind(table.as_str()) + .bind(SCHEMA_FINGERPRINT) + .fetch_optional(pool) + .await + .map_err(|error| RuntimeError::database("load embedded search checkpoint", error))?; + let Some(row) = row else { return Ok(None) }; + let cursor: i64 = row + .try_get("source_cursor") + .map_err(|error| RuntimeError::database("decode search checkpoint cursor", error))?; + let bytes: Vec = row + .try_get("checkpoint_blob") + .map_err(|error| RuntimeError::database("decode search checkpoint blob", error))?; + let checksum: String = row + .try_get("checksum") + .map_err(|error| RuntimeError::database("decode search checkpoint checksum", error))?; + if digest(&bytes) != checksum { + return Ok(None); + } + if embedded + .restore(table.as_str().to_string(), Buffer::from(bytes)) + .await + .is_err() + { + return Ok(None); + } + Ok(Some(cursor)) +} + +pub(super) async fn persist(pool: &PgPool, embedded: &EmbeddedSearchIndex, cursors: [i64; 2]) -> RuntimeResult<()> { + for table in SearchTable::ORDERED { + let cursor = cursors[table.cursor_index()]; + let persisted: Option<(i64, bool)> = sqlx::query_as( + "SELECT source_cursor, updated_at < now() - make_interval(secs => $2) AS expired FROM \ + search_runtime_checkpoints WHERE table_key=$1", + ) + .bind(table.as_str()) + .bind(MAX_CHECKPOINT_AGE_SECONDS as f64) + .fetch_optional(pool) + .await + .map_err(|error| RuntimeError::database("load persisted checkpoint cursor", error))?; + let (persisted_cursor, expired) = persisted.unwrap_or((0, true)); + if cursor <= persisted_cursor || (cursor - persisted_cursor < DIRTY_CHANGE_THRESHOLD && !expired) { + continue; + } + let mut transaction = pool + .begin() + .await + .map_err(|error| RuntimeError::database("begin embedded checkpoint", error))?; + sqlx::query("SET LOCAL synchronous_commit = off") + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("configure embedded checkpoint commit", error))?; + let leader: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0))") + .bind(format!("search-checkpoint/{}", table.as_str())) + .fetch_one(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("acquire embedded checkpoint lease", error))?; + if !leader { + continue; + } + let persisted: Option<(i64, bool)> = sqlx::query_as( + "SELECT source_cursor, updated_at < now() - make_interval(secs => $2) AS expired FROM \ + search_runtime_checkpoints WHERE table_key=$1", + ) + .bind(table.as_str()) + .bind(MAX_CHECKPOINT_AGE_SECONDS as f64) + .fetch_optional(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("reload persisted checkpoint cursor", error))?; + let (persisted_cursor, expired) = persisted.unwrap_or((0, true)); + if cursor <= persisted_cursor || (cursor - persisted_cursor < DIRTY_CHANGE_THRESHOLD && !expired) { + continue; + } + embedded.optimize(table.as_str().to_string()).await?; + let checkpoint = embedded.checkpoint(table.as_str().to_string()).await?; + let bytes = checkpoint.data.to_vec(); + let saved = sqlx::query( + r#"INSERT INTO search_runtime_checkpoints + (table_key,schema_fingerprint,source_cursor,checkpoint_sequence,checkpoint_blob,checksum,blob_size) + VALUES ($1,$2,$3,$4,$5,$6,$7) + ON CONFLICT (table_key) DO UPDATE SET schema_fingerprint=EXCLUDED.schema_fingerprint, + source_cursor=EXCLUDED.source_cursor,checkpoint_sequence=EXCLUDED.checkpoint_sequence, + checkpoint_blob=EXCLUDED.checkpoint_blob,checksum=EXCLUDED.checksum,blob_size=EXCLUDED.blob_size,updated_at=now() + WHERE search_runtime_checkpoints.source_cursor < EXCLUDED.source_cursor"#, + ) + .bind(table.as_str()) + .bind(SCHEMA_FINGERPRINT) + .bind(cursor) + .bind(checkpoint.sequence) + .bind(&bytes) + .bind(digest(&bytes)) + .bind(bytes.len() as i64) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("persist embedded search checkpoint", error))?; + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit embedded search checkpoint", error))?; + if saved.rows_affected() == 1 { + embedded + .mark_checkpoint_persisted(table.as_str().to_string(), checkpoint.sequence) + .await?; + } + } + gc(pool).await +} + +pub(super) async fn gc(pool: &PgPool) -> RuntimeResult<()> { + for table in SearchTable::ORDERED { + let minimum: Option = sqlx::query_scalar( + r#"SELECT COALESCE(MIN(watermark),0) FROM ( + SELECT c.source_cursor AS watermark FROM search_runtime_provider_cursors c + JOIN search_runtime_generations g USING(generation_id) + WHERE c.table_key=$1 AND g.provider<>'embedded' AND g.state IN ('active','pending') + UNION ALL + SELECT checkpoint.source_cursor FROM search_runtime_checkpoints checkpoint + WHERE checkpoint.table_key=$1 AND EXISTS ( + SELECT 1 FROM search_runtime_generations generation + WHERE generation.provider='embedded' AND generation.state IN ('active','pending') + ) + ) retained"#, + ) + .bind(table.as_str()) + .fetch_one(pool) + .await + .map_err(|error| RuntimeError::database("compute search retention watermark", error))?; + let retained_from = minimum.unwrap_or(0).saturating_sub(RETAINED_CHANGES); + let mut transaction = pool + .begin() + .await + .map_err(|error| RuntimeError::database("begin search change gc", error))?; + sqlx::query("DELETE FROM search_runtime_changes WHERE table_key=$1 AND stream_sequence <= $2") + .bind(table.as_str()) + .bind(retained_from) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("gc search changes", error))?; + sqlx::query("UPDATE search_runtime_streams SET retained_from=GREATEST(retained_from,$2) WHERE table_key=$1") + .bind(table.as_str()) + .bind(retained_from) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("advance search retention watermark", error))?; + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit search change gc", error))?; + } + sqlx::query( + r#"DELETE FROM workspace_permission_changes permission_change + USING workspace_permission_revisions head + WHERE permission_change.workspace_id=head.workspace_id + AND permission_change.revision <= ( + SELECT COALESCE(MIN(cursor.permission_revision),head.revision) + FROM search_runtime_permission_cursors cursor + JOIN search_runtime_generations generation USING(generation_id) + WHERE cursor.workspace_id=permission_change.workspace_id AND generation.state IN ('active','pending') + ) - $1"#, + ) + .bind(RETAINED_CHANGES) + .execute(pool) + .await + .map_err(|error| RuntimeError::database("gc search permission changes", error))?; + Ok(()) +} + +fn digest(bytes: &[u8]) -> String { + Sha256::digest(bytes).iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/generation.rs b/packages/backend/native/src/runtime/backend_runtime/search/generation.rs new file mode 100644 index 0000000000..087bd1ac36 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/generation.rs @@ -0,0 +1,189 @@ +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use sqlx::{PgPool, Row}; +use uuid::Uuid; + +use super::{SCHEMA_FINGERPRINT, provider::RemoteProvider, types::SearchTable}; +use crate::runtime::{RuntimeError, RuntimeResult, SearchRuntimeConfig}; + +#[derive(Clone)] +pub(super) struct ActiveGeneration { + pub(super) id: Uuid, + pub(super) manifest: Value, +} + +impl ActiveGeneration { + pub(super) fn physical_table(&self, table: SearchTable) -> RuntimeResult<&str> { + self + .manifest + .get(table.as_str()) + .and_then(Value::as_str) + .ok_or_else(|| RuntimeError::invalid_state("search generation manifest is incomplete")) + } +} + +pub(super) async fn prepare( + pool: &PgPool, + config: &SearchRuntimeConfig, + remote: Option<&RemoteProvider>, +) -> RuntimeResult { + let fingerprint = config_fingerprint(config); + let mut transaction = pool + .begin() + .await + .map_err(|error| RuntimeError::database("begin search generation", error))?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended('search-runtime-generation', 0))") + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("lock search generation", error))?; + let existing = sqlx::query( + r#"SELECT generation_id,provider,manifest FROM search_runtime_generations + WHERE state IN ('active','pending') AND provider=$1 AND config_fingerprint=$2 AND schema_fingerprint=$3 + ORDER BY (state='active') DESC LIMIT 1"#, + ) + .bind(&config.provider) + .bind(&fingerprint) + .bind(SCHEMA_FINGERPRINT) + .fetch_optional(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("load search generation", error))?; + let generation = if let Some(row) = existing { + decode(row)? + } else { + let pending: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM search_runtime_generations WHERE state='pending')") + .fetch_one(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("check pending search generation", error))?; + if pending { + return Err(RuntimeError::invalid_state("search_generation_change_in_progress")); + } + let generation_id = Uuid::new_v4(); + let suffix = generation_id.simple().to_string(); + let manifest = if config.provider == "embedded" { + json!({"doc":"doc","block":"block"}) + } else { + json!({ + "doc":format!("affine_search_doc_{suffix}"), + "block":format!("affine_search_block_{suffix}"), + }) + }; + sqlx::query( + r#"INSERT INTO search_runtime_generations + (generation_id,provider,state,config_fingerprint,schema_fingerprint,manifest) + VALUES ($1,$2,'pending',$3,$4,$5)"#, + ) + .bind(generation_id) + .bind(&config.provider) + .bind(&fingerprint) + .bind(SCHEMA_FINGERPRINT) + .bind(&manifest) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("create pending search generation", error))?; + for table in [SearchTable::Doc, SearchTable::Block] { + sqlx::query("INSERT INTO search_runtime_provider_cursors(generation_id,table_key) VALUES ($1,$2)") + .bind(generation_id) + .bind(table.as_str()) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("initialize search generation cursor", error))?; + } + ActiveGeneration { + id: generation_id, + manifest, + } + }; + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit pending search generation", error))?; + + if let Some(remote) = remote { + for table in [SearchTable::Doc, SearchTable::Block] { + if let Err(error) = remote.provision(generation.physical_table(table)?, table).await { + fail(pool, &generation).await?; + return Err(error); + } + } + } + Ok(generation) +} + +pub(super) async fn load_active( + pool: &PgPool, + config: &SearchRuntimeConfig, +) -> RuntimeResult> { + let fingerprint = config_fingerprint(config); + let row = sqlx::query( + r#"SELECT generation_id,provider,manifest FROM search_runtime_generations + WHERE state='active' AND provider=$1 AND config_fingerprint=$2 AND schema_fingerprint=$3 + ORDER BY activated_at DESC NULLS LAST LIMIT 1"#, + ) + .bind(&config.provider) + .bind(&fingerprint) + .bind(SCHEMA_FINGERPRINT) + .fetch_optional(pool) + .await + .map_err(|error| RuntimeError::database("load active search generation", error))?; + row.map(decode).transpose() +} + +pub(super) async fn fail(pool: &PgPool, generation: &ActiveGeneration) -> RuntimeResult<()> { + sqlx::query("UPDATE search_runtime_generations SET state='failed' WHERE generation_id=$1 AND state='pending'") + .bind(generation.id) + .execute(pool) + .await + .map_err(|error| RuntimeError::database("fail pending search generation", error))?; + Ok(()) +} + +pub(super) async fn activate(pool: &PgPool, generation: &ActiveGeneration) -> RuntimeResult<()> { + let mut transaction = pool + .begin() + .await + .map_err(|error| RuntimeError::database("begin search generation activation", error))?; + sqlx::query("UPDATE search_runtime_generations SET state='draining' WHERE state='active' AND generation_id<>$1") + .bind(generation.id) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("drain previous search generation", error))?; + sqlx::query( + "UPDATE search_runtime_generations SET state='active', activated_at=coalesce(activated_at,now()) WHERE \ + generation_id=$1 AND state IN ('pending','active')", + ) + .bind(generation.id) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("activate search generation", error))?; + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit search generation activation", error)) +} + +fn config_fingerprint(config: &SearchRuntimeConfig) -> String { + let mut hash = Sha256::new(); + for value in [ + &config.provider, + &config.endpoint, + &config.api_key, + &config.username, + &config.password, + ] { + hash.update(value.as_bytes()); + hash.update([0]); + } + hash.finalize().iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn decode(row: sqlx::postgres::PgRow) -> RuntimeResult { + Ok(ActiveGeneration { + id: row + .try_get("generation_id") + .map_err(|error| RuntimeError::database("decode search generation id", error))?, + manifest: row + .try_get("manifest") + .map_err(|error| RuntimeError::database("decode search generation manifest", error))?, + }) +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/mod.rs b/packages/backend/native/src/runtime/backend_runtime/search/mod.rs new file mode 100644 index 0000000000..8b781605e7 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/mod.rs @@ -0,0 +1,33 @@ +mod checkpoint; +mod generation; +mod projection; +mod provider; +mod query; +mod runtime; +mod store; +mod types; +mod worker; + +pub(super) use runtime::SearchRuntime; +pub(super) use types::{RuntimeAggregateRequest, RuntimeSearchRequest}; + +const SCHEMA_FINGERPRINT: &str = "search-runtime-v5"; + +fn exact_token(value: &str) -> String { + use sha2::{Digest, Sha256}; + Sha256::digest(value.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn provider_payload(payload: &serde_json::Value) -> serde_json::Value { + let mut payload = payload.clone(); + if let Some(object) = payload.as_object_mut() { + object.remove("acl_read_user_ids"); + } + payload +} + +#[cfg(test)] +mod tests; diff --git a/packages/backend/native/src/runtime/backend_runtime/search/projection.rs b/packages/backend/native/src/runtime/backend_runtime/search/projection.rs new file mode 100644 index 0000000000..e2cced1532 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/projection.rs @@ -0,0 +1,207 @@ +use serde_json::{Value, json}; +use sqlx::{PgPool, Row}; + +use super::store::ProjectionInput; +use crate::{ + permission::doc_role_allows, + runtime::{RuntimeError, RuntimeResult, storage_runtime::load_current_doc}, +}; + +pub(super) async fn project_document( + pool: &PgPool, + workspace_id: &str, + doc_id: &str, +) -> RuntimeResult)>> { + let Some(current) = load_current_doc(pool, workspace_id, doc_id).await? else { + return Ok(None); + }; + let revision = current.updated_at.timestamp_millis(); + let projection = + affine_doc_loader::project_document_search(current.blob, doc_id.to_string(), revision.to_string()) + .map_err(|error| RuntimeError::invalid_state(format!("search document projection failed: {error}")))?; + let metadata = sqlx::query( + r#"SELECT snapshot.created_at,snapshot.updated_at,snapshot.created_by,snapshot.updated_by, + revision.revision AS acl_revision, + coalesce(doc_policy.visibility,'private') AS visibility, + doc_policy.public_role, + coalesce(doc_policy.member_default_role,workspace_policy.member_default_doc_role,'manager') AS member_default_role + FROM snapshots snapshot + LEFT JOIN workspace_permission_revisions revision ON revision.workspace_id=snapshot.workspace_id + LEFT JOIN workspace_access_policies workspace_policy ON workspace_policy.workspace_id=snapshot.workspace_id + LEFT JOIN doc_access_policies doc_policy + ON doc_policy.workspace_id=snapshot.workspace_id AND doc_policy.doc_id=snapshot.guid + WHERE snapshot.workspace_id=$1 AND snapshot.guid=$2"#, + ) + .bind(workspace_id) + .bind(doc_id) + .fetch_optional(pool) + .await + .map_err(|error| RuntimeError::database("load search projection metadata", error))? + .ok_or_else(|| RuntimeError::invalid_state("search snapshot metadata unavailable"))?; + let acl_revision = metadata + .try_get::, _>("acl_revision") + .map_err(|error| RuntimeError::database("decode search ACL revision", error))? + .ok_or_else(|| RuntimeError::invalid_state("permission_state_unavailable"))?; + let visibility: String = metadata + .try_get("visibility") + .map_err(|error| RuntimeError::database("decode search doc visibility", error))?; + let public_role: Option = metadata + .try_get("public_role") + .map_err(|error| RuntimeError::database("decode search public role", error))?; + let member_default_role: String = metadata + .try_get("member_default_role") + .map_err(|error| RuntimeError::database("decode search member default role", error))?; + let acl_public_readable = visibility == "public" + && public_role + .as_deref() + .is_some_and(|role| doc_role_allows(role, "Doc.Read").unwrap_or(false)); + let acl_member_default_readable = doc_role_allows(&member_default_role, "Doc.Read") + .map_err(|_| RuntimeError::invalid_state("permission_state_unavailable"))?; + let grants = sqlx::query( + "SELECT principal_id,role FROM doc_grants WHERE workspace_id=$1 AND doc_id=$2 AND principal_type='user'", + ) + .bind(workspace_id) + .bind(doc_id) + .fetch_all(pool) + .await + .map_err(|error| RuntimeError::database("load search doc grants", error))?; + let acl_read_user_ids = grants + .into_iter() + .map(|row| { + let id: String = row + .try_get("principal_id") + .map_err(|error| RuntimeError::database("decode grant principal", error))?; + let role: String = row + .try_get("role") + .map_err(|error| RuntimeError::database("decode grant role", error))?; + Ok(doc_role_allows(&role, "Doc.Read").unwrap_or(false).then_some(id)) + }) + .collect::>>()? + .into_iter() + .flatten() + .collect::>(); + let created_at: chrono::DateTime = metadata + .try_get("created_at") + .map_err(|error| RuntimeError::database("decode search created time", error))?; + let updated_at: chrono::DateTime = metadata + .try_get("updated_at") + .map_err(|error| RuntimeError::database("decode search updated time", error))?; + let created_by: Option = metadata + .try_get("created_by") + .map_err(|error| RuntimeError::database("decode search creator", error))?; + let updated_by: Option = metadata + .try_get("updated_by") + .map_err(|error| RuntimeError::database("decode search updater", error))?; + let acl = AclFields { + public_readable: acl_public_readable, + member_default_readable: acl_member_default_readable, + read_user_ids: acl_read_user_ids, + revision: acl_revision, + }; + let document_payload = with_acl( + json!({ + "workspace_id": workspace_id, + "workspace_token": super::exact_token(workspace_id), + "doc_id": doc_id, + "doc_token": super::exact_token(doc_id), + "title": projection.title, + "summary": projection.units.iter().map(|unit| unit.text.as_str()).collect::>().join("\n").chars().take(1000).collect::(), + "created_by_user_id": created_by.clone().unwrap_or_default(), + "updated_by_user_id": updated_by.clone().unwrap_or_default(), + "created_at": created_at.timestamp_millis(), + "updated_at": updated_at.timestamp_millis(), + }), + &acl, + ); + let document = input( + workspace_id, + doc_id, + &format!("{workspace_id}/{doc_id}"), + revision, + document_payload, + &acl, + ); + let blocks = projection + .units + .into_iter() + .map(|unit| { + let block_id = unit.block_id.clone().unwrap_or_else(|| unit.unit_id.clone()); + let payload = with_acl( + json!({ + "workspace_id":workspace_id,"workspace_token":super::exact_token(workspace_id), + "doc_id":doc_id,"doc_token":super::exact_token(doc_id), + "block_id":block_id,"block_token":super::exact_token(&block_id), + "unit_id":unit.unit_id,"projection_version":projection.version, + "source_hash":projection.source_hash,"visibility":serde_json::to_value(unit.visibility).unwrap_or(Value::Null), + "element_id":unit.element_id,"frame_id":unit.frame_id,"source_block_id":unit.block_id, + "blob":unit.blob_id,"ref_doc_id":unit.ref_doc_ids,"ref":unit.refs,"content":unit.text, + "flavour":format!("affine:{}",unit.unit_type),"parent_flavour":unit.parent_flavour, + "parent_block_id":unit.parent_block_id,"additional":unit.additional, + "created_by_user_id":created_by.clone().unwrap_or_default(),"updated_by_user_id":updated_by.clone().unwrap_or_default(), + "created_at":created_at.timestamp_millis(),"updated_at":updated_at.timestamp_millis(), + }), + &acl, + ); + input( + workspace_id, + doc_id, + &format!("{workspace_id}/{doc_id}/{block_id}"), + revision, + payload, + &acl, + ) + }) + .collect(); + Ok(Some((document, blocks))) +} + +struct AclFields { + public_readable: bool, + member_default_readable: bool, + read_user_ids: Vec, + revision: i64, +} + +fn with_acl(mut payload: Value, acl: &AclFields) -> Value { + let object = payload.as_object_mut().expect("search projection payload is an object"); + object.insert("acl_public_readable".to_string(), json!(acl.public_readable)); + object.insert( + "acl_member_default_readable".to_string(), + json!(acl.member_default_readable), + ); + let mut tokens = acl + .read_user_ids + .iter() + .map(|user_id| super::exact_token(user_id)) + .collect::>(); + if acl.member_default_readable { + tokens.push("member".to_string()); + } + if acl.public_readable { + tokens.push("public".to_string()); + } + object.insert("acl_read_tokens".to_string(), json!(tokens)); + object.insert("acl_revision".to_string(), json!(acl.revision)); + payload +} + +fn input( + workspace_id: &str, + doc_id: &str, + external_id: &str, + revision: i64, + payload: Value, + acl: &AclFields, +) -> ProjectionInput { + ProjectionInput { + external_id: external_id.to_string(), + workspace_id: workspace_id.to_string(), + doc_id: doc_id.to_string(), + revision, + payload, + acl_public_readable: acl.public_readable, + acl_member_default_readable: acl.member_default_readable, + acl_read_user_ids: acl.read_user_ids.clone(), + acl_revision: acl.revision, + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/provider/manticore.rs b/packages/backend/native/src/runtime/backend_runtime/search/provider/manticore.rs new file mode 100644 index 0000000000..7ef123e5bd --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/provider/manticore.rs @@ -0,0 +1,326 @@ +use serde_json::{Value, json}; + +use crate::runtime::{RuntimeError, RuntimeResult}; + +pub(super) fn prepare_manticore_payload( + payload: &mut Value, + token_ids: &std::collections::HashMap, +) -> RuntimeResult<()> { + let object = payload.as_object_mut().expect("search payload is an object"); + object.remove("acl_read_user_ids"); + if let Some(Value::Array(tokens)) = object.remove("acl_read_tokens") { + object.insert( + "acl_read_token_ids".to_string(), + Value::Array( + tokens + .iter() + .filter_map(Value::as_str) + .map(|token| { + token_ids + .get(token) + .copied() + .map(Value::from) + .ok_or_else(|| RuntimeError::invalid_state("Manticore exact token mapping is incomplete")) + }) + .collect::>>()?, + ), + ); + } + if let Some(Value::Array(tokens)) = object.get("ref_doc_id").cloned() { + object.insert( + "ref_doc_token_ids".to_string(), + Value::Array( + tokens + .iter() + .filter_map(Value::as_str) + .map(|token| { + token_ids + .get(token) + .copied() + .map(Value::from) + .ok_or_else(|| RuntimeError::invalid_state("Manticore exact token mapping is incomplete")) + }) + .collect::>>()?, + ), + ); + } + for field in ["created_at", "updated_at"] { + if let Some(value) = object.get_mut(field) + && let Some(milliseconds) = value.as_i64() + { + *value = json!(milliseconds / 1_000); + } + } + for (field, value) in object.iter_mut() { + if let Value::Array(values) = value { + *value = if matches!(field.as_str(), "acl_read_token_ids" | "ref_doc_token_ids") { + continue; + } else if field == "content" { + Value::String(values.iter().filter_map(Value::as_str).collect::>().join(" ")) + } else { + Value::String( + serde_json::to_string(values).map_err(|error| RuntimeError::json("encode manticore array", error))?, + ) + }; + } else if value.is_null() { + *value = Value::String(String::new()); + } + } + Ok(()) +} + +pub(super) fn prepare_manticore_search( + dsl: &mut Value, + cursor: Option, + size: u64, + initial_offset: u64, + requested_fields: &[String], + token_ids: &std::collections::HashMap, +) -> RuntimeResult { + normalize_manticore_terms(dsl, token_ids)?; + let object = dsl.as_object_mut().expect("search DSL is an object"); + let mut source = object + .get("_source") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_string) + .collect::>(); + source.extend(requested_fields.iter().cloned()); + object.insert("_source".to_string(), json!(source)); + object.remove("fields"); + if let Some(highlight) = object.get_mut("highlight") + && let Some(options) = highlight + .get("fields") + .and_then(Value::as_object) + .and_then(|fields| fields.values().next()) + .cloned() + { + *highlight = options; + } + let offset = if let Some(Value::String(cursor)) = cursor { + let offset = serde_json::from_str::(&cursor) + .map_err(|error| RuntimeError::json("invalid search cursor", error))? + .get("offset") + .and_then(Value::as_u64) + .ok_or_else(|| RuntimeError::invalid_input("invalid search cursor"))?; + if offset.saturating_add(size) > 10_000 { + return Err(RuntimeError::invalid_input("search cursor exceeds 10000")); + } + object.insert("from".to_string(), json!(offset)); + offset + } else if cursor.is_some() { + return Err(RuntimeError::invalid_input("invalid search cursor")); + } else { + initial_offset + }; + Ok(offset) +} + +pub(super) fn manticore_fields(source: Option<&Value>, requested_fields: &[String]) -> Value { + let source = source.and_then(Value::as_object); + Value::Object( + requested_fields + .iter() + .filter_map(|field| { + let mut value = source?.get(field)?.clone(); + if matches!(field.as_str(), "created_at" | "updated_at") + && let Some(seconds) = value.as_i64() + { + value = json!(seconds * 1_000); + } else if let Some(encoded) = value.as_str() + && encoded.starts_with('[') + && let Ok(decoded) = serde_json::from_str(encoded) + { + value = decoded; + } + if !value.is_array() { + value = Value::Array(vec![value]); + } + Some((field.clone(), value)) + }) + .collect(), + ) +} + +fn normalize_manticore_terms( + value: &mut Value, + token_ids: &std::collections::HashMap, +) -> RuntimeResult<()> { + if let Some(term) = manticore_term(value, token_ids)? { + *value = term; + return Ok(()); + } + match value { + Value::Object(object) => { + if let Some(Value::Object(boolean)) = object.get_mut("bool") + && boolean.get("boost").and_then(Value::as_f64) == Some(1.0) + { + boolean.remove("boost"); + } + if let Some(Value::Object(terms)) = object.get_mut("terms") { + terms.entry("order").or_insert_with(|| json!({"_count":"desc"})); + } + for child in object.values_mut() { + normalize_manticore_terms(child, token_ids)?; + } + } + Value::Array(array) => { + for child in array { + normalize_manticore_terms(child, token_ids)?; + } + } + _ => {} + } + Ok(()) +} + +fn manticore_term(value: &Value, token_ids: &std::collections::HashMap) -> RuntimeResult> { + let Some(term) = value.get("term").and_then(Value::as_object) else { + return Ok(None); + }; + if term.len() != 1 { + return Ok(None); + } + let Some((field, clause)) = term.iter().next() else { + return Ok(None); + }; + let value = clause.get("value").unwrap_or(clause); + Ok(match value { + Value::String(value) => { + if matches!(field.as_str(), "acl_read_tokens" | "ref_doc_id") { + let token_id = token_ids + .get(value) + .copied() + .ok_or_else(|| RuntimeError::invalid_state("Manticore exact token mapping is incomplete"))?; + let field = if field == "acl_read_tokens" { + "acl_read_token_ids" + } else { + "ref_doc_token_ids" + }; + return Ok(Some(json!({"equals":{field:token_id}}))); + } + let (field, value) = match field.as_str() { + "workspace_id" => ("workspace_token", super::super::exact_token(value)), + "doc_id" => ("doc_token", super::super::exact_token(value)), + "block_id" => ("block_token", super::super::exact_token(value)), + _ => (field.as_str(), value.clone()), + }; + if let Some(boost) = clause.get("boost").and_then(Value::as_f64) { + Some(json!({"match":{field:{"query":value,"boost":boost}}})) + } else { + Some(json!({"equals":{field:value}})) + } + } + Value::Bool(value) => Some(json!({"equals":{field:u8::from(*value)}})), + Value::Number(value) => Some(json!({"equals":{field:value}})), + _ => None, + }) +} + +pub(super) fn manticore_exact_tokens(value: &Value) -> Vec { + let mut tokens = Vec::new(); + collect_exact_tokens(value, &mut tokens); + tokens +} + +fn collect_exact_tokens(value: &Value, tokens: &mut Vec) { + match value { + Value::Object(object) => { + if let Some(token) = object + .get("term") + .and_then(|term| term.get("acl_read_tokens").or_else(|| term.get("ref_doc_id"))) + .and_then(|clause| clause.get("value").unwrap_or(clause).as_str()) + { + tokens.push(token.to_string()); + } + for field in ["acl_read_tokens", "ref_doc_id"] { + if let Some(values) = object.get(field).and_then(Value::as_array) { + tokens.extend(values.iter().filter_map(Value::as_str).map(str::to_string)); + } + } + for child in object.values() { + collect_exact_tokens(child, tokens); + } + } + Value::Array(values) => { + for child in values { + collect_exact_tokens(child, tokens); + } + } + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::{super::super::exact_token, *}; + + #[test] + fn payload_and_fields_preserve_terminal_types() { + let mut payload = json!({ + "content":["hello","world"], + "ref_doc_id":["doc-a","doc-b"], + "summary":null, + "created_at":2_000, + "updated_at":3_000, + "acl_read_tokens":["member"] + }); + prepare_manticore_payload( + &mut payload, + &[ + ("member".to_string(), 7), + ("doc-a".to_string(), 8), + ("doc-b".to_string(), 9), + ] + .into_iter() + .collect(), + ) + .unwrap(); + assert_eq!(payload["content"], "hello world"); + assert_eq!(payload["ref_doc_id"], "[\"doc-a\",\"doc-b\"]"); + assert_eq!(payload["summary"], ""); + assert_eq!(payload["created_at"], 2); + assert_eq!(payload["acl_read_token_ids"], json!([7])); + assert_eq!(payload["ref_doc_token_ids"], json!([8, 9])); + + let fields = manticore_fields( + Some(&payload), + &[ + "ref_doc_id".to_string(), + "summary".to_string(), + "updated_at".to_string(), + ], + ); + assert_eq!(fields["ref_doc_id"], json!(["doc-a", "doc-b"])); + assert_eq!(fields["summary"], json!([""])); + assert_eq!(fields["updated_at"], json!([3_000])); + } + + #[test] + fn nested_terms_use_exact_identity_and_acl_tokens() { + let mut dsl = json!({"query":{"bool":{"must":[ + {"term":{"workspace_id":{"value":"workspace","boost":2.0}}}, + {"bool":{"must_not":[{"term":{"doc_id":{"value":"doc"}}}]}}, + {"term":{"acl_read_tokens":{"value":"member"}}}, + {"term":{"ref_doc_id":{"value":"ref-doc"}}} + ],"boost":1.0}}}); + normalize_manticore_terms( + &mut dsl, + &[("member".to_string(), 9), ("ref-doc".to_string(), 10)] + .into_iter() + .collect(), + ) + .unwrap(); + assert_eq!( + dsl, + json!({"query":{"bool":{"must":[ + {"match":{"workspace_token":{"query":exact_token("workspace"),"boost":2.0}}}, + {"bool":{"must_not":[{"equals":{"doc_token":exact_token("doc")}}]}}, + {"equals":{"acl_read_token_ids":9}}, + {"equals":{"ref_doc_token_ids":10}} + ]}}}) + ); + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/provider/mod.rs b/packages/backend/native/src/runtime/backend_runtime/search/provider/mod.rs new file mode 100644 index 0000000000..333beafee5 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/provider/mod.rs @@ -0,0 +1,125 @@ +mod manticore; +mod remote; + +pub(super) use remote::RemoteProvider; +use serde_json::{Value, json}; + +use super::types::SearchTable; + +pub(super) fn mapping(table: SearchTable, provider: &str) -> Value { + let text_field = table.text_field(); + let mut properties = serde_json::Map::from_iter([ + ("workspace_id".into(), json!({"type":"keyword"})), + ("workspace_token".into(), json!({"type":"keyword"})), + ("doc_id".into(), json!({"type":"keyword"})), + ("doc_token".into(), json!({"type":"keyword"})), + (text_field.into(), json!({"type":"text"})), + ( + "created_at".into(), + json!({"type":if provider == "manticoresearch" { "long" } else { "date" }}), + ), + ( + "updated_at".into(), + json!({"type":if provider == "manticoresearch" { "long" } else { "date" }}), + ), + ("created_by_user_id".into(), json!({"type":"keyword"})), + ("updated_by_user_id".into(), json!({"type":"keyword"})), + ("acl_public_readable".into(), json!({"type":"boolean"})), + ("acl_member_default_readable".into(), json!({"type":"boolean"})), + ( + "acl_read_tokens".into(), + if provider == "manticoresearch" { + json!({"type":"keyword","mva":true}) + } else { + json!({"type":"keyword"}) + }, + ), + ("acl_revision".into(), json!({"type":"long"})), + ]); + if table == SearchTable::Block { + for field in [ + "block_id", + "block_token", + "unit_id", + "source_hash", + "visibility", + "element_id", + "frame_id", + "source_block_id", + "flavour", + "blob", + "ref_doc_id", + "parent_flavour", + "parent_block_id", + ] { + properties.insert(field.into(), json!({"type":"keyword"})); + } + properties.insert("projection_version".into(), json!({"type":"integer"})); + for field in ["ref", "additional", "markdown_preview"] { + properties.insert(field.into(), json!({"type":"text","index":false})); + } + } else { + properties.insert("summary".into(), json!({"type":"text","index":false})); + properties.insert("journal".into(), json!({"type":"keyword"})); + } + json!({"mappings":{"properties":properties}}) +} + +pub(super) fn manticore_schema(table: SearchTable, physical_table: &str) -> String { + let common = r#" + workspace_id string attribute indexed, + workspace_token string attribute indexed, + doc_id string attribute indexed, + doc_token string attribute indexed,"#; + let fields = match table { + SearchTable::Doc => format!( + r#"{common} + title text, + summary string stored, + journal string stored, + created_by_user_id string attribute indexed, + updated_by_user_id string attribute indexed, + created_at timestamp, + updated_at timestamp, + acl_public_readable bool, + acl_member_default_readable bool, + acl_read_token_ids multi64, + acl_revision bigint"#, + ), + SearchTable::Block => format!( + r#"{common} + block_id string attribute indexed, + block_token string attribute indexed, + unit_id string attribute indexed, + projection_version bigint, + source_hash string attribute indexed, + visibility string attribute indexed, + element_id string attribute indexed, + frame_id string attribute indexed, + source_block_id string attribute indexed, + content text, + flavour string attribute indexed, + blob string attribute indexed, + ref_doc_id string attribute indexed, + ref_doc_token_ids multi64, + ref string stored, + parent_flavour string attribute indexed, + parent_block_id string attribute indexed, + additional string stored, + markdown_preview string stored, + created_by_user_id string attribute indexed, + updated_by_user_id string attribute indexed, + created_at timestamp, + updated_at timestamp, + acl_public_readable bool, + acl_member_default_readable bool, + acl_read_token_ids multi64, + acl_revision bigint"#, + ), + }; + format!( + "CREATE TABLE IF NOT EXISTS {physical_table} ({fields}) charset_table='non_cjk, chinese' ngram_len='1' \ + ngram_chars='U+1100..U+11FF, U+3130..U+318F, U+A960..U+A97F, U+AC00..U+D7AF, U+D7B0..U+D7FF, U+3040..U+30FF, \ + U+0E00..U+0E7F' index_field_lengths='1'" + ) +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/provider/remote.rs b/packages/backend/native/src/runtime/backend_runtime/search/provider/remote.rs new file mode 100644 index 0000000000..4ca7ae2a5b --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/provider/remote.rs @@ -0,0 +1,401 @@ +use std::time::Duration; + +use reqwest::{Client, redirect::Policy}; +use serde_json::{Value, json}; +use sqlx::PgPool; + +use super::{ + super::{store::SearchChange, types::SearchTable}, + manticore::{manticore_exact_tokens, manticore_fields, prepare_manticore_payload, prepare_manticore_search}, +}; +use crate::runtime::{RuntimeError, RuntimeResult, SearchRuntimeConfig}; + +const MAX_RESPONSE_BYTES: usize = 50 * 1024 * 1024; + +pub(in crate::runtime::backend_runtime::search) struct RemoteProvider { + client: Client, + endpoint: String, + provider: String, + api_key: String, + username: String, + password: String, + pool: PgPool, +} + +impl RemoteProvider { + pub(in crate::runtime::backend_runtime::search) fn new( + config: &SearchRuntimeConfig, + pool: PgPool, + ) -> RuntimeResult { + let endpoint = config.endpoint.trim_end_matches('/'); + let url = url::Url::parse(endpoint).map_err(|_| RuntimeError::config("invalid search provider endpoint"))?; + if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { + return Err(RuntimeError::config("invalid search provider endpoint")); + } + let mut client = Client::builder() + .redirect(Policy::none()) + .timeout(Duration::from_secs(30)); + if config.provider == "manticoresearch" { + client = client.pool_max_idle_per_host(0); + } + let client = client + .build() + .map_err(|error| RuntimeError::invalid_state(format!("search HTTP client failed: {error}")))?; + Ok(Self { + client, + endpoint: endpoint.to_string(), + provider: config.provider.clone(), + api_key: config.api_key.clone(), + username: config.username.clone(), + password: config.password.clone(), + pool, + }) + } + + pub(in crate::runtime::backend_runtime::search) async fn search( + &self, + physical_table: &str, + mut dsl: Value, + ) -> RuntimeResult { + let requested_fields = dsl + .get("fields") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_string) + .collect::>(); + dsl["track_total_hits"] = json!(true); + let size = dsl.get("size").and_then(Value::as_u64).unwrap_or(10); + let mut offset = dsl.get("from").and_then(Value::as_u64).unwrap_or(0); + let cursor = dsl.as_object_mut().and_then(|object| object.remove("cursor")); + if self.provider == "manticoresearch" { + let token_ids = self.resolve_manticore_tokens(manticore_exact_tokens(&dsl)).await?; + offset = prepare_manticore_search(&mut dsl, cursor, size, offset, &requested_fields, &token_ids)?; + } else if let Some(cursor) = cursor { + let cursor = cursor + .as_str() + .ok_or_else(|| RuntimeError::invalid_input("invalid search cursor"))?; + dsl["search_after"] = + serde_json::from_str(cursor).map_err(|error| RuntimeError::json("invalid search cursor", error))?; + } + let mut request = self + .client + .post(format!("{}/{physical_table}/_search", self.endpoint)) + .json(&dsl); + if !self.api_key.is_empty() { + request = request.header("Authorization", format!("ApiKey {}", self.api_key)); + } else if !self.username.is_empty() { + request = request.basic_auth(&self.username, Some(&self.password)); + } + let response = request + .send() + .await + .map_err(|_| RuntimeError::SearchProviderUnavailable)?; + let status = response.status(); + let bytes = read_response(response).await?; + if !status.is_success() { + return Err(if status.as_u16() == 400 { + RuntimeError::SearchUnsupportedQuery + } else { + RuntimeError::SearchProviderUnavailable + }); + } + let value: Value = + serde_json::from_slice(&bytes).map_err(|error| RuntimeError::json("invalid search provider response", error))?; + normalize( + value, + self.provider == "manticoresearch", + offset, + size, + &requested_fields, + ) + } + + pub(in crate::runtime::backend_runtime::search) async fn aggregate( + &self, + physical_table: &str, + mut dsl: Value, + ) -> RuntimeResult { + if self.provider == "manticoresearch" { + return Err(RuntimeError::SearchUnsupportedQuery); + } + dsl["track_total_hits"] = json!(true); + let response = self + .request(reqwest::Method::POST, &format!("{physical_table}/_search")) + .json(&dsl) + .send() + .await + .map_err(|_| RuntimeError::SearchProviderUnavailable)?; + let status = response.status(); + let bytes = read_response(response).await?; + if !status.is_success() { + return Err(RuntimeError::SearchUnsupportedQuery); + } + let value: Value = + serde_json::from_slice(&bytes).map_err(|error| RuntimeError::json("invalid search provider response", error))?; + normalize_aggregate(value) + } + + pub(in crate::runtime::backend_runtime::search) async fn provision( + &self, + physical_table: &str, + table: SearchTable, + ) -> RuntimeResult<()> { + if self.provider == "manticoresearch" { + let response = self + .request(reqwest::Method::POST, "cli") + .header("content-type", "text/plain") + .body(super::manticore_schema(table, physical_table)) + .send() + .await + .map_err(|_| RuntimeError::SearchProviderUnavailable)?; + return response + .status() + .is_success() + .then_some(()) + .ok_or(RuntimeError::SearchProviderUnavailable); + } + if self + .request(reqwest::Method::HEAD, physical_table) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + return Ok(()); + } + let response = self + .request(reqwest::Method::PUT, physical_table) + .json(&super::mapping(table, &self.provider)) + .send() + .await; + match response { + Ok(response) if response.status().is_success() => Ok(()), + _ => Err(RuntimeError::SearchProviderUnavailable), + } + } + + pub(in crate::runtime::backend_runtime::search) async fn apply( + &self, + physical_table: &str, + changes: &[SearchChange], + ) -> RuntimeResult<()> { + if changes.is_empty() { + return Ok(()); + } + let token_ids = if self.provider == "manticoresearch" { + self + .resolve_manticore_tokens( + changes + .iter() + .filter_map(|change| change.payload.as_ref()) + .flat_map(manticore_exact_tokens), + ) + .await? + } else { + Default::default() + }; + let mut body = String::new(); + for change in changes { + if change.operation == "delete" { + body.push_str( + &serde_json::to_string(&json!({"delete":{"_index":physical_table,"_id":change.external_id}})) + .map_err(|error| RuntimeError::json("encode provider delete", error))?, + ); + body.push('\n'); + } else if let Some(payload) = &change.payload { + body.push_str( + &serde_json::to_string(&json!({"index":{"_index":physical_table,"_id":change.external_id}})) + .map_err(|error| RuntimeError::json("encode provider upsert", error))?, + ); + body.push('\n'); + let mut payload = super::super::provider_payload(payload); + if self.provider == "manticoresearch" { + prepare_manticore_payload(&mut payload, &token_ids)?; + } + body.push_str( + &serde_json::to_string(&payload).map_err(|error| RuntimeError::json("encode provider document", error))?, + ); + body.push('\n'); + } + } + let path = if self.provider == "elasticsearch" { + "_bulk?refresh=wait_for" + } else { + "_bulk" + }; + let response = self + .request(reqwest::Method::POST, path) + .header("content-type", "application/x-ndjson") + .body(body) + .send() + .await + .map_err(|_| RuntimeError::SearchProviderUnavailable)?; + if !response.status().is_success() { + return Err(RuntimeError::SearchProviderUnavailable); + } + let value: Value = response + .json() + .await + .map_err(|error| RuntimeError::invalid_state(format!("invalid provider bulk response: {error}")))?; + if value.get("errors").and_then(Value::as_bool) == Some(true) { + return Err(RuntimeError::invalid_state("provider_apply_failed")); + } + Ok(()) + } + + fn request(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder { + let mut request = self.client.request(method, format!("{}/{path}", self.endpoint)); + if !self.api_key.is_empty() { + request = request.header("Authorization", format!("ApiKey {}", self.api_key)); + } else if !self.username.is_empty() { + request = request.basic_auth(&self.username, Some(&self.password)); + } + request + } + + async fn resolve_manticore_tokens( + &self, + tokens: impl IntoIterator, + ) -> RuntimeResult> { + let tokens = tokens.into_iter().collect::>(); + if tokens.is_empty() { + return Ok(Default::default()); + } + let tokens = tokens.into_iter().collect::>(); + let mut transaction = self + .pool + .begin() + .await + .map_err(|error| RuntimeError::database("begin Manticore exact token resolution", error))?; + sqlx::query( + r#"INSERT INTO search_runtime_acl_tokens(token) + SELECT candidate.token FROM unnest($1::text[]) candidate(token) + LEFT JOIN search_runtime_acl_tokens existing USING(token) + WHERE existing.token IS NULL ON CONFLICT DO NOTHING"#, + ) + .bind(&tokens) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("allocate Manticore exact token IDs", error))?; + let rows: Vec<(String, i64)> = + sqlx::query_as("SELECT token,token_id FROM search_runtime_acl_tokens WHERE token=ANY($1)") + .bind(&tokens) + .fetch_all(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("load Manticore exact token IDs", error))?; + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit Manticore exact token resolution", error))?; + if rows.len() != tokens.len() { + return Err(RuntimeError::invalid_state( + "Manticore exact token mapping is incomplete", + )); + } + Ok(rows.into_iter().collect()) + } +} + +async fn read_response(mut response: reqwest::Response) -> RuntimeResult> { + if response + .content_length() + .is_some_and(|length| length > MAX_RESPONSE_BYTES as u64) + { + return Err(RuntimeError::invalid_state("provider_response_too_large")); + } + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| RuntimeError::SearchProviderUnavailable)? + { + if bytes.len() + chunk.len() > MAX_RESPONSE_BYTES { + return Err(RuntimeError::invalid_state("provider_response_too_large")); + } + bytes.extend_from_slice(&chunk); + } + Ok(bytes) +} + +fn normalize( + value: Value, + manticore: bool, + _offset: u64, + _size: u64, + requested_fields: &[String], +) -> RuntimeResult { + let hits = value + .pointer("/hits/hits") + .and_then(Value::as_array) + .ok_or_else(|| RuntimeError::invalid_state("invalid provider response"))?; + let total = value + .pointer("/hits/total/value") + .or_else(|| value.pointer("/hits/total")) + .and_then(Value::as_u64) + .ok_or_else(|| RuntimeError::invalid_state("inexact provider total"))?; + let nodes = hits + .iter() + .map(|hit| { + let fields = if manticore { + manticore_fields(hit.get("_source"), requested_fields) + } else { + hit.get("fields").cloned().unwrap_or_else(|| json!({})) + }; + json!({ + "id":hit.get("_id").and_then(Value::as_str).unwrap_or_default(), + "score":hit.get("_score").and_then(Value::as_f64).unwrap_or_default(), + "fields":fields, + "highlights":hit.get("highlight").cloned().unwrap_or_else(||json!({})), + "_source":hit.get("_source").cloned().unwrap_or_else(||json!({})), + }) + }) + .collect::>(); + let next_cursor = if manticore { + (!hits.is_empty()) + .then(|| serde_json::to_string(&json!({"offset":_offset + _size}))) + .transpose() + .map_err(|error| RuntimeError::json("encode provider cursor", error))? + } else { + hits + .last() + .and_then(|hit| hit.get("sort")) + .map(serde_json::to_string) + .transpose() + .map_err(|error| RuntimeError::json("encode provider cursor", error))? + }; + Ok(json!({"total":total,"nodes":nodes,"nextCursor":next_cursor})) +} + +fn normalize_aggregate(value: Value) -> RuntimeResult { + let buckets = value + .pointer("/aggregations/result/buckets") + .and_then(Value::as_array) + .ok_or_else(|| RuntimeError::invalid_state("invalid provider aggregate response"))?; + let nodes = buckets + .iter() + .map(|bucket| { + let hits = bucket + .pointer("/result/hits/hits") + .and_then(Value::as_array) + .ok_or_else(|| RuntimeError::invalid_state("invalid provider aggregate hits"))?; + Ok(json!({ + "key":bucket.get("key").cloned().unwrap_or(Value::Null), + "count":bucket.get("doc_count").cloned().unwrap_or(json!(0)), + "hits":{"total":bucket.pointer("/result/hits/total/value").or_else(||bucket.pointer("/result/hits/total")).cloned().unwrap_or(json!(0)), + "nodes":hits.iter().map(normalize_hit).collect::>()} + })) + }) + .collect::>>()?; + Ok(json!({"total":nodes.len(),"hasMore":false,"buckets":nodes})) +} + +fn normalize_hit(hit: &Value) -> Value { + json!({ + "id":hit.get("_id").and_then(Value::as_str).unwrap_or_default(), + "score":hit.get("_score").and_then(Value::as_f64).unwrap_or_default(), + "fields":hit.get("fields").cloned().unwrap_or_else(||json!({})), + "highlights":hit.get("highlight").cloned().unwrap_or_else(||json!({})), + "_source":hit.get("_source").cloned().unwrap_or_else(||json!({})), + }) +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/query.rs b/packages/backend/native/src/runtime/backend_runtime/search/query.rs new file mode 100644 index 0000000000..743bcbe121 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/query.rs @@ -0,0 +1,336 @@ +use serde_json::{Value, json}; + +use super::types::{AggregateRequest, SearchOptions, SearchQuery, SearchRequest, SearchTable}; +use crate::runtime::{ + RuntimeError, RuntimeResult, + backend_runtime::permission::{AuthorizedSearchScope, DocReadScope}, +}; + +pub(super) fn compile(request: &SearchRequest, scope: &AuthorizedSearchScope) -> RuntimeResult { + let query = compile_query(request.table, &request.query)?; + let mut must = vec![json!({"term":{"workspace_id":{"value":scope.workspace_id}}}), query]; + if let DocReadScope::ProjectedAcl(predicate) = &scope.docs { + let mut should = vec![json!({"term":{"acl_read_tokens":{"value":super::exact_token(&predicate.actor_user_id)}}})]; + if predicate.active_member { + should.push(json!({"term":{"acl_read_tokens":{"value":"member"}}})); + } + if predicate.sharing_enabled { + should.push(json!({"term":{"acl_read_tokens":{"value":"public"}}})); + } + must.push(json!({"bool":{"should":should}})); + } + let fields = request + .options + .fields + .iter() + .map(|field| validate_field(request.table, field).map(str::to_string)) + .collect::>>()?; + let mut dsl = json!({ + "_source":["workspace_id","doc_id"], + "fields":fields, + "query":{"bool":{"must":must}}, + "sort": stable_sort(request.table), + }); + let pagination = &request.options.pagination; + if pagination.limit.unwrap_or(10) > 10_000 { + return Err(RuntimeError::invalid_input("search limit exceeds 10000")); + } + dsl["size"] = json!(pagination.limit.unwrap_or(10)); + if let Some(skip) = pagination.skip { + if skip.saturating_add(pagination.limit.unwrap_or(10)) > 10_000 { + return Err(RuntimeError::invalid_input("search offset exceeds 10000")); + } + dsl["from"] = json!(skip); + } + if let Some(cursor) = &pagination.cursor { + dsl["cursor"] = json!(cursor); + } + if !request.options.highlights.is_empty() { + let mut highlights = serde_json::Map::new(); + for highlight in &request.options.highlights { + let field = validate_field(request.table, &highlight.field)?; + highlights.insert( + field.to_string(), + json!({"pre_tags":[highlight.before],"post_tags":[highlight.end]}), + ); + } + dsl["highlight"] = json!({"fields":highlights}); + } + Ok(dsl) +} + +pub(super) fn compile_aggregate(request: &AggregateRequest, scope: &AuthorizedSearchScope) -> RuntimeResult { + let hits = SearchOptions { + fields: request.options.hits.fields.clone(), + highlights: request.options.hits.highlights.clone(), + pagination: request.options.hits.pagination.clone(), + }; + let search = SearchRequest { + table: request.table, + query: request.query.clone(), + options: hits, + }; + let hit_dsl = compile(&search, scope)?; + let field = validate_field(request.table, &request.field)?; + let limit = request.options.pagination.limit.unwrap_or(10); + if limit > 10_000 { + return Err(RuntimeError::invalid_input("aggregate limit exceeds 10000")); + } + Ok(json!({ + "query":hit_dsl["query"], + "from":request.options.pagination.skip.unwrap_or(0), + "size":0, + "aggs":{"result":{"terms":{"field":field,"size":limit},"aggs":{"result":{"top_hits":{ + "size":hit_dsl["size"],"_source":hit_dsl["_source"],"fields":hit_dsl["fields"], + "sort":hit_dsl["sort"],"highlight":hit_dsl.get("highlight").cloned().unwrap_or_else(||json!({})) + }}}}} + })) +} + +fn compile_query(table: SearchTable, query: &SearchQuery) -> RuntimeResult { + let boost = query.boost.unwrap_or(1.0); + if !boost.is_finite() || boost <= 0.0 { + return Err(RuntimeError::invalid_input("invalid search boost")); + } + match query.query_type.as_str() { + "match" => { + let field = validate_field( + table, + query + .field + .as_deref() + .ok_or_else(|| RuntimeError::invalid_input("match field is required"))?, + )?; + let value = query + .match_value + .as_deref() + .filter(|value| !value.is_empty()) + .ok_or_else(|| RuntimeError::invalid_input("match value is required"))?; + if field == table.text_field() { + Ok(json!({"match":{field:{"query":value,"boost":boost}}})) + } else { + Ok(json!({"term":{field:{"value":value,"boost":boost}}})) + } + } + "boolean" => { + let occur = query + .occur + .as_deref() + .filter(|occur| matches!(*occur, "must" | "should" | "must_not")) + .ok_or_else(|| RuntimeError::invalid_input("invalid boolean occurrence"))?; + let clauses = query + .queries + .as_deref() + .ok_or_else(|| RuntimeError::invalid_input("boolean queries are required"))? + .iter() + .map(|query| compile_query(table, query)) + .collect::>>()?; + Ok(json!({"bool":{occur:clauses,"boost":boost}})) + } + "exists" => { + let field = validate_field( + table, + query + .field + .as_deref() + .ok_or_else(|| RuntimeError::invalid_input("exists field is required"))?, + )?; + Ok(json!({"exists":{"field":field,"boost":boost}})) + } + "all" => Ok(json!({"match_all":{"boost":boost}})), + "boost" => { + let mut nested = query + .query + .as_deref() + .ok_or_else(|| RuntimeError::invalid_input("boost query is required"))? + .clone(); + nested.boost = Some(boost); + compile_query(table, &nested) + } + _ => Err(RuntimeError::invalid_input("unsupported search query")), + } +} + +fn stable_sort(table: SearchTable) -> Value { + match table { + SearchTable::Doc => json!(["_score", {"updated_at":"desc"}, "doc_id"]), + SearchTable::Block => json!(["_score", {"updated_at":"desc"}, "doc_id", "block_id"]), + } +} + +fn validate_field(table: SearchTable, field: &str) -> RuntimeResult<&'static str> { + let normalized = match field { + "workspaceId" => "workspace_id", + "docId" => "doc_id", + "blockId" => "block_id", + "createdByUserId" => "created_by_user_id", + "updatedByUserId" => "updated_by_user_id", + "createdAt" => "created_at", + "updatedAt" => "updated_at", + "refDocId" => "ref_doc_id", + "parentFlavour" => "parent_flavour", + "parentBlockId" => "parent_block_id", + "unitId" => "unit_id", + "projectionVersion" => "projection_version", + "sourceHash" => "source_hash", + "elementId" => "element_id", + "frameId" => "frame_id", + "sourceBlockId" => "source_block_id", + "markdownPreview" => "markdown_preview", + value => value, + }; + let allowed = match table { + SearchTable::Doc => [ + "workspace_id", + "doc_id", + "title", + "summary", + "journal", + "created_by_user_id", + "updated_by_user_id", + "created_at", + "updated_at", + ] + .as_slice(), + SearchTable::Block => [ + "workspace_id", + "doc_id", + "block_id", + "unit_id", + "projection_version", + "source_hash", + "visibility", + "element_id", + "frame_id", + "source_block_id", + "content", + "flavour", + "blob", + "ref_doc_id", + "ref", + "parent_flavour", + "parent_block_id", + "additional", + "markdown_preview", + "created_by_user_id", + "updated_by_user_id", + "created_at", + "updated_at", + ] + .as_slice(), + }; + allowed + .iter() + .find(|candidate| **candidate == normalized) + .copied() + .ok_or_else(|| RuntimeError::invalid_input("unknown or internal search field")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::runtime::backend_runtime::permission::{AuthorizedSearchScope, DocReadScope}; + + fn scope() -> AuthorizedSearchScope { + AuthorizedSearchScope { + workspace_id: "workspace".to_string(), + permission_revision: 1, + docs: DocReadScope::All, + } + } + + fn request(query: Value) -> SearchRequest { + serde_json::from_value(json!({ + "table": "block", + "query": query, + "options": { + "fields": ["docId", "createdAt"], + "highlights": [{"field": "content", "before": "", "end": ""}], + "pagination": {"limit": 20, "skip": 5} + } + })) + .unwrap() + } + + #[test] + fn compiles_supported_query_variants_and_options() { + let cases = [ + (json!({"type":"all"}), json!({"match_all":{"boost":1.0}})), + ( + json!({"type":"exists","field":"refDocId"}), + json!({"exists":{"field":"ref_doc_id","boost":1.0}}), + ), + ( + json!({"type":"boost","boost":2.5,"query":{"type":"match","field":"content","match":"hello"}}), + json!({"match":{"content":{"query":"hello","boost":2.5}}}), + ), + ( + json!({"type":"boolean","occur":"must_not","queries":[{"type":"match","field":"docId","match":"doc"}]}), + json!({"bool":{"must_not":[{"term":{"doc_id":{"value":"doc","boost":1.0}}}],"boost":1.0}}), + ), + ]; + for (query, expected) in cases { + let dsl = compile(&request(query), &scope()).unwrap(); + assert_eq!(dsl["query"]["bool"]["must"][1], expected); + assert_eq!(dsl["fields"], json!(["doc_id", "created_at"])); + assert_eq!(dsl["from"], 5); + assert_eq!(dsl["size"], 20); + assert_eq!( + dsl["highlight"]["fields"]["content"], + json!({"pre_tags":[""],"post_tags":[""]}) + ); + } + } + + #[test] + fn validates_query_fields_and_pagination_limits() { + for query in [ + json!({"type":"match","field":"aclReadTokens","match":"member"}), + json!({"type":"exists","field":"unknown"}), + json!({"type":"boolean","occur":"invalid","queries":[]}), + json!({"type":"boost","boost":0,"query":{"type":"all"}}), + ] { + assert!(compile(&request(query), &scope()).is_err()); + } + + let mut oversized = request(json!({"type":"all"})); + oversized.options.pagination.limit = Some(10_001); + assert!(compile(&oversized, &scope()).is_err()); + oversized.options.pagination.limit = Some(10_000); + oversized.options.pagination.skip = Some(1); + assert!(compile(&oversized, &scope()).is_err()); + } + + #[test] + fn compiles_aggregate_contract_and_rejects_invalid_fields() { + let aggregate: AggregateRequest = serde_json::from_value(json!({ + "table":"block", + "query":{"type":"match","field":"content","match":"hello"}, + "field":"docId", + "options":{ + "hits":{ + "fields":["docId","content"], + "highlights":[{"field":"content","before":"","end":""}], + "pagination":{"limit":2} + }, + "pagination":{"limit":50,"skip":3} + } + })) + .unwrap(); + let dsl = compile_aggregate(&aggregate, &scope()).unwrap(); + assert_eq!(dsl["from"], 3); + assert_eq!(dsl["aggs"]["result"]["terms"], json!({"field":"doc_id","size":50})); + assert_eq!(dsl["aggs"]["result"]["aggs"]["result"]["top_hits"]["size"], 2); + assert_eq!( + dsl["aggs"]["result"]["aggs"]["result"]["top_hits"]["highlight"]["fields"]["content"], + json!({"pre_tags":[""],"post_tags":[""]}) + ); + + let mut invalid = aggregate; + invalid.field = "aclReadTokens".to_string(); + assert!(compile_aggregate(&invalid, &scope()).is_err()); + invalid.field = "docId".to_string(); + invalid.options.pagination.limit = Some(10_001); + assert!(compile_aggregate(&invalid, &scope()).is_err()); + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/runtime.rs b/packages/backend/native/src/runtime/backend_runtime/search/runtime.rs new file mode 100644 index 0000000000..b353d78ead --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/runtime.rs @@ -0,0 +1,470 @@ +use sqlx::PgPool; +use tokio::sync::{Mutex, RwLock}; + +use super::{ + super::permission::{PermissionAuthorizer, SearchActor, SystemSearchCapability}, + generation::{self, ActiveGeneration}, + projection::project_document, + provider::RemoteProvider, + query, + store::{SearchStore, SearchTable}, + types::{RuntimeAggregateRequest, RuntimeSearchRequest}, +}; +use crate::{ + runtime::{RuntimeError, RuntimeResult, SearchRuntimeConfig}, + search_index::EmbeddedSearchIndex, +}; + +pub(in crate::runtime::backend_runtime) struct SearchRuntime { + pool: PgPool, + store: SearchStore, + authorizer: PermissionAuthorizer, + pub(super) embedded: EmbeddedSearchIndex, + remote: Option, + config: SearchRuntimeConfig, + generation: RwLock>, + embedded_cursors: RwLock<[i64; 2]>, + embedded_permission_cursors: RwLock>, + sync_lock: Mutex<()>, +} + +impl SearchRuntime { + pub(in crate::runtime::backend_runtime) fn new(pool: PgPool, config: SearchRuntimeConfig) -> RuntimeResult { + if !matches!( + config.provider.as_str(), + "embedded" | "elasticsearch" | "manticoresearch" + ) { + return Err(RuntimeError::config("unsupported search provider")); + } + let remote = (config.provider != "embedded") + .then(|| RemoteProvider::new(&config, pool.clone())) + .transpose()?; + Ok(Self { + store: SearchStore::new(pool.clone()), + authorizer: PermissionAuthorizer::new(pool.clone()), + embedded: EmbeddedSearchIndex::new(), + remote, + config, + generation: RwLock::new(None), + embedded_cursors: RwLock::new([0; 2]), + embedded_permission_cursors: RwLock::new(std::collections::HashMap::new()), + sync_lock: Mutex::new(()), + pool, + }) + } + + pub(in crate::runtime::backend_runtime) async fn initialize(&self) -> RuntimeResult<()> { + let stream_count = + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM search_runtime_streams WHERE table_key IN ('doc', 'block')") + .fetch_one(&self.pool) + .await + .map_err(|error| RuntimeError::database("check search runtime streams", error))?; + if stream_count != SearchTable::ORDERED.len() as i64 { + return Ok(()); + } + let active = generation::prepare(&self.pool, &self.config, self.remote.as_ref()).await?; + if let Err(error) = super::worker::rebuild( + &self.pool, + &self.store, + &self.embedded, + self.remote.as_ref(), + &active, + &self.embedded_cursors, + true, + ) + .await + { + generation::fail(&self.pool, &active).await?; + return Err(error); + } + generation::activate(&self.pool, &active).await?; + *self.generation.write().await = Some(active); + self.refresh_all_permission_cursors().await?; + Ok(()) + } + + async fn project_document(&self, workspace_id: &str, doc_id: &str) -> RuntimeResult<()> { + match project_document(&self.pool, workspace_id, doc_id).await? { + Some((document, blocks)) => self.store.replace_document(document, blocks).await?, + None => { + let revision = chrono::Utc::now().timestamp_millis(); + self.store.delete_document(workspace_id, doc_id, revision).await?; + } + } + Ok(()) + } + + pub(in crate::runtime::backend_runtime) async fn project_document_only( + &self, + workspace_id: &str, + doc_id: &str, + ) -> RuntimeResult<()> { + self.project_document(workspace_id, doc_id).await + } + + pub(in crate::runtime::backend_runtime) async fn index_document( + &self, + workspace_id: &str, + doc_id: &str, + ) -> RuntimeResult<()> { + self.project_document(workspace_id, doc_id).await?; + self.sync().await?; + self.refresh_permission_cursor(workspace_id).await + } + + pub(in crate::runtime::backend_runtime) async fn delete_document_only( + &self, + workspace_id: &str, + doc_id: &str, + ) -> RuntimeResult<()> { + self + .store + .delete_document(workspace_id, doc_id, chrono::Utc::now().timestamp_millis()) + .await?; + Ok(()) + } + + pub(in crate::runtime::backend_runtime) async fn delete_document( + &self, + workspace_id: &str, + doc_id: &str, + ) -> RuntimeResult<()> { + self.delete_document_only(workspace_id, doc_id).await?; + self.sync().await?; + self.refresh_permission_cursor(workspace_id).await + } + + pub(in crate::runtime::backend_runtime) async fn search_authorized( + &self, + actor_user_id: &str, + workspace_id: &str, + request: RuntimeSearchRequest, + ) -> RuntimeResult { + let request = request.into_search_request()?; + for attempt in 0..=1 { + let scope = self + .authorizer + .authorize_search( + &SearchActor::User { + user_id: actor_user_id.to_string(), + }, + workspace_id, + ) + .await?; + self + .check_permission_revision(workspace_id, scope.permission_revision) + .await?; + let generation = self.active_generation().await?; + self.ensure_query_ready(&generation).await?; + let dsl = query::compile(&request, &scope)?; + let result = if let Some(remote) = &self.remote { + remote.search(generation.physical_table(request.table)?, dsl).await? + } else { + let result = self + .embedded + .search( + request.table.as_str().to_string(), + serde_json::to_string(&dsl).map_err(|error| RuntimeError::json("encode embedded search", error))?, + ) + .await?; + serde_json::from_str(&result).map_err(|error| RuntimeError::json("decode embedded search", error))? + }; + if self.authorizer.revision(workspace_id).await? == scope.permission_revision { + return Ok(result); + } + if attempt == 1 { + return Err(RuntimeError::SearchPermissionUnavailable); + } + } + unreachable!() + } + + pub(in crate::runtime::backend_runtime) async fn aggregate_authorized( + &self, + actor_user_id: &str, + workspace_id: &str, + request: RuntimeAggregateRequest, + ) -> RuntimeResult { + let request = request.into_aggregate_request()?; + for attempt in 0..=1 { + let scope = self + .authorizer + .authorize_search( + &SearchActor::User { + user_id: actor_user_id.to_string(), + }, + workspace_id, + ) + .await?; + self + .check_permission_revision(workspace_id, scope.permission_revision) + .await?; + let generation = self.active_generation().await?; + self.ensure_query_ready(&generation).await?; + let dsl = query::compile_aggregate(&request, &scope)?; + let result = if let Some(remote) = &self.remote { + remote.aggregate(generation.physical_table(request.table)?, dsl).await? + } else { + let result = self + .embedded + .aggregate( + request.table.as_str().to_string(), + serde_json::to_string(&dsl).map_err(|error| RuntimeError::json("encode embedded aggregate", error))?, + ) + .await?; + let mut value: serde_json::Value = + serde_json::from_str(&result).map_err(|error| RuntimeError::json("decode embedded aggregate", error))?; + if let Some(buckets) = value.get_mut("buckets").and_then(serde_json::Value::as_array_mut) { + for bucket in buckets { + let hits = bucket + .as_object_mut() + .and_then(|bucket| bucket.remove("hits")) + .unwrap_or_else(|| serde_json::json!([])); + bucket["hits"] = serde_json::json!({"nodes":hits}); + } + } + value + }; + if self.authorizer.revision(workspace_id).await? == scope.permission_revision { + return Ok(result); + } + if attempt == 1 { + return Err(RuntimeError::SearchPermissionUnavailable); + } + } + unreachable!() + } + + async fn active_generation(&self) -> RuntimeResult { + if let Some(active) = self.generation.read().await.clone() { + return Ok(active); + } + if let Some(active) = generation::load_active(&self.pool, &self.config).await? { + *self.generation.write().await = Some(active.clone()); + return Ok(active); + } + Err(RuntimeError::invalid_state("search_runtime_not_ready")) + } + + pub(super) async fn sync(&self) -> RuntimeResult<()> { + let _guard = self.sync_lock.lock().await; + let generation = self.active_generation().await?; + let result = super::worker::sync( + &self.pool, + &self.store, + &self.embedded, + self.remote.as_ref(), + &generation, + &self.embedded_cursors, + ) + .await; + if matches!(result, Err(RuntimeError::SearchReplayGap)) && self.remote.is_none() { + return super::worker::rebuild( + &self.pool, + &self.store, + &self.embedded, + None, + &generation, + &self.embedded_cursors, + false, + ) + .await; + } + result + } + + async fn ensure_query_ready(&self, generation: &ActiveGeneration) -> RuntimeResult<()> { + if self.remote.is_none() { + let heads = sqlx::query_as::<_, (String, i64)>( + "SELECT table_key,head FROM search_runtime_streams WHERE table_key IN ('doc','block')", + ) + .fetch_all(&self.pool) + .await + .map_err(|error| RuntimeError::database("load embedded search stream heads", error))?; + let cursors = *self.embedded_cursors.read().await; + for (table, head) in heads { + let cursor = cursors[if table == "doc" { 0 } else { 1 }]; + if cursor != head { + return Err(RuntimeError::SearchProviderUnavailable); + } + } + return Ok(()); + } + + let rows = sqlx::query_as::<_, (String, i64, i64)>( + "SELECT streams.table_key,streams.head,cursors.source_cursor + FROM search_runtime_streams streams + JOIN search_runtime_provider_cursors cursors + ON cursors.table_key=streams.table_key AND cursors.generation_id=$1 + WHERE streams.table_key IN ('doc','block')", + ) + .bind(generation.id) + .fetch_all(&self.pool) + .await + .map_err(|error| RuntimeError::database("load remote search readiness", error))?; + if rows.len() != SearchTable::ORDERED.len() || rows.iter().any(|(_, head, cursor)| head != cursor) { + return Err(RuntimeError::SearchProviderUnavailable); + } + Ok(()) + } + + async fn check_permission_revision(&self, workspace_id: &str, revision: i64) -> RuntimeResult<()> { + let generation = self.active_generation().await?; + let applied = if self.remote.is_none() { + self.embedded_permission_cursors.read().await.get(workspace_id).copied() + } else { + sqlx::query_scalar( + "SELECT permission_revision FROM search_runtime_permission_cursors WHERE generation_id=$1 AND workspace_id=$2", + ) + .bind(generation.id) + .bind(workspace_id) + .fetch_optional(&self.pool) + .await + .map_err(|error| RuntimeError::database("load search permission cursor", error))? + }; + let applied = applied.unwrap_or(-1); + if applied >= revision { + return Ok(()); + } + Err(RuntimeError::SearchPermissionUnavailable) + } + + async fn refresh_permission_cursor(&self, workspace_id: &str) -> RuntimeResult<()> { + let revision: i64 = sqlx::query_scalar( + "SELECT coalesce(max(revision),0)::bigint FROM workspace_permission_changes WHERE workspace_id=$1", + ) + .bind(workspace_id) + .fetch_one(&self.pool) + .await + .map_err(|error| RuntimeError::database("load search permission revision", error))?; + let generation = self.active_generation().await?; + if self.remote.is_none() { + let mut cursors = self.embedded_permission_cursors.write().await; + let applied = cursors.entry(workspace_id.to_string()).or_insert(revision); + *applied = (*applied).max(revision); + } else { + sqlx::query( + r#"INSERT INTO search_runtime_permission_cursors(generation_id,workspace_id,permission_revision) + VALUES ($1,$2,$3) ON CONFLICT (generation_id,workspace_id) DO UPDATE SET + permission_revision=GREATEST(search_runtime_permission_cursors.permission_revision,EXCLUDED.permission_revision), updated_at=now()"#, + ) + .bind(generation.id) + .bind(workspace_id) + .bind(revision) + .execute(&self.pool) + .await + .map_err(|error| RuntimeError::database("advance search permission cursor", error))?; + } + Ok(()) + } + + async fn refresh_all_permission_cursors(&self) -> RuntimeResult<()> { + let workspace_ids: Vec = sqlx::query_scalar("SELECT id FROM workspaces") + .fetch_all(&self.pool) + .await + .map_err(|error| RuntimeError::database("load search permission workspaces", error))?; + for workspace_id in workspace_ids { + self.refresh_permission_cursor(&workspace_id).await?; + } + Ok(()) + } + + pub(in crate::runtime::backend_runtime) async fn reconcile_workspace( + &self, + capability: SystemSearchCapability, + workspace_id: &str, + ) -> RuntimeResult<()> { + match capability { + SystemSearchCapability::ReconcileIndex => {} + } + let doc_ids: Vec = sqlx::query_scalar("SELECT page_id FROM workspace_pages WHERE workspace_id=$1") + .bind(workspace_id) + .fetch_all(&self.pool) + .await + .map_err(|error| RuntimeError::database("load search workspace documents", error))?; + let indexed_doc_ids: Vec = + sqlx::query_scalar("SELECT DISTINCT doc_id FROM search_runtime_projections WHERE workspace_id=$1") + .bind(workspace_id) + .fetch_all(&self.pool) + .await + .map_err(|error| RuntimeError::database("load indexed workspace documents", error))?; + let live_doc_ids = doc_ids.iter().cloned().collect::>(); + for doc_id in doc_ids { + self.project_document(workspace_id, &doc_id).await?; + } + let deletion_revision = chrono::Utc::now().timestamp_millis(); + for doc_id in indexed_doc_ids { + if !live_doc_ids.contains(&doc_id) { + self + .store + .delete_document(workspace_id, &doc_id, deletion_revision) + .await?; + } + } + self.sync().await?; + self.refresh_permission_cursor(workspace_id).await + } + + pub(in crate::runtime::backend_runtime) async fn delete_workspace(&self, workspace_id: &str) -> RuntimeResult<()> { + let doc_ids: Vec = + sqlx::query_scalar("SELECT DISTINCT doc_id FROM search_runtime_projections WHERE workspace_id=$1") + .bind(workspace_id) + .fetch_all(&self.pool) + .await + .map_err(|error| RuntimeError::database("load indexed workspace documents", error))?; + for doc_id in doc_ids { + self + .store + .delete_document(workspace_id, &doc_id, chrono::Utc::now().timestamp_millis()) + .await?; + } + self.sync().await?; + self.refresh_permission_cursor(workspace_id).await + } + + pub(in crate::runtime::backend_runtime) async fn status(&self) -> RuntimeResult { + let generation = match self.active_generation().await { + Ok(generation) => generation, + Err(RuntimeError::InvalidState(message)) if message == "search_runtime_not_ready" => { + return Ok(serde_json::json!({ + "ready": false, + "provider": self.config.provider, + "tables": [], + })); + } + Err(error) => return Err(error), + }; + use sqlx::Row; + let heads = sqlx::query( + "SELECT table_key,head FROM search_runtime_streams WHERE table_key IN ('doc','block') ORDER BY table_key", + ) + .fetch_all(&self.pool) + .await + .map_err(|error| RuntimeError::database("load search runtime heads", error))?; + let local_cursors = *self.embedded_cursors.read().await; + let mut tables = Vec::with_capacity(heads.len()); + for row in heads { + let table_key: String = row.get("table_key"); + let head: i64 = row.get("head"); + let cursor = if self.remote.is_none() { + local_cursors[if table_key == "doc" { 0 } else { 1 }] + } else { + sqlx::query_scalar( + "SELECT source_cursor FROM search_runtime_provider_cursors WHERE generation_id=$1 AND table_key=$2", + ) + .bind(generation.id) + .bind(&table_key) + .fetch_one(&self.pool) + .await + .map_err(|error| RuntimeError::database("load search provider cursor", error))? + }; + tables.push(serde_json::json!({"table":table_key,"head":head,"cursor":cursor,"lag":head-cursor})); + } + Ok(serde_json::json!({ + "ready":tables.len()==2 && tables.iter().all(|table|table["lag"]==0), + "generationId":generation.id.to_string(), + "provider":self.config.provider, + "tables":tables, + })) + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/store/mod.rs b/packages/backend/native/src/runtime/backend_runtime/search/store/mod.rs new file mode 100644 index 0000000000..3ea8897e7a --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/store/mod.rs @@ -0,0 +1,6 @@ +mod projection; +pub(super) mod stream; +mod types; + +pub(super) use projection::SearchStore; +pub(super) use types::{ProjectionInput, SearchChange, SearchSnapshot, SearchTable}; diff --git a/packages/backend/native/src/runtime/backend_runtime/search/store/projection.rs b/packages/backend/native/src/runtime/backend_runtime/search/store/projection.rs new file mode 100644 index 0000000000..545dcf8925 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/store/projection.rs @@ -0,0 +1,450 @@ +use std::collections::{HashMap, HashSet}; + +use serde_json::Value; +use sqlx::{PgPool, Postgres, Row, Transaction}; + +use super::{ProjectionInput, SearchChange, SearchSnapshot, SearchTable, stream::allocate}; +use crate::runtime::{RuntimeError, RuntimeResult}; + +pub(in crate::runtime::backend_runtime::search) struct SearchStore { + pool: PgPool, +} + +impl SearchStore { + pub(in crate::runtime::backend_runtime::search) fn new(pool: PgPool) -> Self { + Self { pool } + } + + pub(in crate::runtime::backend_runtime::search) async fn replace_document( + &self, + mut document: ProjectionInput, + mut blocks: Vec, + ) -> RuntimeResult<()> { + if blocks + .iter() + .any(|block| block.workspace_id != document.workspace_id || block.doc_id != document.doc_id) + { + return Err(RuntimeError::invalid_input("block identity does not match document")); + } + let mut transaction = self + .pool + .begin() + .await + .map_err(|error| RuntimeError::database("begin search projection transaction", error))?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(format!("{}/{}", document.workspace_id, document.doc_id)) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("lock search document projection", error))?; + + let current_document = load_rows( + &mut transaction, + SearchTable::Doc, + &document.workspace_id, + &document.doc_id, + ) + .await?; + if let Some(existing) = current_document.get(&document.external_id) + && existing != &document + && existing.acl_revision < document.acl_revision + { + document.revision = existing.revision + 1; + for block in &mut blocks { + block.revision = document.revision; + } + } + if let Some(existing) = current_document.get(&document.external_id) { + if existing.revision > document.revision { + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit stale search projection", error))?; + return Ok(()); + } + if existing.revision == document.revision { + let current_blocks = load_rows( + &mut transaction, + SearchTable::Block, + &document.workspace_id, + &document.doc_id, + ) + .await?; + let incoming_blocks = blocks + .iter() + .map(|block| (block.external_id.clone(), block)) + .collect::>(); + let blocks_match = current_blocks.len() == incoming_blocks.len() + && current_blocks + .iter() + .all(|(id, block)| incoming_blocks.get(id).is_some_and(|incoming| block == *incoming)); + if existing != &document || !blocks_match { + return Err(RuntimeError::invalid_state("conflicting search projection revision")); + } + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit duplicate search projection", error))?; + return Ok(()); + } + } + + self + .replace_table( + &mut transaction, + SearchTable::Doc, + vec![document.clone()], + Some(document.revision), + ) + .await?; + self + .replace_table(&mut transaction, SearchTable::Block, blocks, Some(document.revision)) + .await?; + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit search projection transaction", error)) + } + + pub(in crate::runtime::backend_runtime::search) async fn delete_document( + &self, + workspace_id: &str, + doc_id: &str, + revision: i64, + ) -> RuntimeResult<()> { + let mut transaction = self + .pool + .begin() + .await + .map_err(|error| RuntimeError::database("begin search delete transaction", error))?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(format!("{workspace_id}/{doc_id}")) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("lock search document deletion", error))?; + for table in SearchTable::ORDERED { + let rows = load_rows(&mut transaction, table, workspace_id, doc_id).await?; + let deletions = rows + .into_values() + .filter(|row| row.revision <= revision) + .collect::>(); + self + .apply(&mut transaction, table, Vec::new(), deletions, revision) + .await?; + } + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit search delete transaction", error)) + } + + async fn replace_table( + &self, + transaction: &mut Transaction<'_, Postgres>, + table: SearchTable, + inputs: Vec, + delete_revision: Option, + ) -> RuntimeResult<()> { + let Some(first) = inputs.first() else { + return Ok(()); + }; + let current = load_rows(transaction, table, &first.workspace_id, &first.doc_id).await?; + let input_ids = inputs + .iter() + .map(|input| input.external_id.clone()) + .collect::>(); + let mut upserts = Vec::new(); + for input in inputs { + match current.get(&input.external_id) { + Some(existing) if existing.revision > input.revision => continue, + Some(existing) if existing.revision == input.revision => { + if existing != &input { + return Err(RuntimeError::invalid_state("conflicting search projection revision")); + } + } + Some(existing) if existing == &input => {} + _ => upserts.push(input), + } + } + let deletions = current + .into_values() + .filter(|row| { + !input_ids.contains(row.external_id.as_str()) + && delete_revision.is_some_and(|revision| row.revision <= revision) + }) + .collect(); + self + .apply( + transaction, + table, + upserts, + deletions, + delete_revision.unwrap_or_default(), + ) + .await + } + + async fn apply( + &self, + transaction: &mut Transaction<'_, Postgres>, + table: SearchTable, + upserts: Vec, + deletions: Vec, + delete_revision: i64, + ) -> RuntimeResult<()> { + let first_sequence = allocate(transaction, table, upserts.len() + deletions.len()).await?; + let mut sequence = first_sequence; + for input in upserts { + insert_change( + transaction, + table, + sequence, + "upsert", + &input, + Some(&input.payload), + input.revision, + ) + .await?; + sqlx::query( + r#"INSERT INTO search_runtime_projections + (table_key, external_id, workspace_id, doc_id, revision, payload, + acl_public_readable, acl_member_default_readable, acl_read_user_ids, acl_revision) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) + ON CONFLICT (table_key, external_id) DO UPDATE SET + workspace_id=EXCLUDED.workspace_id, doc_id=EXCLUDED.doc_id, + revision=EXCLUDED.revision, payload=EXCLUDED.payload, + acl_public_readable=EXCLUDED.acl_public_readable, + acl_member_default_readable=EXCLUDED.acl_member_default_readable, + acl_read_user_ids=EXCLUDED.acl_read_user_ids, + acl_revision=EXCLUDED.acl_revision, updated_at=now() + WHERE search_runtime_projections.revision < EXCLUDED.revision"#, + ) + .bind(table.as_str()) + .bind(&input.external_id) + .bind(&input.workspace_id) + .bind(&input.doc_id) + .bind(input.revision) + .bind(&input.payload) + .bind(input.acl_public_readable) + .bind(input.acl_member_default_readable) + .bind(&input.acl_read_user_ids) + .bind(input.acl_revision) + .execute(&mut **transaction) + .await + .map_err(|error| RuntimeError::database("upsert search projection", error))?; + sequence += 1; + } + for input in deletions { + insert_change( + transaction, + table, + sequence, + "delete", + &input, + Some(&input.payload), + delete_revision, + ) + .await?; + sqlx::query("DELETE FROM search_runtime_projections WHERE table_key=$1 AND external_id=$2 AND revision <= $3") + .bind(table.as_str()) + .bind(&input.external_id) + .bind(delete_revision) + .execute(&mut **transaction) + .await + .map_err(|error| RuntimeError::database("delete search projection", error))?; + sequence += 1; + } + Ok(()) + } + + pub(in crate::runtime::backend_runtime::search) async fn snapshot( + &self, + table: SearchTable, + ) -> RuntimeResult { + let mut transaction = self + .pool + .begin() + .await + .map_err(|error| RuntimeError::database("begin search snapshot", error))?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("configure search snapshot", error))?; + let head = sqlx::query_scalar("SELECT head FROM search_runtime_streams WHERE table_key=$1") + .bind(table.as_str()) + .fetch_one(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("read search snapshot head", error))?; + let rows = sqlx::query( + r#"SELECT external_id, workspace_id, doc_id, revision, payload, + acl_public_readable, acl_member_default_readable, acl_read_user_ids, acl_revision + FROM search_runtime_projections WHERE table_key=$1 ORDER BY external_id"#, + ) + .bind(table.as_str()) + .fetch_all(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("read search projection snapshot", error))?; + let projections = rows.iter().map(decode_projection).collect::>>()?; + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("commit search snapshot", error))?; + Ok(SearchSnapshot { head, projections }) + } + + pub(in crate::runtime::backend_runtime::search) async fn changes( + &self, + table: SearchTable, + after: i64, + limit: i64, + ) -> RuntimeResult<(i64, Vec)> { + if after < 0 || limit <= 0 { + return Err(RuntimeError::invalid_input("invalid search replay cursor")); + } + let state = sqlx::query("SELECT head, retained_from FROM search_runtime_streams WHERE table_key=$1") + .bind(table.as_str()) + .fetch_one(&self.pool) + .await + .map_err(|error| RuntimeError::database("read search stream state", error))?; + let head: i64 = state + .try_get("head") + .map_err(|error| RuntimeError::database("decode stream head", error))?; + let retained_from: i64 = state + .try_get("retained_from") + .map_err(|error| RuntimeError::database("decode retained cursor", error))?; + if after < retained_from { + return Err(RuntimeError::SearchReplayGap); + } + let rows = sqlx::query( + r#"SELECT stream_sequence, external_id, workspace_id, doc_id, revision, operation, payload + FROM search_runtime_changes WHERE table_key=$1 AND stream_sequence>$2 + ORDER BY stream_sequence LIMIT $3"#, + ) + .bind(table.as_str()) + .bind(after) + .bind(limit) + .fetch_all(&self.pool) + .await + .map_err(|error| RuntimeError::database("read search stream changes", error))?; + if after < head + && rows + .first() + .and_then(|row| row.try_get::("stream_sequence").ok()) + != Some(after + 1) + { + return Err(RuntimeError::SearchReplayGap); + } + let changes = rows + .into_iter() + .map(|row| { + Ok(SearchChange { + sequence: row + .try_get("stream_sequence") + .map_err(|error| RuntimeError::database("decode change sequence", error))?, + external_id: row + .try_get("external_id") + .map_err(|error| RuntimeError::database("decode change external id", error))?, + workspace_id: row + .try_get("workspace_id") + .map_err(|error| RuntimeError::database("decode change workspace", error))?, + doc_id: row + .try_get("doc_id") + .map_err(|error| RuntimeError::database("decode change doc", error))?, + revision: row + .try_get("revision") + .map_err(|error| RuntimeError::database("decode change revision", error))?, + operation: row + .try_get("operation") + .map_err(|error| RuntimeError::database("decode change operation", error))?, + payload: row + .try_get("payload") + .map_err(|error| RuntimeError::database("decode change payload", error))?, + }) + }) + .collect::>>()?; + Ok((head, changes)) + } +} + +async fn load_rows( + transaction: &mut Transaction<'_, Postgres>, + table: SearchTable, + workspace_id: &str, + doc_id: &str, +) -> RuntimeResult> { + let rows = sqlx::query( + r#"SELECT external_id, workspace_id, doc_id, revision, payload, + acl_public_readable, acl_member_default_readable, acl_read_user_ids, acl_revision + FROM search_runtime_projections + WHERE table_key=$1 AND workspace_id=$2 AND doc_id=$3 FOR UPDATE"#, + ) + .bind(table.as_str()) + .bind(workspace_id) + .bind(doc_id) + .fetch_all(&mut **transaction) + .await + .map_err(|error| RuntimeError::database("load search projections", error))?; + rows + .iter() + .map(|row| decode_projection(row).map(|projection| (projection.external_id.clone(), projection))) + .collect() +} + +fn decode_projection(row: &sqlx::postgres::PgRow) -> RuntimeResult { + Ok(ProjectionInput { + external_id: row + .try_get("external_id") + .map_err(|error| RuntimeError::database("decode projection id", error))?, + workspace_id: row + .try_get("workspace_id") + .map_err(|error| RuntimeError::database("decode projection workspace", error))?, + doc_id: row + .try_get("doc_id") + .map_err(|error| RuntimeError::database("decode projection doc", error))?, + revision: row + .try_get("revision") + .map_err(|error| RuntimeError::database("decode projection revision", error))?, + payload: row + .try_get("payload") + .map_err(|error| RuntimeError::database("decode projection payload", error))?, + acl_public_readable: row + .try_get("acl_public_readable") + .map_err(|error| RuntimeError::database("decode projection public ACL", error))?, + acl_member_default_readable: row + .try_get("acl_member_default_readable") + .map_err(|error| RuntimeError::database("decode projection member ACL", error))?, + acl_read_user_ids: row + .try_get("acl_read_user_ids") + .map_err(|error| RuntimeError::database("decode projection ACL users", error))?, + acl_revision: row + .try_get("acl_revision") + .map_err(|error| RuntimeError::database("decode projection ACL revision", error))?, + }) +} + +async fn insert_change( + transaction: &mut Transaction<'_, Postgres>, + table: SearchTable, + sequence: i64, + operation: &str, + input: &ProjectionInput, + payload: Option<&Value>, + revision: i64, +) -> RuntimeResult<()> { + sqlx::query( + r#"INSERT INTO search_runtime_changes + (table_key,stream_sequence,external_id,workspace_id,doc_id,revision,operation,payload) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8)"#, + ) + .bind(table.as_str()) + .bind(sequence) + .bind(&input.external_id) + .bind(&input.workspace_id) + .bind(&input.doc_id) + .bind(revision) + .bind(operation) + .bind(payload) + .execute(&mut **transaction) + .await + .map_err(|error| RuntimeError::database("insert search stream change", error))?; + Ok(()) +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/store/stream.rs b/packages/backend/native/src/runtime/backend_runtime/search/store/stream.rs new file mode 100644 index 0000000000..d0b71fb1ca --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/store/stream.rs @@ -0,0 +1,26 @@ +use sqlx::{Postgres, Row, Transaction}; + +use super::SearchTable; +use crate::runtime::{RuntimeError, RuntimeResult}; + +pub(in crate::runtime::backend_runtime::search) async fn allocate( + transaction: &mut Transaction<'_, Postgres>, + table: SearchTable, + count: usize, +) -> RuntimeResult { + if count == 0 { + return Ok(0); + } + let row = sqlx::query( + "UPDATE search_runtime_streams SET head = head + $2, updated_at = now() WHERE table_key = $1 RETURNING head", + ) + .bind(table.as_str()) + .bind(count as i64) + .fetch_one(&mut **transaction) + .await + .map_err(|error| RuntimeError::database("allocate search stream sequence", error))?; + let head: i64 = row + .try_get("head") + .map_err(|error| RuntimeError::database("decode search stream head", error))?; + Ok(head - count as i64 + 1) +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/store/types.rs b/packages/backend/native/src/runtime/backend_runtime/search/store/types.rs new file mode 100644 index 0000000000..afb273d2d4 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/store/types.rs @@ -0,0 +1,54 @@ +use serde_json::Value; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::runtime::backend_runtime::search) enum SearchTable { + Doc, + Block, +} + +impl SearchTable { + pub(in crate::runtime::backend_runtime::search) const ORDERED: [Self; 2] = [Self::Doc, Self::Block]; + + pub(in crate::runtime::backend_runtime::search) fn as_str(self) -> &'static str { + match self { + Self::Doc => "doc", + Self::Block => "block", + } + } + + pub(in crate::runtime::backend_runtime::search) fn cursor_index(self) -> usize { + match self { + Self::Doc => 0, + Self::Block => 1, + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub(in crate::runtime::backend_runtime::search) struct ProjectionInput { + pub(in crate::runtime::backend_runtime::search) external_id: String, + pub(in crate::runtime::backend_runtime::search) workspace_id: String, + pub(in crate::runtime::backend_runtime::search) doc_id: String, + pub(in crate::runtime::backend_runtime::search) revision: i64, + pub(in crate::runtime::backend_runtime::search) payload: Value, + pub(in crate::runtime::backend_runtime::search) acl_public_readable: bool, + pub(in crate::runtime::backend_runtime::search) acl_member_default_readable: bool, + pub(in crate::runtime::backend_runtime::search) acl_read_user_ids: Vec, + pub(in crate::runtime::backend_runtime::search) acl_revision: i64, +} + +#[derive(Clone, Debug, PartialEq)] +pub(in crate::runtime::backend_runtime::search) struct SearchChange { + pub(in crate::runtime::backend_runtime::search) sequence: i64, + pub(in crate::runtime::backend_runtime::search) external_id: String, + pub(in crate::runtime::backend_runtime::search) workspace_id: String, + pub(in crate::runtime::backend_runtime::search) doc_id: Option, + pub(in crate::runtime::backend_runtime::search) revision: i64, + pub(in crate::runtime::backend_runtime::search) operation: String, + pub(in crate::runtime::backend_runtime::search) payload: Option, +} + +pub(in crate::runtime::backend_runtime::search) struct SearchSnapshot { + pub(in crate::runtime::backend_runtime::search) head: i64, + pub(in crate::runtime::backend_runtime::search) projections: Vec, +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/tests.rs b/packages/backend/native/src/runtime/backend_runtime/search/tests.rs new file mode 100644 index 0000000000..5cbe39200b --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/tests.rs @@ -0,0 +1,635 @@ +use serde_json::json; +use sqlx::PgPool; + +use super::{ + SearchRuntime, generation, + projection::project_document, + provider::RemoteProvider, + query, + store::{ProjectionInput, SearchChange, SearchStore, SearchTable, stream::allocate}, + types::SearchRequest, +}; +use crate::runtime::{ + SearchRuntimeConfig, + backend_runtime::permission::{AclPredicate, AuthorizedSearchScope, DocReadScope}, + migrations::migrate_search_tables, +}; + +static SEARCH_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +#[test] +fn canonical_query_injects_workspace_and_projected_acl() { + let request = SearchRequest::parse(serde_json::json!({ + "table":"block", + "query":{"type":"match","field":"content","match":"hello"}, + "options":{"fields":["docId","content"],"pagination":{"limit":20}} + })) + .unwrap(); + let scope = AuthorizedSearchScope { + workspace_id: "workspace".to_string(), + permission_revision: 7, + docs: DocReadScope::ProjectedAcl(AclPredicate { + actor_user_id: "user".to_string(), + active_member: true, + sharing_enabled: false, + }), + }; + let dsl = query::compile(&request, &scope).unwrap(); + assert_eq!(dsl["size"], 20); + assert_eq!( + dsl["query"]["bool"]["must"][0]["term"]["workspace_id"]["value"], + "workspace" + ); + let acl = &dsl["query"]["bool"]["must"][2]["bool"]["should"]; + assert_eq!(acl.as_array().unwrap().len(), 2); + assert!(dsl.to_string().contains("acl_read_tokens")); +} + +fn projection(table: SearchTable, id: &str, revision: i64) -> ProjectionInput { + let block_id = (table == SearchTable::Block).then_some(id); + ProjectionInput { + external_id: id.to_string(), + workspace_id: "search-runtime-test-workspace".to_string(), + doc_id: "search-runtime-test-doc".to_string(), + revision, + payload: json!({ + "workspace_id": "search-runtime-test-workspace", + "doc_id": "search-runtime-test-doc", + "block_id": block_id, + "revision": revision, + }), + acl_public_readable: false, + acl_member_default_readable: true, + acl_read_user_ids: vec!["search-runtime-test-user".to_string()], + acl_revision: revision, + } +} + +async fn pool() -> Option { + let database_url = std::env::var("DATABASE_URL").ok()?; + let pool = PgPool::connect(&database_url).await.unwrap(); + migrate_search_tables(&pool).await.unwrap(); + sqlx::raw_sql( + "DELETE FROM search_runtime_changes; DELETE FROM search_runtime_projections; UPDATE search_runtime_streams SET \ + head=0, retained_from=0", + ) + .execute(&pool) + .await + .unwrap(); + Some(pool) +} + +#[tokio::test] +async fn projection_replace_replay_delete_and_stale_revision_are_monotonic() { + let _guard = SEARCH_TEST_LOCK.lock().await; + let Some(pool) = pool().await else { return }; + let store = SearchStore::new(pool); + + store + .replace_document( + projection(SearchTable::Doc, "doc", 2), + vec![projection(SearchTable::Block, "block-a", 2)], + ) + .await + .unwrap(); + store + .replace_document( + projection(SearchTable::Doc, "doc", 1), + vec![projection(SearchTable::Block, "block-stale", 1)], + ) + .await + .unwrap(); + store + .replace_document( + projection(SearchTable::Doc, "doc", 2), + vec![projection(SearchTable::Block, "block-a", 2)], + ) + .await + .unwrap(); + + let doc = store.snapshot(SearchTable::Doc).await.unwrap(); + let block = store.snapshot(SearchTable::Block).await.unwrap(); + assert_eq!((doc.head, doc.projections.len()), (1, 1)); + assert_eq!((block.head, block.projections.len()), (1, 1)); + let (_, changes) = store.changes(SearchTable::Block, 0, 10).await.unwrap(); + assert_eq!(changes.len(), 1); + assert_eq!(changes[0].external_id, "block-a"); + + store + .delete_document("search-runtime-test-workspace", "search-runtime-test-doc", 3) + .await + .unwrap(); + assert!(store.snapshot(SearchTable::Doc).await.unwrap().projections.is_empty()); + assert!(store.snapshot(SearchTable::Block).await.unwrap().projections.is_empty()); + let (_, changes) = store.changes(SearchTable::Block, 1, 10).await.unwrap(); + assert_eq!(changes[0].operation, "delete"); + assert_eq!(changes[0].revision, 3); +} + +#[tokio::test] +async fn document_projection_loads_snapshot_metadata_and_search_units() { + let _guard = SEARCH_TEST_LOCK.lock().await; + let Some(pool) = pool().await else { return }; + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let workspace_id = format!("search-projection-workspace-{suffix}"); + let user_id = format!("search-projection-user-{suffix}"); + let doc_id = format!("search-projection-doc-{suffix}"); + sqlx::query( + "INSERT INTO users(id,name,email,registered,email_verified,disabled) VALUES($1,'Search Projection \ + User',$2,true,now(),false)", + ) + .bind(&user_id) + .bind(format!("search-projection-{suffix}@example.com")) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO workspaces(id) VALUES($1)") + .bind(&workspace_id) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO workspace_access_policies(workspace_id) VALUES($1) ON CONFLICT DO NOTHING") + .bind(&workspace_id) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO workspace_members(workspace_id,user_id,role,state) VALUES($1,$2,'owner','active')") + .bind(&workspace_id) + .bind(&user_id) + .execute(&pool) + .await + .unwrap(); + let blob = affine_doc_loader::build_full_doc( + "Projection title", + "Projection body\n\n![Asset](blob://projection-blob)", + &doc_id, + ) + .unwrap(); + sqlx::query( + "INSERT INTO snapshots(workspace_id,guid,blob,created_by,updated_by,updated_at) \ + VALUES($1,$2,$3,$4,$4,clock_timestamp())", + ) + .bind(&workspace_id) + .bind(&doc_id) + .bind(blob) + .bind(&user_id) + .execute(&pool) + .await + .unwrap(); + + let (document, blocks) = project_document(&pool, &workspace_id, &doc_id).await.unwrap().unwrap(); + assert_eq!(document.payload["title"], "Projection title"); + assert_eq!(document.payload["created_by_user_id"], user_id); + assert!( + document.payload["summary"] + .as_str() + .unwrap() + .contains("Projection body") + ); + assert!(document.acl_revision > 0); + assert!(blocks.iter().any(|block| block.payload["content"] == "Projection body")); + assert!(blocks.iter().any(|block| block.payload["blob"] == "projection-blob")); + assert!( + blocks + .iter() + .all(|block| block.payload["acl_revision"] == document.payload["acl_revision"]) + ); +} + +#[tokio::test] +async fn stream_sequence_follows_commit_order_and_rollback_has_no_gap() { + let _guard = SEARCH_TEST_LOCK.lock().await; + let Some(pool) = pool().await else { return }; + + let mut first = pool.begin().await.unwrap(); + assert_eq!(allocate(&mut first, SearchTable::Doc, 1).await.unwrap(), 1); + let second_pool = pool.clone(); + let second = tokio::spawn(async move { + let mut transaction = second_pool.begin().await.unwrap(); + let sequence = allocate(&mut transaction, SearchTable::Doc, 1).await.unwrap(); + transaction.commit().await.unwrap(); + sequence + }); + tokio::task::yield_now().await; + let visible_head: i64 = sqlx::query_scalar("SELECT head FROM search_runtime_streams WHERE table_key='doc'") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(visible_head, 0); + first.commit().await.unwrap(); + assert_eq!(second.await.unwrap(), 2); + + sqlx::query("UPDATE search_runtime_streams SET head=0 WHERE table_key='doc'") + .execute(&pool) + .await + .unwrap(); + let mut rolled_back = pool.begin().await.unwrap(); + assert_eq!(allocate(&mut rolled_back, SearchTable::Doc, 1).await.unwrap(), 1); + rolled_back.rollback().await.unwrap(); + let mut committed = pool.begin().await.unwrap(); + assert_eq!(allocate(&mut committed, SearchTable::Doc, 1).await.unwrap(), 1); + committed.commit().await.unwrap(); +} + +#[tokio::test] +async fn replay_rejects_retention_gap() { + let _guard = SEARCH_TEST_LOCK.lock().await; + let Some(pool) = pool().await else { return }; + sqlx::query("UPDATE search_runtime_streams SET head=5, retained_from=3 WHERE table_key='doc'") + .execute(&pool) + .await + .unwrap(); + let error = SearchStore::new(pool) + .changes(SearchTable::Doc, 2, 10) + .await + .unwrap_err(); + assert!(matches!(error, crate::runtime::RuntimeError::SearchReplayGap)); +} + +#[tokio::test] +async fn concurrent_generation_prepare_reuses_pending_and_restart_preserves_active() { + let _guard = SEARCH_TEST_LOCK.lock().await; + let Some(pool) = pool().await else { return }; + sqlx::query("DELETE FROM search_runtime_generations") + .execute(&pool) + .await + .unwrap(); + let config = SearchRuntimeConfig::default(); + let (first, second) = tokio::join!( + generation::prepare(&pool, &config, None), + generation::prepare(&pool, &config, None) + ); + let first = first.unwrap(); + let second = second.unwrap(); + assert_eq!(first.id, second.id); + generation::activate(&pool, &first).await.unwrap(); + let restarted = generation::prepare(&pool, &config, None).await.unwrap(); + assert_eq!(restarted.id, first.id); + generation::activate(&pool, &restarted).await.unwrap(); + let active: i64 = sqlx::query_scalar("SELECT count(*) FROM search_runtime_generations WHERE state='active'") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(active, 1); + let unavailable = SearchRuntimeConfig { + provider: "elasticsearch".into(), + endpoint: "http://127.0.0.1:1".into(), + ..Default::default() + }; + let remote = RemoteProvider::new(&unavailable, pool.clone()).unwrap(); + assert!(generation::prepare(&pool, &unavailable, Some(&remote)).await.is_err()); + let states: (i64, i64, i64) = sqlx::query_as( + "SELECT count(*) FILTER (WHERE state='active'), count(*) FILTER (WHERE state='pending'), count(*) FILTER (WHERE \ + state='failed') FROM search_runtime_generations", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(states, (1, 0, 1)); +} + +#[tokio::test] +async fn embedded_replicas_share_one_checkpoint_and_rebuild_a_corrupt_snapshot() { + let _guard = SEARCH_TEST_LOCK.lock().await; + let Some(pool) = pool().await else { return }; + sqlx::raw_sql( + "DELETE FROM search_runtime_generations; DELETE FROM search_runtime_checkpoints; DELETE FROM \ + search_runtime_changes; DELETE FROM search_runtime_projections; UPDATE search_runtime_streams SET head=0, \ + retained_from=0", + ) + .execute(&pool) + .await + .unwrap(); + SearchStore::new(pool.clone()) + .replace_document( + ProjectionInput { + payload: json!({ + "workspace_id":"replica-workspace","doc_id":"replica-doc","title":"replica search", + "summary":"","created_by_user_id":"user","updated_by_user_id":"user", + "created_at":1,"updated_at":1,"acl_public_readable":false, + "acl_member_default_readable":true,"acl_read_tokens":["member"], + "acl_revision":1 + }), + external_id: "replica-workspace/replica-doc".into(), + workspace_id: "replica-workspace".into(), + doc_id: "replica-doc".into(), + revision: 1, + acl_public_readable: false, + acl_member_default_readable: true, + acl_read_user_ids: vec![], + acl_revision: 1, + }, + vec![], + ) + .await + .unwrap(); + let config = SearchRuntimeConfig::default(); + let first = SearchRuntime::new(pool.clone(), config.clone()).unwrap(); + let second = SearchRuntime::new(pool.clone(), config.clone()).unwrap(); + let third = SearchRuntime::new(pool.clone(), config.clone()).unwrap(); + let (first_result, second_result, third_result) = + tokio::join!(first.initialize(), second.initialize(), third.initialize()); + first_result.unwrap(); + second_result.unwrap(); + third_result.unwrap(); + SearchStore::new(pool.clone()) + .replace_document( + ProjectionInput { + payload: json!({ + "workspace_id":"replica-workspace","doc_id":"replica-doc-2","title":"replica search second", + "summary":"","created_by_user_id":"user","updated_by_user_id":"user", + "created_at":2,"updated_at":2,"acl_public_readable":false, + "acl_member_default_readable":true,"acl_read_tokens":["member"], + "acl_revision":1 + }), + external_id: "replica-workspace/replica-doc-2".into(), + workspace_id: "replica-workspace".into(), + doc_id: "replica-doc-2".into(), + revision: 2, + acl_public_readable: false, + acl_member_default_readable: true, + acl_read_user_ids: vec![], + acl_revision: 1, + }, + vec![], + ) + .await + .unwrap(); + first.sync().await.unwrap(); + sqlx::query("UPDATE search_runtime_streams SET retained_from=head WHERE table_key='doc'") + .execute(&pool) + .await + .unwrap(); + second.sync().await.unwrap(); + let second_replica_result: serde_json::Value = serde_json::from_str( + &second + .embedded + .search( + "doc".into(), + json!({"query":{"match_all":{}},"fields":["doc_id"],"sort":["doc_id"],"size":10}).to_string(), + ) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(second_replica_result["total"], 2); + let checkpoints: i64 = sqlx::query_scalar("SELECT count(*) FROM search_runtime_checkpoints") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(checkpoints, 1); + sqlx::query("UPDATE search_runtime_checkpoints SET checkpoint_blob='\\x010203' WHERE table_key='doc'") + .execute(&pool) + .await + .unwrap(); + let recovered = SearchRuntime::new(pool, config).unwrap(); + recovered.initialize().await.unwrap(); + let result: serde_json::Value = serde_json::from_str( + &recovered + .embedded + .search( + "doc".into(), + json!({"query":{"match_all":{}},"fields":["doc_id"],"sort":["doc_id"],"size":10}).to_string(), + ) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(result["total"], 2); +} + +#[tokio::test] +async fn remote_providers_apply_search_and_delete_the_same_contract() { + let _guard = SEARCH_TEST_LOCK.lock().await; + let require_remote = std::env::var("SEARCH_REQUIRE_REMOTE_TESTS").as_deref() == Ok("1"); + let pool = match pool().await { + Some(pool) => pool, + None if require_remote => panic!("DATABASE_URL is required"), + None => return, + }; + let mut tested_providers = 0; + for (provider, variable) in [("elasticsearch", "SEARCH_ES_URL"), ("manticoresearch", "SEARCH_MS_URL")] { + let Ok(endpoint) = std::env::var(variable) else { + continue; + }; + tested_providers += 1; + let remote = RemoteProvider::new( + &SearchRuntimeConfig { + provider: provider.to_string(), + endpoint: endpoint.clone(), + ..Default::default() + }, + pool.clone(), + ) + .unwrap(); + let table = format!("affine_search_contract_{}", uuid::Uuid::new_v4().simple()); + let block_table = format!("affine_search_contract_block_{}", uuid::Uuid::new_v4().simple()); + let cleanup_tables = [table.clone(), block_table.clone()]; + let contract = tokio::spawn(async move { + remote.provision(&table, super::types::SearchTable::Doc).await.unwrap(); + let upsert = SearchChange { + sequence: 1, + external_id: "workspace/doc".into(), + workspace_id: "workspace".into(), + doc_id: Some("doc".into()), + revision: 1, + operation: "upsert".into(), + payload: Some(json!({ + "workspace_id":"workspace","workspace_token":super::exact_token("workspace"), + "doc_id":"doc","doc_token":super::exact_token("doc"),"title":"search contract", + "summary":"","created_by_user_id":"user","updated_by_user_id":"user", + "created_at":1,"updated_at":1,"acl_public_readable":false, + "acl_member_default_readable":true,"acl_read_tokens":["member"],"acl_revision":1 + })), + }; + let mut second = upsert.clone(); + second.sequence = 2; + second.external_id = "workspace/doc-2".into(); + second.doc_id = Some("doc-2".into()); + second.payload.as_mut().unwrap()["doc_id"] = json!("doc-2"); + second.payload.as_mut().unwrap()["doc_token"] = json!(super::exact_token("doc-2")); + remote.apply(&table, &[upsert.clone(), second.clone()]).await.unwrap(); + let dsl = json!({ + "query":{"bool":{"must":[ + {"term":{"workspace_id":{"value":"workspace"}}}, + {"match":{"title":{"query":"contract"}}}, + {"bool":{"should":[{"term":{"acl_read_tokens":{"value":"member"}}}]}} + ],"boost":1.0}}, + "fields":["doc_id","title"],"_source":["workspace_id","doc_id"], + "highlight":{"fields":{"title":{"pre_tags":[""],"post_tags":[""]}}}, + "sort":["doc_id"],"size":1 + }); + let result = remote.search(&table, dsl.clone()).await.unwrap(); + assert_eq!(result["total"], 2, "provider {provider}"); + assert!(result["nodes"][0]["fields"]["doc_id"].is_array(), "provider {provider}"); + assert!( + result["nodes"][0]["highlights"]["title"].is_array(), + "provider {provider}" + ); + let first_doc = result["nodes"][0]["fields"]["doc_id"][0].clone(); + let mut next_dsl = dsl.clone(); + let first_cursor = result["nextCursor"].clone(); + next_dsl["cursor"] = first_cursor.clone(); + let next = remote.search(&table, next_dsl).await.unwrap(); + assert_ne!( + first_doc, next["nodes"][0]["fields"]["doc_id"][0], + "provider {provider}" + ); + assert_ne!(first_cursor, next["nextCursor"], "provider {provider}"); + let aggregate_dsl = json!({ + "query":{"term":{"workspace_id":{"value":"workspace"}}}, + "size":0, + "aggs":{"result":{"terms":{"field":"doc_id","size":10},"aggs":{"result":{"top_hits":{ + "size":1,"_source":["workspace_id","doc_id"],"fields":["doc_id","title"],"sort":["doc_id"] + }}}}} + }); + if provider == "elasticsearch" { + let aggregate = remote.aggregate(&table, aggregate_dsl).await.unwrap(); + assert_eq!(aggregate["total"], 2); + assert_eq!(aggregate["buckets"].as_array().unwrap().len(), 2); + assert!(aggregate["buckets"][0]["hits"]["nodes"][0]["fields"]["doc_id"].is_array()); + } else { + assert!(matches!( + remote.aggregate(&table, aggregate_dsl).await, + Err(crate::runtime::RuntimeError::SearchUnsupportedQuery) + )); + } + + remote + .provision(&block_table, super::types::SearchTable::Block) + .await + .unwrap(); + let block = SearchChange { + sequence: 1, + external_id: "workspace/doc/block".into(), + workspace_id: "workspace".into(), + doc_id: Some("doc".into()), + revision: 1, + operation: "upsert".into(), + payload: Some(json!({ + "workspace_id":"workspace","workspace_token":super::exact_token("workspace"), + "doc_id":"doc","doc_token":super::exact_token("doc"), + "block_id":"block","block_token":super::exact_token("block"), + "content":"笔记应用 다람쥐 いろはにほへと https://linear.app/affine-design/issue/AF-1379/slash-commands", + "flavour":"affine:paragraph", + "ref_doc_id":["ref-a","ref-b"],"blob":["blob-a","blob-b"], + "created_by_user_id":"user","updated_by_user_id":"user", + "created_at":2_000,"updated_at":3_000,"acl_public_readable":false, + "acl_member_default_readable":true,"acl_read_tokens":["member"],"acl_revision":1 + })), + }; + remote.apply(&block_table, std::slice::from_ref(&block)).await.unwrap(); + let exists = remote + .search( + &block_table, + json!({ + "query":{"exists":{"field":"ref_doc_id"}}, + "fields":["block_id","ref_doc_id"],"_source":["workspace_id","doc_id"], + "sort":["block_id"],"size":10 + }), + ) + .await + .unwrap(); + assert_eq!(exists["total"], 1, "provider {provider}"); + let exact_ref = remote + .search( + &block_table, + json!({ + "query":{"bool":{"must":[ + {"term":{"workspace_id":{"value":"workspace"}}}, + {"term":{"ref_doc_id":{"value":"ref-a"}}}, + {"bool":{"must_not":[{"term":{"doc_id":{"value":"other-doc"}}}]}} + ]}}, + "fields":["block_id","ref_doc_id"],"_source":["workspace_id","doc_id"], + "sort":["block_id"],"size":10 + }), + ) + .await + .unwrap(); + assert_eq!(exact_ref["total"], 1, "provider {provider}"); + let terms = if provider == "elasticsearch" { + ["记", "https://linear.app"].as_slice() + } else { + ["쥐", "へ", "https://linear.app"].as_slice() + }; + for term in terms { + let language = remote + .search( + &block_table, + json!({ + "query":{"match":{"content":{"query":term}}}, + "fields":["block_id","ref_doc_id","blob","created_at","updated_at"], + "_source":["workspace_id","doc_id"], + "highlight":{"fields":{"content":{"pre_tags":[""],"post_tags":[""]}}}, + "sort":["block_id"],"size":10 + }), + ) + .await + .unwrap(); + assert_eq!(language["total"], 1, "provider {provider}, term {term}"); + assert_eq!(language["nodes"][0]["fields"]["ref_doc_id"], json!(["ref-a", "ref-b"])); + assert!(language["nodes"][0]["fields"]["created_at"].is_array()); + assert!(language["nodes"][0]["highlights"]["content"].is_array()); + } + let mut revoked = upsert.clone(); + revoked.sequence = 3; + revoked.revision = 3; + revoked.payload.as_mut().unwrap()["acl_read_tokens"] = json!([]); + remote.apply(&table, &[revoked]).await.unwrap(); + let revoked_result = remote.search(&table, dsl.clone()).await.unwrap(); + assert_eq!(revoked_result["total"], 1, "provider {provider}"); + assert_eq!(revoked_result["nodes"][0]["fields"]["doc_id"][0], "doc-2"); + let deletion = SearchChange { + operation: "delete".into(), + payload: None, + sequence: 4, + revision: 4, + ..upsert.clone() + }; + let second_deletion = SearchChange { + operation: "delete".into(), + payload: None, + sequence: 5, + revision: 5, + ..second + }; + remote.apply(&table, &[deletion, second_deletion]).await.unwrap(); + let result = remote + .search( + &table, + json!({"query":{"match_all":{}},"fields":["doc_id"],"sort":["doc_id"],"size":10}), + ) + .await + .unwrap(); + assert_eq!(result["total"], 0, "provider {provider}"); + }) + .await; + let client = reqwest::Client::new(); + if provider == "elasticsearch" { + for physical_table in &cleanup_tables { + let response = client + .delete(format!("{endpoint}/{physical_table}")) + .send() + .await + .unwrap(); + assert!(response.status().is_success() || response.status() == reqwest::StatusCode::NOT_FOUND); + } + } else { + for physical_table in &cleanup_tables { + client + .post(format!("{endpoint}/cli")) + .header("content-type", "text/plain") + .body(format!("DROP TABLE IF EXISTS {physical_table}")) + .send() + .await + .unwrap() + .error_for_status() + .unwrap(); + } + } + contract.unwrap(); + } + if require_remote { + assert_eq!(tested_providers, 2, "SEARCH_ES_URL and SEARCH_MS_URL are required"); + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/types.rs b/packages/backend/native/src/runtime/backend_runtime/search/types.rs new file mode 100644 index 0000000000..02d6193321 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/types.rs @@ -0,0 +1,241 @@ +use serde::{Deserialize, Serialize}; + +use crate::runtime::{RuntimeError, RuntimeResult}; + +#[napi_derive::napi(string_enum = "snake_case")] +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SearchTable { + Doc, + Block, +} + +impl SearchTable { + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Doc => "doc", + Self::Block => "block", + } + } + + pub(super) fn text_field(self) -> &'static str { + match self { + Self::Doc => "title", + Self::Block => "content", + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct SearchQuery { + #[serde(rename = "type")] + pub(super) query_type: String, + pub(super) field: Option, + #[serde(rename = "match")] + pub(super) match_value: Option, + pub(super) query: Option>, + pub(super) queries: Option>, + pub(super) occur: Option, + pub(super) boost: Option, +} + +#[napi_derive::napi(object)] +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchPagination { + pub limit: Option, + pub skip: Option, + pub cursor: Option, +} + +#[napi_derive::napi(object)] +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchHighlight { + pub field: String, + pub before: String, + pub end: String, +} + +#[napi_derive::napi(object)] +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchOptions { + pub fields: Vec, + #[serde(default)] + pub highlights: Vec, + #[serde(default)] + pub pagination: SearchPagination, +} + +#[napi_derive::napi(object)] +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AggregateHitsOptions { + pub fields: Vec, + #[serde(default)] + pub highlights: Vec, + #[serde(default)] + pub pagination: SearchPagination, +} + +#[napi_derive::napi(object)] +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AggregateOptions { + pub hits: AggregateHitsOptions, + #[serde(default)] + pub pagination: SearchPagination, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct AggregateRequest { + pub(super) table: SearchTable, + pub(super) query: SearchQuery, + pub(super) field: String, + pub(super) options: AggregateOptions, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct SearchRequest { + pub(super) table: SearchTable, + pub(super) query: SearchQuery, + pub(super) options: SearchOptions, +} + +#[napi_derive::napi(object)] +pub struct RuntimeSearchQuery { + pub query_type: String, + pub field: Option, + pub match_value: Option, + pub query: Option, + pub queries: Option>, + pub occur: Option, + pub boost: Option, +} + +#[napi_derive::napi(object)] +pub struct RuntimeSearchRequest { + pub table: SearchTable, + pub queries: Vec, + pub root_query: u32, + pub options: SearchOptions, +} + +#[napi_derive::napi(object)] +pub struct RuntimeAggregateRequest { + pub table: SearchTable, + pub queries: Vec, + pub root_query: u32, + pub field: String, + pub options: AggregateOptions, +} + +#[cfg(test)] +impl SearchRequest { + pub(super) fn parse(value: serde_json::Value) -> RuntimeResult { + serde_json::from_value(value).map_err(|error| RuntimeError::json("invalid search request", error)) + } +} + +impl RuntimeSearchRequest { + pub(super) fn into_search_request(self) -> RuntimeResult { + let mut decoded_nodes = 0; + Ok(SearchRequest { + table: self.table, + query: decode_query(&self.queries, self.root_query, 0, &mut decoded_nodes)?, + options: self.options, + }) + } +} + +impl RuntimeAggregateRequest { + pub(super) fn into_aggregate_request(self) -> RuntimeResult { + let mut decoded_nodes = 0; + Ok(AggregateRequest { + table: self.table, + query: decode_query(&self.queries, self.root_query, 0, &mut decoded_nodes)?, + field: self.field, + options: self.options, + }) + } +} + +const MAX_QUERY_GRAPH_NODES: usize = 100; +const MAX_QUERY_DEPTH: usize = 100; +const MAX_DECODED_QUERY_NODES: usize = 1_000; + +fn decode_query( + nodes: &[RuntimeSearchQuery], + index: u32, + depth: usize, + decoded_nodes: &mut usize, +) -> RuntimeResult { + if nodes.len() > MAX_QUERY_GRAPH_NODES || depth > MAX_QUERY_DEPTH || *decoded_nodes >= MAX_DECODED_QUERY_NODES { + return Err(RuntimeError::invalid_input("search query is too complex")); + } + *decoded_nodes += 1; + let node = nodes + .get(index as usize) + .ok_or_else(|| RuntimeError::invalid_input("invalid search query node"))?; + Ok(SearchQuery { + query_type: node.query_type.clone(), + field: node.field.clone(), + match_value: node.match_value.clone(), + query: node + .query + .map(|index| decode_query(nodes, index, depth + 1, decoded_nodes).map(Box::new)) + .transpose()?, + queries: node + .queries + .as_ref() + .map(|indices| { + indices + .iter() + .map(|index| decode_query(nodes, *index, depth + 1, decoded_nodes)) + .collect() + }) + .transpose()?, + occur: node.occur.clone(), + boost: node.boost, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn node(query_type: &str) -> RuntimeSearchQuery { + RuntimeSearchQuery { + query_type: query_type.to_string(), + field: None, + match_value: None, + query: None, + queries: None, + occur: None, + boost: None, + } + } + + #[test] + fn rejects_invalid_or_overly_complex_query_graphs() { + assert!(decode_query(&[node("all")], 1, 0, &mut 0).is_err()); + + let mut oversized = (0..101).map(|_| node("all")).collect::>(); + oversized[0].query = Some(1); + assert!(decode_query(&oversized, 0, 0, &mut 0).is_err()); + + let mut recursive = vec![node("boost")]; + recursive[0].query = Some(0); + assert!(decode_query(&recursive, 0, 0, &mut 0).is_err()); + + let mut shared_child = (0..100).map(|_| node("boolean")).collect::>(); + for (index, node) in shared_child.iter_mut().enumerate().take(99) { + node.queries = Some(vec![(index + 1) as u32, (index + 1) as u32]); + } + assert!(decode_query(&shared_child, 0, 0, &mut 0).is_err()); + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/search/worker.rs b/packages/backend/native/src/runtime/backend_runtime/search/worker.rs new file mode 100644 index 0000000000..5120c80f64 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/search/worker.rs @@ -0,0 +1,272 @@ +use sqlx::PgPool; +use tokio::sync::RwLock; + +use super::{ + generation::ActiveGeneration, + provider::RemoteProvider, + store::{SearchChange, SearchStore, SearchTable}, +}; +use crate::{ + runtime::{RuntimeError, RuntimeResult}, + search_index::EmbeddedSearchIndex, +}; + +pub(super) async fn rebuild( + pool: &PgPool, + store: &SearchStore, + embedded: &EmbeddedSearchIndex, + remote: Option<&RemoteProvider>, + generation: &ActiveGeneration, + embedded_cursors: &RwLock<[i64; 2]>, + restore_checkpoint: bool, +) -> RuntimeResult<()> { + for table in SearchTable::ORDERED { + if restore_checkpoint + && remote.is_none() + && let Some(cursor) = super::checkpoint::restore(pool, embedded, table).await? + { + set_cursor(pool, remote, generation, embedded_cursors, table, cursor).await?; + continue; + } + let snapshot = store.snapshot(table).await?; + if let Some(remote) = remote { + let changes = snapshot + .projections + .into_iter() + .enumerate() + .map(|(offset, projection)| SearchChange { + sequence: offset as i64 + 1, + external_id: projection.external_id, + workspace_id: projection.workspace_id, + doc_id: Some(projection.doc_id), + revision: projection.revision, + operation: "upsert".into(), + payload: Some(projection.payload), + }) + .collect::>(); + for batch in changes.chunks(1000) { + remote + .apply(generation.physical_table(runtime_table(table))?, batch) + .await?; + } + } else { + embedded.reset(table.as_str().to_string()).await?; + for documents in snapshot.projections.chunks(1000) { + embedded + .write( + table.as_str().to_string(), + serde_json::to_string( + &documents + .iter() + .map(|projection| super::provider_payload(&projection.payload)) + .collect::>(), + ) + .map_err(|error| RuntimeError::json("encode embedded snapshot", error))?, + ) + .await?; + } + } + set_cursor(pool, remote, generation, embedded_cursors, table, snapshot.head).await?; + } + sync(pool, store, embedded, remote, generation, embedded_cursors).await?; + Ok(()) +} + +pub(super) async fn sync( + pool: &PgPool, + store: &SearchStore, + embedded: &EmbeddedSearchIndex, + remote: Option<&RemoteProvider>, + generation: &ActiveGeneration, + embedded_cursors: &RwLock<[i64; 2]>, +) -> RuntimeResult<()> { + for table in SearchTable::ORDERED { + loop { + let cursor = cursor(pool, remote, generation, embedded_cursors, table).await?; + let (head, changes) = store.changes(table, cursor, 1000).await?; + if changes.is_empty() { + if cursor < head { + return Err(RuntimeError::invalid_state( + "search provider cursor did not reach stream head", + )); + } + break; + } + if let Some(remote) = remote { + remote + .apply(generation.physical_table(runtime_table(table))?, &changes) + .await?; + } else { + apply_embedded(embedded, table, &changes).await?; + } + set_cursor( + pool, + remote, + generation, + embedded_cursors, + table, + changes.last().expect("non-empty changes").sequence, + ) + .await?; + } + } + if remote.is_none() { + super::checkpoint::persist(pool, embedded, *embedded_cursors.read().await).await?; + } else { + super::checkpoint::gc(pool).await?; + } + Ok(()) +} + +async fn apply_embedded( + embedded: &EmbeddedSearchIndex, + table: SearchTable, + changes: &[SearchChange], +) -> RuntimeResult<()> { + let mut upserts = Vec::new(); + for change in changes { + if change.operation == "delete" { + if !upserts.is_empty() { + embedded + .write( + table.as_str().to_string(), + serde_json::to_string(&upserts).map_err(|error| RuntimeError::json("encode embedded changes", error))?, + ) + .await?; + upserts.clear(); + } + embedded + .delete(table.as_str().to_string(), change.external_id.clone()) + .await?; + } else if let Some(payload) = &change.payload { + upserts.push(super::provider_payload(payload)); + } + } + if !upserts.is_empty() { + embedded + .write( + table.as_str().to_string(), + serde_json::to_string(&upserts).map_err(|error| RuntimeError::json("encode embedded changes", error))?, + ) + .await?; + } + Ok(()) +} + +async fn set_cursor( + pool: &PgPool, + remote: Option<&RemoteProvider>, + generation: &ActiveGeneration, + embedded_cursors: &RwLock<[i64; 2]>, + table: SearchTable, + cursor: i64, +) -> RuntimeResult<()> { + if remote.is_none() { + let mut cursors = embedded_cursors.write().await; + cursors[table.cursor_index()] = cursors[table.cursor_index()].max(cursor); + return Ok(()); + } + sqlx::query( + "UPDATE search_runtime_provider_cursors SET source_cursor=GREATEST(source_cursor,$3), updated_at=now() WHERE \ + generation_id=$1 AND table_key=$2", + ) + .bind(generation.id) + .bind(table.as_str()) + .bind(cursor) + .execute(pool) + .await + .map_err(|error| RuntimeError::database("advance search provider cursor", error))?; + Ok(()) +} + +async fn cursor( + pool: &PgPool, + remote: Option<&RemoteProvider>, + generation: &ActiveGeneration, + embedded_cursors: &RwLock<[i64; 2]>, + table: SearchTable, +) -> RuntimeResult { + if remote.is_none() { + return Ok(embedded_cursors.read().await[table.cursor_index()]); + } + sqlx::query_scalar( + "SELECT source_cursor FROM search_runtime_provider_cursors WHERE generation_id=$1 AND table_key=$2", + ) + .bind(generation.id) + .bind(table.as_str()) + .fetch_one(pool) + .await + .map_err(|error| RuntimeError::database("load search provider cursor", error)) +} + +fn runtime_table(table: SearchTable) -> super::types::SearchTable { + match table { + SearchTable::Doc => super::types::SearchTable::Doc, + SearchTable::Block => super::types::SearchTable::Block, + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[tokio::test] + async fn embedded_changes_follow_stream_order() { + let embedded = EmbeddedSearchIndex::new(); + let changes = vec![ + SearchChange { + sequence: 1, + external_id: "workspace/doc".into(), + workspace_id: "workspace".into(), + doc_id: Some("doc".into()), + revision: 1, + operation: "delete".into(), + payload: Some(json!({ + "workspace_id": "workspace", + "doc_id": "doc", + "title": "deleted", + "created_at": 1, + "updated_at": 1 + })), + }, + SearchChange { + sequence: 2, + external_id: "workspace/doc".into(), + workspace_id: "workspace".into(), + doc_id: Some("doc".into()), + revision: 2, + operation: "upsert".into(), + payload: Some(json!({ + "workspace_id": "workspace", + "doc_id": "doc", + "title": "restored", + "created_at": 1, + "updated_at": 2 + })), + }, + ]; + + apply_embedded(&embedded, SearchTable::Doc, &changes).await.unwrap(); + + let result: serde_json::Value = serde_json::from_str( + &embedded + .search( + "doc".into(), + json!({ + "query": {"match_all": {}}, + "fields": ["doc_id", "title"], + "sort": ["doc_id"], + "size": 10 + }) + .to_string(), + ) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(result["total"], 1); + assert_eq!(result["nodes"][0]["fields"]["title"], json!(["restored"])); + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/tests.rs b/packages/backend/native/src/runtime/backend_runtime/tests.rs index 5af0a19e08..01dacdb7a9 100644 --- a/packages/backend/native/src/runtime/backend_runtime/tests.rs +++ b/packages/backend/native/src/runtime/backend_runtime/tests.rs @@ -24,6 +24,7 @@ fn migrations_include_runtime_tables_without_worker_heartbeats() { assert!(RUNTIME_MIGRATIONS.contains("storage_reconciliation_checkpoints")); assert!(RUNTIME_MIGRATIONS.contains("document_cleanup_candidates")); assert!(RUNTIME_MIGRATIONS.contains("doc_blob_refs")); + assert!(RUNTIME_MIGRATIONS.contains("doc_blob_ref_projections")); assert!(RUNTIME_MIGRATIONS.contains("blob_cleanup_candidates")); assert!(!RUNTIME_MIGRATIONS.contains("runtime_worker_heartbeats")); } @@ -100,12 +101,15 @@ async fn runtime_from_database_url() -> AnyResult> { Ok(Some(BackendRuntime { config_source: Default::default(), + role: ServerRole::AllInOne, + script_mode: false, config: Arc::new(RwLock::new(Arc::new(BackendRuntimeConfig { database_url, invite_quota: Default::default(), private_key: Arc::new(zeroize::Zeroizing::new("test-private-key".to_string())), deployment: crate::llm::Deployment::Cloud, copilot: Default::default(), + search: Default::default(), }))), config_reload: Mutex::new(()), pool: Mutex::new(Some(pool)), @@ -114,6 +118,8 @@ async fn runtime_from_database_url() -> AnyResult> { crate::runtime::object_storage::ObjectStorageService::from_config_files()?, )), embedding: Mutex::new(None), + embedding_worker: Mutex::new(None), + search: Mutex::new(None), managed_token_providers: Arc::new(Default::default()), })) } @@ -260,12 +266,16 @@ async fn runtime_gate_sql_semantics_are_atomic_and_ttl_bound() { for _ in 0..16 { let runtime = BackendRuntime { config_source: Default::default(), + role: ServerRole::AllInOne, + script_mode: false, config: Arc::new(RwLock::new(runtime.config().unwrap())), config_reload: Mutex::new(()), pool: Mutex::new(Some(runtime.pool().await.unwrap())), embedding_health: RwLock::new(super::EmbeddingHealth::disabled("test", None)), object_storage: RwLock::new(runtime.object_storage().unwrap()), embedding: Mutex::new(None), + embedding_worker: Mutex::new(None), + search: Mutex::new(None), managed_token_providers: Arc::new(Default::default()), }; tasks.push(tokio::spawn(async move { @@ -597,12 +607,16 @@ async fn coordination_lease_sql_semantics_are_fenced_and_ttl_bound() { for index in 0..16 { let runtime = BackendRuntime { config_source: Default::default(), + role: ServerRole::AllInOne, + script_mode: false, config: Arc::new(RwLock::new(runtime.config().unwrap())), config_reload: Mutex::new(()), pool: Mutex::new(Some(runtime.pool().await.unwrap())), embedding_health: RwLock::new(super::EmbeddingHealth::disabled("test", None)), object_storage: RwLock::new(runtime.object_storage().unwrap()), embedding: Mutex::new(None), + embedding_worker: Mutex::new(None), + search: Mutex::new(None), managed_token_providers: Arc::new(Default::default()), }; tasks.push(tokio::spawn(async move { @@ -807,12 +821,16 @@ async fn verification_token_sql_state_machine_handles_keep_verify_and_cleanup() for _ in 0..16 { let runtime = BackendRuntime { config_source: Default::default(), + role: ServerRole::AllInOne, + script_mode: false, config: Arc::new(RwLock::new(runtime.config().unwrap())), config_reload: Mutex::new(()), pool: Mutex::new(Some(runtime.pool().await.unwrap())), embedding_health: RwLock::new(super::EmbeddingHealth::disabled("test", None)), object_storage: RwLock::new(runtime.object_storage().unwrap()), embedding: Mutex::new(None), + embedding_worker: Mutex::new(None), + search: Mutex::new(None), managed_token_providers: Arc::new(Default::default()), }; let token = concurrent_token.clone(); diff --git a/packages/backend/native/src/runtime/config.rs b/packages/backend/native/src/runtime/config.rs index 930cb2100c..59d42a89fb 100644 --- a/packages/backend/native/src/runtime/config.rs +++ b/packages/backend/native/src/runtime/config.rs @@ -20,6 +20,30 @@ pub(crate) struct BackendRuntimeConfig { pub(crate) private_key: Arc>, pub(crate) deployment: Deployment, pub(crate) copilot: CopilotRuntimeConfig, + pub(crate) search: SearchRuntimeConfig, +} + +#[derive(Clone, Debug)] +pub(crate) struct SearchRuntimeConfig { + pub(crate) enabled: bool, + pub(crate) provider: String, + pub(crate) endpoint: String, + pub(crate) api_key: String, + pub(crate) username: String, + pub(crate) password: String, +} + +impl Default for SearchRuntimeConfig { + fn default() -> Self { + Self { + enabled: false, + provider: "embedded".to_string(), + endpoint: String::new(), + api_key: String::new(), + username: String::new(), + password: String::new(), + } + } } #[derive(Clone, Debug)] @@ -348,6 +372,7 @@ impl BackendRuntimeConfig { .map(TryInto::try_into) .transpose()? .unwrap_or_default(), + search: app_config.indexer.map(Into::into).unwrap_or_default(), } .validated() } @@ -385,6 +410,10 @@ impl BackendRuntimeConfig { .map(TryInto::try_into) .transpose()? .unwrap_or_else(|| self.copilot.clone()), + search: app_config + .indexer + .map(Into::into) + .unwrap_or_else(|| self.search.clone()), } .validated() } @@ -453,6 +482,42 @@ struct AppConfigFile { db: Option, crypto: Option, copilot: Option, + indexer: Option, +} + +#[derive(Default, Deserialize)] +#[serde(rename_all = "camelCase", default)] +struct SearchRuntimeConfigFile { + enabled: bool, + provider: SearchProviderConfigFile, +} + +#[derive(Default, Deserialize)] +#[serde(rename_all = "camelCase", default)] +struct SearchProviderConfigFile { + #[serde(rename = "type")] + provider: String, + endpoint: String, + api_key: String, + username: String, + password: String, +} + +impl From for SearchRuntimeConfig { + fn from(value: SearchRuntimeConfigFile) -> Self { + Self { + enabled: value.enabled, + provider: if value.provider.provider.is_empty() { + "embedded".to_string() + } else { + value.provider.provider + }, + endpoint: value.provider.endpoint, + api_key: value.provider.api_key, + username: value.provider.username, + password: value.provider.password, + } + } } #[derive(Default, Deserialize)] @@ -815,6 +880,35 @@ mod tests { assert!(copilot.byok.allow_custom_endpoint); } + #[test] + fn search_config_keeps_disabled_state_separate_from_embedded_provider() { + let disabled = app_config_from_flat_overrides([ + ("indexer.enabled", serde_json::json!(false)), + ("indexer.provider.type", serde_json::json!("embedded")), + ]) + .unwrap(); + let disabled: SearchRuntimeConfig = disabled.indexer.unwrap().into(); + assert!(!disabled.enabled); + assert_eq!(disabled.provider, "embedded"); + + let enabled = app_config_from_flat_overrides([ + ("indexer.enabled", serde_json::json!(true)), + ("indexer.provider.type", serde_json::json!("elasticsearch")), + ]) + .unwrap(); + let enabled: SearchRuntimeConfig = enabled.indexer.unwrap().into(); + assert!(enabled.enabled); + assert_eq!(enabled.provider, "elasticsearch"); + + let enabled_without_provider = app_config_from_module_json(serde_json::json!({ + "indexer": { "enabled": true } + })) + .unwrap(); + let enabled_without_provider: SearchRuntimeConfig = enabled_without_provider.indexer.unwrap().into(); + assert!(enabled_without_provider.enabled); + assert_eq!(enabled_without_provider.provider, "embedded"); + } + #[test] fn partial_database_config_preserves_file_config_siblings() { let mut file_config = expand_module_config_paths(serde_json::json!({ @@ -874,6 +968,7 @@ mod tests { private_key: Arc::new(Zeroizing::new("active-private-key".to_string())), deployment: Deployment::Cloud, copilot: CopilotRuntimeConfig::default(), + search: SearchRuntimeConfig::default(), }; let empty = serde_json::Value::Object(Map::new()); diff --git a/packages/backend/native/src/runtime/error.rs b/packages/backend/native/src/runtime/error.rs index 1b2f283af4..625b3a2898 100644 --- a/packages/backend/native/src/runtime/error.rs +++ b/packages/backend/native/src/runtime/error.rs @@ -15,6 +15,21 @@ pub(crate) enum RuntimeError { #[error("{0}")] InvalidState(String), + #[error("workspace access denied")] + SearchWorkspaceDenied, + + #[error("search permission state unavailable")] + SearchPermissionUnavailable, + + #[error("search provider unavailable")] + SearchProviderUnavailable, + + #[error("search query is not supported by the active provider")] + SearchUnsupportedQuery, + + #[error("search stream replay gap")] + SearchReplayGap, + #[error("{context}: {source}")] Database { context: String, @@ -94,6 +109,11 @@ impl RuntimeError { | Self::NapiBoundary(message) => { message.contains("NoSuchKey") || message.contains("NotFound") || message.contains("not found") } + Self::SearchWorkspaceDenied + | Self::SearchPermissionUnavailable + | Self::SearchProviderUnavailable + | Self::SearchUnsupportedQuery + | Self::SearchReplayGap => false, _ => false, } } diff --git a/packages/backend/native/src/runtime/migrations.rs b/packages/backend/native/src/runtime/migrations.rs index fca4fb44d8..0fb4bcf51b 100644 --- a/packages/backend/native/src/runtime/migrations.rs +++ b/packages/backend/native/src/runtime/migrations.rs @@ -5,7 +5,10 @@ use super::{RuntimeError, RuntimeResult, types::EmbeddingHealth}; pub(crate) const RUNTIME_MIGRATIONS: &str = include_str!("sql/runtime_migrations.sql"); const EMBEDDING_MIGRATION: &str = include_str!("sql/embedding.sql"); +const SEARCH_MIGRATION: &str = include_str!("sql/search.sql"); +const SEARCH_ACL_TOKEN_MIGRATION: &str = include_str!("sql/search_acl_tokens.sql"); const EMBEDDING_ADVISORY_LOCK: i64 = 0x4146_4649_4e45_0046; +const SEARCH_ADVISORY_LOCK: i64 = 0x4146_4649_4e45_0053; #[cfg(test)] pub(crate) static EMBEDDING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); @@ -17,6 +20,14 @@ pub(crate) async fn migrate_runtime_tables(pool: &PgPool) -> RuntimeResult<()> { Ok(()) } +pub(crate) async fn migrate_all_tables(pool: &PgPool) -> RuntimeResult { + migrate_runtime_tables(pool).await?; + let embedding = migrate_embedding_tables_inner(pool).await?; + migrate_search_tables(pool).await?; + Ok(embedding) +} + +#[cfg(test)] pub(crate) async fn migrate_embedding_tables(pool: &PgPool) -> EmbeddingHealth { match migrate_embedding_tables_inner(pool).await { Ok(health) => health, @@ -24,7 +35,7 @@ pub(crate) async fn migrate_embedding_tables(pool: &PgPool) -> EmbeddingHealth { } } -async fn migrate_embedding_tables_inner(pool: &PgPool) -> RuntimeResult { +pub(crate) async fn embedding_schema_health(pool: &PgPool) -> RuntimeResult { let Some(version) = pgvector_version(pool).await? else { return Ok(EmbeddingHealth::disabled("pgvector_unavailable", None)); }; @@ -32,33 +43,19 @@ async fn migrate_embedding_tables_inner(pool: &PgPool) -> RuntimeResult RuntimeResult RuntimeResult<()> { + migrate_component( + pool, + "search", + SEARCH_ADVISORY_LOCK, + &[(1, &[SEARCH_MIGRATION]), (2, &[SEARCH_ACL_TOKEN_MIGRATION])], + ) + .await?; + sqlx::query("INSERT INTO search_runtime_streams(table_key) VALUES ('doc'), ('block') ON CONFLICT DO NOTHING") + .execute(pool) + .await + .map_err(|error| RuntimeError::database("Repair search runtime streams", error))?; + Ok(()) +} + +async fn migrate_embedding_tables_inner(pool: &PgPool) -> RuntimeResult { + let Some(version) = pgvector_version(pool).await? else { + return Ok(EmbeddingHealth::disabled("pgvector_unavailable", None)); + }; + if !pgvector_at_least_0_8(&version) { + return Ok(EmbeddingHealth::disabled("pgvector_version_unsupported", Some(version))); + } + + migrate_component( + pool, + "embedding", + EMBEDDING_ADVISORY_LOCK, + &[(1, &[EMBEDDING_MIGRATION])], + ) + .await?; + + Ok(EmbeddingHealth { + enabled: true, + state: "ready".to_string(), + reason: None, + pgvector_version: Some(version), + schema_version: Some(1), + worker_running: false, + }) +} + +async fn migrate_component( + pool: &PgPool, + component: &str, + advisory_lock: i64, + migrations: &[(i32, &[&str])], +) -> RuntimeResult<()> { + let mut transaction = pool + .begin() + .await + .map_err(|error| RuntimeError::database("Native migration transaction failed", error))?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(advisory_lock) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("Native migration lock failed", error))?; + transaction + .execute( + r#"CREATE TABLE IF NOT EXISTS native_schema_migrations ( + component TEXT NOT NULL, + version INTEGER NOT NULL, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (component, version) + )"#, + ) + .await + .map_err(|error| RuntimeError::database("Native migration ledger failed", error))?; + + for (version, statements) in migrations { + apply_migration(&mut transaction, component, *version, statements).await?; + } + transaction + .commit() + .await + .map_err(|error| RuntimeError::database("Native migration commit failed", error)) +} + async fn apply_migration( transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, + component: &str, version: i32, statements: &[&str], ) -> RuntimeResult<()> { let checksum = migration_checksum(statements); - let applied = sqlx::query("SELECT checksum FROM native_schema_migrations WHERE component='embedding' AND version=$1") + let applied = sqlx::query("SELECT checksum FROM native_schema_migrations WHERE component=$1 AND version=$2") + .bind(component) .bind(version) .fetch_optional(&mut **transaction) .await - .map_err(|error| RuntimeError::database("Embedding migration ledger read failed", error))?; + .map_err(|error| RuntimeError::database("Native migration ledger read failed", error))?; if let Some(applied) = applied { let stored: String = applied .try_get("checksum") - .map_err(|error| RuntimeError::database("Embedding migration checksum decode failed", error))?; + .map_err(|error| RuntimeError::database("Native migration checksum decode failed", error))?; if stored != checksum { - return Err(RuntimeError::invalid_state("Embedding migration checksum mismatch")); + return Err(RuntimeError::invalid_state("Native migration checksum mismatch")); } return Ok(()); } @@ -94,14 +171,15 @@ async fn apply_migration( transaction .execute(*statement) .await - .map_err(|error| RuntimeError::database("Embedding migration failed", error))?; + .map_err(|error| RuntimeError::database("Native component migration failed", error))?; } - sqlx::query("INSERT INTO native_schema_migrations(component,version,checksum) VALUES('embedding',$1,$2)") + sqlx::query("INSERT INTO native_schema_migrations(component,version,checksum) VALUES($1,$2,$3)") + .bind(component) .bind(version) .bind(checksum) .execute(&mut **transaction) .await - .map_err(|error| RuntimeError::database("Embedding migration record failed", error))?; + .map_err(|error| RuntimeError::database("Native migration record failed", error))?; Ok(()) } @@ -189,4 +267,29 @@ mod tests { .unwrap(); assert_eq!(dimensions, 1024); } + + #[test] + fn search_schema_uses_transactional_stream_heads() { + assert!(SEARCH_MIGRATION.contains("CREATE TABLE search_runtime_streams")); + assert!(SEARCH_MIGRATION.contains("PRIMARY KEY (table_key, stream_sequence)")); + assert!(!SEARCH_MIGRATION.contains("BIGSERIAL")); + assert!(!SEARCH_MIGRATION.contains("CREATE SEQUENCE")); + } + + #[tokio::test] + async fn concurrent_search_migration_is_idempotent() { + let Ok(database_url) = std::env::var("DATABASE_URL") else { + return; + }; + let pool = PgPool::connect(&database_url).await.unwrap(); + let (first, second) = tokio::join!(migrate_search_tables(&pool), migrate_search_tables(&pool)); + first.unwrap(); + second.unwrap(); + let versions: Vec = + sqlx::query_scalar("SELECT version FROM native_schema_migrations WHERE component='search' ORDER BY version") + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(versions, vec![1, 2]); + } } diff --git a/packages/backend/native/src/runtime/mod.rs b/packages/backend/native/src/runtime/mod.rs index cdfa733ec7..0c102ec7e5 100644 --- a/packages/backend/native/src/runtime/mod.rs +++ b/packages/backend/native/src/runtime/mod.rs @@ -10,7 +10,7 @@ pub(crate) mod types; pub(crate) use config::{ BackendRuntimeConfig, ConfigSource, CopilotManagedProfileConfig, CopilotManagedProfileConfigFile, - CopilotRuntimeConfig, CopilotRuntimeConfigFile, InviteQuotaConfig, + CopilotRuntimeConfig, CopilotRuntimeConfigFile, InviteQuotaConfig, SearchRuntimeConfig, }; use config::{SUPPORTED_BYOK_PROVIDERS, validate_copilot_config}; pub use config_descriptor::{AppConfigDescriptor, app_config_descriptors, validate_app_config_value}; diff --git a/packages/backend/native/src/runtime/sql/runtime_migrations.sql b/packages/backend/native/src/runtime/sql/runtime_migrations.sql index 4ebf4fc7fc..fec4a2ab45 100644 --- a/packages/backend/native/src/runtime/sql/runtime_migrations.sql +++ b/packages/backend/native/src/runtime/sql/runtime_migrations.sql @@ -113,6 +113,28 @@ CREATE INDEX IF NOT EXISTS doc_blob_refs_workspace_blob_idx CREATE INDEX IF NOT EXISTS doc_blob_refs_workspace_status_idx ON doc_blob_refs (workspace_id, status); +CREATE TABLE IF NOT EXISTS doc_blob_ref_projections ( + workspace_id TEXT NOT NULL, + doc_id TEXT NOT NULL, + source_revision TIMESTAMPTZ(3), + parser_version INTEGER NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'fresh', 'failed', 'missing')), + indexed_at TIMESTAMPTZ(3), + error_code TEXT, + error_summary TEXT, + attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0), + updated_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (workspace_id, doc_id), + CHECK (status <> 'fresh' OR (source_revision IS NOT NULL AND indexed_at IS NOT NULL)), + CHECK (error_summary IS NULL OR octet_length(error_summary) <= 512) +); + +CREATE INDEX IF NOT EXISTS doc_blob_ref_projections_workspace_status_idx + ON doc_blob_ref_projections (workspace_id, status, updated_at DESC); + +CREATE INDEX IF NOT EXISTS doc_blob_ref_projections_workspace_revision_idx + ON doc_blob_ref_projections (workspace_id, source_revision); + CREATE TABLE IF NOT EXISTS blob_cleanup_candidates ( workspace_id TEXT NOT NULL, blob_key TEXT NOT NULL, diff --git a/packages/backend/native/src/runtime/sql/search.sql b/packages/backend/native/src/runtime/sql/search.sql new file mode 100644 index 0000000000..b75ad9e637 --- /dev/null +++ b/packages/backend/native/src/runtime/sql/search.sql @@ -0,0 +1,252 @@ +CREATE TABLE search_runtime_streams ( + table_key TEXT PRIMARY KEY CHECK (table_key IN ('doc', 'block')), + head BIGINT NOT NULL DEFAULT 0 CHECK (head >= 0), + retained_from BIGINT NOT NULL DEFAULT 0 CHECK (retained_from >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (retained_from <= head) +); + +INSERT INTO search_runtime_streams(table_key) +VALUES ('doc'), ('block') +ON CONFLICT DO NOTHING; + +CREATE TABLE search_runtime_projections ( + table_key TEXT NOT NULL CHECK (table_key IN ('doc', 'block')), + external_id TEXT NOT NULL, + workspace_id VARCHAR NOT NULL, + doc_id VARCHAR NOT NULL, + revision BIGINT NOT NULL CHECK (revision >= 0), + payload JSONB NOT NULL, + acl_public_readable BOOLEAN NOT NULL DEFAULT false, + acl_member_default_readable BOOLEAN NOT NULL DEFAULT false, + acl_read_user_ids TEXT[] NOT NULL DEFAULT '{}', + acl_revision BIGINT NOT NULL DEFAULT 0 CHECK (acl_revision >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (table_key, external_id) +); + +CREATE INDEX search_runtime_projections_workspace_doc + ON search_runtime_projections(workspace_id, doc_id, table_key); + +CREATE TABLE search_runtime_changes ( + table_key TEXT NOT NULL CHECK (table_key IN ('doc', 'block')), + stream_sequence BIGINT NOT NULL CHECK (stream_sequence > 0), + external_id TEXT NOT NULL, + workspace_id VARCHAR NOT NULL, + doc_id VARCHAR, + revision BIGINT NOT NULL CHECK (revision >= 0), + operation TEXT NOT NULL CHECK (operation IN ('upsert', 'delete', 'invalidate')), + payload JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (table_key, stream_sequence) +); + +CREATE INDEX search_runtime_changes_workspace + ON search_runtime_changes(workspace_id, table_key, stream_sequence); + +CREATE TABLE search_runtime_generations ( + generation_id UUID PRIMARY KEY, + provider TEXT NOT NULL CHECK (provider IN ('embedded', 'elasticsearch', 'manticoresearch')), + state TEXT NOT NULL CHECK (state IN ('pending', 'active', 'draining', 'failed')), + config_fingerprint TEXT NOT NULL, + schema_fingerprint TEXT NOT NULL, + manifest JSONB NOT NULL DEFAULT '{}', + applied_permission_revision BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + activated_at TIMESTAMPTZ +); + +CREATE INDEX search_runtime_generations_fingerprint + ON search_runtime_generations(provider, config_fingerprint, schema_fingerprint); + +CREATE UNIQUE INDEX search_runtime_single_active_generation + ON search_runtime_generations ((state)) WHERE state = 'active'; +CREATE UNIQUE INDEX search_runtime_single_pending_generation + ON search_runtime_generations ((state)) WHERE state = 'pending'; + +CREATE TABLE search_runtime_provider_cursors ( + generation_id UUID NOT NULL REFERENCES search_runtime_generations(generation_id) ON DELETE CASCADE, + table_key TEXT NOT NULL CHECK (table_key IN ('doc', 'block')), + source_cursor BIGINT NOT NULL DEFAULT 0 CHECK (source_cursor >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (generation_id, table_key) +); + +CREATE TABLE search_runtime_permission_cursors ( + generation_id UUID NOT NULL REFERENCES search_runtime_generations(generation_id) ON DELETE CASCADE, + workspace_id VARCHAR NOT NULL, + permission_revision BIGINT NOT NULL DEFAULT 0 CHECK (permission_revision >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (generation_id, workspace_id) +); + +CREATE TABLE search_runtime_checkpoints ( + table_key TEXT PRIMARY KEY CHECK (table_key IN ('doc', 'block')), + schema_fingerprint TEXT NOT NULL, + source_cursor BIGINT NOT NULL CHECK (source_cursor >= 0), + checkpoint_sequence BIGINT NOT NULL CHECK (checkpoint_sequence >= 0), + checkpoint_blob BYTEA NOT NULL, + checksum TEXT NOT NULL, + blob_size BIGINT NOT NULL CHECK (blob_size >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +ALTER TABLE search_runtime_checkpoints ALTER COLUMN checkpoint_blob SET STORAGE EXTERNAL; + +CREATE TABLE workspace_permission_revisions ( + workspace_id VARCHAR PRIMARY KEY REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE CASCADE, + revision BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE workspace_permission_changes ( + workspace_id VARCHAR NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE CASCADE, + revision BIGINT NOT NULL, + doc_id VARCHAR, + scope TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, revision) +); + +CREATE INDEX workspace_permission_changes_created_at + ON workspace_permission_changes(created_at); + +INSERT INTO workspace_permission_revisions(workspace_id, revision) +SELECT id, 0 FROM workspaces; + +CREATE FUNCTION record_workspace_permission_change( + target_workspace_id VARCHAR, + target_doc_id VARCHAR, + target_scope TEXT +) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + next_revision BIGINT; +BEGIN + IF target_workspace_id IS NULL OR + NOT EXISTS (SELECT 1 FROM workspaces WHERE id = target_workspace_id) THEN + RETURN; + END IF; + + INSERT INTO workspace_permission_revisions(workspace_id, revision) + VALUES (target_workspace_id, 1) + ON CONFLICT (workspace_id) DO UPDATE + SET revision = workspace_permission_revisions.revision + 1, + updated_at = now() + RETURNING revision INTO next_revision; + + INSERT INTO workspace_permission_changes(workspace_id, revision, doc_id, scope) + VALUES (target_workspace_id, next_revision, target_doc_id, target_scope); +END; +$$; + +CREATE FUNCTION initialize_workspace_permission_revision() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + INSERT INTO workspace_permission_revisions(workspace_id, revision) VALUES (NEW.id, 0); + RETURN NEW; +END; +$$; + +CREATE FUNCTION bump_workspace_permission_revision() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + old_workspace_id VARCHAR; + new_workspace_id VARCHAR; + old_doc_id VARCHAR; + new_doc_id VARCHAR; + target_scope TEXT; +BEGIN + IF TG_TABLE_NAME = 'entitlements' THEN + old_workspace_id := CASE WHEN TG_OP <> 'INSERT' AND OLD.target_type = 'workspace' THEN OLD.target_id END; + new_workspace_id := CASE WHEN TG_OP <> 'DELETE' AND NEW.target_type = 'workspace' THEN NEW.target_id END; + target_scope := 'capability'; + ELSIF TG_TABLE_NAME = 'workspace_members' THEN + old_workspace_id := CASE WHEN TG_OP <> 'INSERT' THEN OLD.workspace_id END; + new_workspace_id := CASE WHEN TG_OP <> 'DELETE' THEN NEW.workspace_id END; + target_scope := 'membership'; + ELSIF TG_TABLE_NAME = 'workspace_access_policies' THEN + old_workspace_id := CASE WHEN TG_OP <> 'INSERT' THEN OLD.workspace_id END; + new_workspace_id := CASE WHEN TG_OP <> 'DELETE' THEN NEW.workspace_id END; + target_scope := 'workspace_policy'; + ELSIF TG_TABLE_NAME = 'doc_access_policies' THEN + old_workspace_id := CASE WHEN TG_OP <> 'INSERT' THEN OLD.workspace_id END; + new_workspace_id := CASE WHEN TG_OP <> 'DELETE' THEN NEW.workspace_id END; + old_doc_id := CASE WHEN TG_OP <> 'INSERT' THEN OLD.doc_id END; + new_doc_id := CASE WHEN TG_OP <> 'DELETE' THEN NEW.doc_id END; + target_scope := 'doc_policy'; + ELSIF TG_TABLE_NAME = 'doc_grants' THEN + old_workspace_id := CASE WHEN TG_OP <> 'INSERT' THEN OLD.workspace_id END; + new_workspace_id := CASE WHEN TG_OP <> 'DELETE' THEN NEW.workspace_id END; + old_doc_id := CASE WHEN TG_OP <> 'INSERT' THEN OLD.doc_id END; + new_doc_id := CASE WHEN TG_OP <> 'DELETE' THEN NEW.doc_id END; + target_scope := 'doc_grant'; + END IF; + + IF old_workspace_id IS NOT NULL AND old_workspace_id IS DISTINCT FROM new_workspace_id THEN + PERFORM record_workspace_permission_change(old_workspace_id, old_doc_id, target_scope); + END IF; + IF new_workspace_id IS NOT NULL THEN + PERFORM record_workspace_permission_change(new_workspace_id, new_doc_id, target_scope); + ELSIF old_workspace_id IS NOT NULL THEN + PERFORM record_workspace_permission_change(old_workspace_id, old_doc_id, target_scope); + END IF; + + RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; +END; +$$; + +CREATE TRIGGER workspaces_initialize_permission_revision +AFTER INSERT ON workspaces +FOR EACH ROW EXECUTE FUNCTION initialize_workspace_permission_revision(); + +CREATE TRIGGER workspace_members_permission_revision_mutation +AFTER INSERT OR DELETE ON workspace_members +FOR EACH ROW EXECUTE FUNCTION bump_workspace_permission_revision(); +CREATE TRIGGER workspace_members_permission_revision_update +AFTER UPDATE OF workspace_id, user_id, role, state, source ON workspace_members +FOR EACH ROW WHEN (ROW(OLD.workspace_id, OLD.user_id, OLD.role, OLD.state, OLD.source) + IS DISTINCT FROM ROW(NEW.workspace_id, NEW.user_id, NEW.role, NEW.state, NEW.source)) +EXECUTE FUNCTION bump_workspace_permission_revision(); + +CREATE TRIGGER workspace_access_policies_permission_revision_mutation +AFTER INSERT OR DELETE ON workspace_access_policies +FOR EACH ROW EXECUTE FUNCTION bump_workspace_permission_revision(); +CREATE TRIGGER workspace_access_policies_permission_revision_update +AFTER UPDATE OF workspace_id, visibility, sharing_enabled, member_default_doc_role ON workspace_access_policies +FOR EACH ROW WHEN (ROW(OLD.workspace_id, OLD.visibility, OLD.sharing_enabled, OLD.member_default_doc_role) + IS DISTINCT FROM ROW(NEW.workspace_id, NEW.visibility, NEW.sharing_enabled, NEW.member_default_doc_role)) +EXECUTE FUNCTION bump_workspace_permission_revision(); + +CREATE TRIGGER doc_access_policies_permission_revision_mutation +AFTER INSERT OR DELETE ON doc_access_policies +FOR EACH ROW EXECUTE FUNCTION bump_workspace_permission_revision(); +CREATE TRIGGER doc_access_policies_permission_revision_update +AFTER UPDATE OF workspace_id, doc_id, visibility, public_role, member_default_role ON doc_access_policies +FOR EACH ROW WHEN (ROW(OLD.workspace_id, OLD.doc_id, OLD.visibility, OLD.public_role, OLD.member_default_role) + IS DISTINCT FROM ROW(NEW.workspace_id, NEW.doc_id, NEW.visibility, NEW.public_role, NEW.member_default_role)) +EXECUTE FUNCTION bump_workspace_permission_revision(); + +CREATE TRIGGER doc_grants_permission_revision_mutation +AFTER INSERT OR DELETE ON doc_grants +FOR EACH ROW EXECUTE FUNCTION bump_workspace_permission_revision(); +CREATE TRIGGER doc_grants_permission_revision_update +AFTER UPDATE OF workspace_id, doc_id, principal_type, principal_id, role ON doc_grants +FOR EACH ROW WHEN (ROW(OLD.workspace_id, OLD.doc_id, OLD.principal_type, OLD.principal_id, OLD.role) + IS DISTINCT FROM ROW(NEW.workspace_id, NEW.doc_id, NEW.principal_type, NEW.principal_id, NEW.role)) +EXECUTE FUNCTION bump_workspace_permission_revision(); + +CREATE TRIGGER entitlements_permission_revision_mutation +AFTER INSERT OR DELETE ON entitlements +FOR EACH ROW EXECUTE FUNCTION bump_workspace_permission_revision(); +CREATE TRIGGER entitlements_permission_revision_update +AFTER UPDATE OF target_type, target_id, source, plan, status, signed_payload, validated_at, expires_at, grace_until ON entitlements +FOR EACH ROW WHEN (ROW(OLD.target_type, OLD.target_id, OLD.source, OLD.plan, OLD.status, OLD.signed_payload, OLD.validated_at, OLD.expires_at, OLD.grace_until) + IS DISTINCT FROM ROW(NEW.target_type, NEW.target_id, NEW.source, NEW.plan, NEW.status, NEW.signed_payload, NEW.validated_at, NEW.expires_at, NEW.grace_until)) +EXECUTE FUNCTION bump_workspace_permission_revision(); diff --git a/packages/backend/native/src/runtime/sql/search_acl_tokens.sql b/packages/backend/native/src/runtime/sql/search_acl_tokens.sql new file mode 100644 index 0000000000..c3c92f84f7 --- /dev/null +++ b/packages/backend/native/src/runtime/sql/search_acl_tokens.sql @@ -0,0 +1,5 @@ +CREATE TABLE search_runtime_acl_tokens ( + token TEXT PRIMARY KEY, + token_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE CHECK (token_id > 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/packages/backend/native/src/runtime/storage_runtime/blob_cleanup.rs b/packages/backend/native/src/runtime/storage_runtime/blob_cleanup.rs index 6bd55446c3..eef5ed9992 100644 --- a/packages/backend/native/src/runtime/storage_runtime/blob_cleanup.rs +++ b/packages/backend/native/src/runtime/storage_runtime/blob_cleanup.rs @@ -5,7 +5,7 @@ use sqlx::{FromRow, PgPool}; use super::{ RuntimeBlobCleanupExecuteResult, RuntimeBlobCleanupPlanResult, RuntimeError, RuntimeResult, StorageRuntime, - napi_error, + doc_blob_refs::PARSER_VERSION, load_workspace_canonical_doc_ids, napi_error, }; #[derive(FromRow)] @@ -83,6 +83,70 @@ async fn projection_is_stale(pool: &PgPool, workspace_id: &str) -> RuntimeResult .fetch_one(pool) .await .map_err(|err| RuntimeError::database("Blob cleanup retention activity check failed", err))?; + if sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM updates WHERE workspace_id = $1)") + .bind(workspace_id) + .fetch_one(pool) + .await + .map_err(|err| RuntimeError::database("Blob cleanup pending update check failed", err))? + { + return Ok(true); + } + let mut current_doc_ids = match load_workspace_canonical_doc_ids(pool, workspace_id).await { + Ok(ids) => ids, + Err(_) => return Ok(true), + }; + current_doc_ids.push(workspace_id.to_string()); + current_doc_ids.extend( + sqlx::query_scalar::<_, String>( + "SELECT doc_id FROM document_cleanup_candidates WHERE workspace_id = $1 AND status IN ('marked', 'failed')", + ) + .bind(workspace_id) + .fetch_all(pool) + .await + .map_err(|err| RuntimeError::database("Blob cleanup retained document load failed", err))?, + ); + current_doc_ids.sort(); + current_doc_ids.dedup(); + let has_nonfresh_projection = sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS( + SELECT 1 + FROM unnest($2::text[]) AS ids(doc_id) + LEFT JOIN snapshots s + ON s.workspace_id = $1 AND s.guid = ids.doc_id + LEFT JOIN doc_blob_ref_projections p + ON p.workspace_id = $1 AND p.doc_id = ids.doc_id + WHERE s.guid IS NULL + OR p.doc_id IS NULL + OR p.status <> 'fresh' + OR p.parser_version <> $3 + OR p.source_revision IS DISTINCT FROM s.updated_at + ) + OR EXISTS( + SELECT 1 FROM doc_blob_ref_projections + WHERE workspace_id = $1 AND status <> 'fresh' + ) + OR EXISTS( + SELECT 1 + FROM doc_blob_refs r + LEFT JOIN doc_blob_ref_projections p + ON p.workspace_id = r.workspace_id AND p.doc_id = r.doc_id + WHERE r.workspace_id = $1 + AND ( + p.doc_id IS NULL + OR p.status <> 'fresh' + OR r.parser_version <> p.parser_version + OR r.snapshot_updated_at IS DISTINCT FROM p.source_revision + ) + ) + "#, + ) + .bind(workspace_id) + .bind(¤t_doc_ids) + .bind(PARSER_VERSION) + .fetch_one(pool) + .await + .map_err(|err| RuntimeError::database("Blob cleanup projection state check failed", err))?; let has_stale_rows = sqlx::query_scalar::<_, bool>( "SELECT EXISTS(SELECT 1 FROM doc_blob_refs WHERE workspace_id = $1 AND status <> 'fresh')", ) @@ -90,7 +154,7 @@ async fn projection_is_stale(pool: &PgPool, workspace_id: &str) -> RuntimeResult .fetch_one(pool) .await .map_err(|err| RuntimeError::database("Blob cleanup projection freshness check failed", err))?; - Ok(activity_after_checkpoint || has_stale_rows) + Ok(activity_after_checkpoint || has_nonfresh_projection || has_stale_rows) } async fn stale_projection_workspaces(pool: &PgPool, workspace_id: &str) -> RuntimeResult> { @@ -107,18 +171,31 @@ async fn metadata_backfill_is_complete(pool: &PgPool, workspace_id: &str) -> Run async fn has_doc_ref(pool: &PgPool, workspace_id: &str, key: &str) -> RuntimeResult { sqlx::query_scalar::<_, bool>( - "SELECT EXISTS(SELECT 1 FROM doc_blob_refs WHERE workspace_id = $1 AND blob_key = $2 AND status = 'fresh')", + r#" + SELECT EXISTS( + SELECT 1 + FROM doc_blob_refs r + JOIN doc_blob_ref_projections p + ON p.workspace_id = r.workspace_id AND p.doc_id = r.doc_id + WHERE r.workspace_id = $1 + AND r.blob_key = $2 + AND r.status = 'fresh' + AND p.status = 'fresh' + AND p.parser_version = $3 + AND r.parser_version = p.parser_version + AND r.snapshot_updated_at = p.source_revision + ) + "#, ) .bind(workspace_id) .bind(key) + .bind(PARSER_VERSION) .fetch_one(pool) .await .map_err(|err| RuntimeError::database("Blob cleanup doc ref check failed", err)) } async fn has_other_ref(pool: &PgPool, workspace_id: &str, key: &str) -> RuntimeResult { - // Remove the ai_contexts branch after stable and beta no longer run binaries - // built with the 115-migration schema. let required_ref = sqlx::query_scalar::<_, bool>( r#" SELECT EXISTS(SELECT 1 FROM workspaces WHERE id = $1 AND avatar_key = $2) @@ -131,17 +208,6 @@ async fn has_other_ref(pool: &PgPool, workspace_id: &str, key: &str) -> RuntimeR AND storage_key = concat($1, '/', $2) AND status IN ('reserving', 'ready') ) - OR EXISTS( - SELECT 1 - FROM ai_contexts c - JOIN ai_sessions_metadata s ON s.id = c.session_id - WHERE s.workspace_id = $1 - AND jsonb_path_exists( - c.config::jsonb, - '$.** ? (@ == $blobKey)', - jsonb_build_object('blobKey', to_jsonb($2::text)) - ) - ) "#, ) .bind(workspace_id) diff --git a/packages/backend/native/src/runtime/storage_runtime/current_doc.rs b/packages/backend/native/src/runtime/storage_runtime/current_doc.rs index 17cee0fdca..c1b380c9d0 100644 --- a/packages/backend/native/src/runtime/storage_runtime/current_doc.rs +++ b/packages/backend/native/src/runtime/storage_runtime/current_doc.rs @@ -6,8 +6,6 @@ use super::{RuntimeError, RuntimeResult}; #[derive(FromRow)] pub(in crate::runtime) struct CurrentDoc { - pub(in crate::runtime) workspace_id: String, - pub(in crate::runtime) doc_id: String, pub(in crate::runtime) blob: Vec, pub(in crate::runtime) updated_at: DateTime, } @@ -25,7 +23,7 @@ pub(in crate::runtime) async fn load_current_doc( ) -> RuntimeResult> { let snapshot = sqlx::query_as::<_, CurrentDoc>( r#" - SELECT workspace_id, guid AS doc_id, blob, updated_at + SELECT blob, updated_at FROM snapshots WHERE workspace_id = $1 AND guid = $2 "#, @@ -48,12 +46,38 @@ pub(in crate::runtime) async fn load_current_doc( .fetch_all(pool) .await .map_err(|err| RuntimeError::database("Current doc updates load failed", err))?; - merge_current_doc(workspace_id, doc_id, snapshot, updates) + merge_current_doc(snapshot, updates) +} + +pub(super) async fn load_canonical_doc( + pool: &PgPool, + workspace_id: &str, + doc_id: &str, +) -> RuntimeResult> { + sqlx::query_as::<_, CurrentDoc>( + r#" + SELECT blob, updated_at + FROM snapshots + WHERE workspace_id = $1 AND guid = $2 + "#, + ) + .bind(workspace_id) + .bind(doc_id) + .fetch_optional(pool) + .await + .map_err(|err| RuntimeError::database("Canonical doc snapshot load failed", err)) +} + +pub(super) async fn has_pending_updates(pool: &PgPool, workspace_id: &str, doc_id: &str) -> RuntimeResult { + sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM updates WHERE workspace_id = $1 AND guid = $2)") + .bind(workspace_id) + .bind(doc_id) + .fetch_one(pool) + .await + .map_err(|err| RuntimeError::database("Pending doc updates check failed", err)) } pub(super) fn merge_current_doc( - workspace_id: &str, - doc_id: &str, snapshot: Option, updates: Vec, ) -> RuntimeResult> { @@ -84,16 +108,18 @@ pub(super) fn merge_current_doc( .encode_update_v1() .map_err(|err| RuntimeError::invalid_state(format!("Current doc encode failed: {err}")))?; - Ok(Some(CurrentDoc { - workspace_id: workspace_id.to_string(), - doc_id: doc_id.to_string(), - blob, - updated_at, - })) + Ok(Some(CurrentDoc { blob, updated_at })) } pub(super) async fn load_workspace_live_doc_ids(pool: &PgPool, workspace_id: &str) -> RuntimeResult> { - workspace_live_doc_ids(load_current_doc(pool, workspace_id, workspace_id).await?) + load_workspace_canonical_doc_ids(pool, workspace_id).await +} + +pub(super) async fn load_workspace_canonical_doc_ids(pool: &PgPool, workspace_id: &str) -> RuntimeResult> { + if has_pending_updates(pool, workspace_id, workspace_id).await? { + return Err(RuntimeError::invalid_state("Workspace root doc has pending updates")); + } + workspace_live_doc_ids(load_canonical_doc(pool, workspace_id, workspace_id).await?) } fn workspace_live_doc_ids(root: Option) -> RuntimeResult> { @@ -120,11 +146,7 @@ mod tests { let snapshot = affine_doc_loader::add_doc_to_root_doc(Vec::new(), "live", None).unwrap(); let pending = affine_doc_loader::add_doc_to_root_doc(snapshot.clone(), "trash", None).unwrap(); let merged = merge_current_doc( - "workspace", - "workspace", Some(CurrentDoc { - workspace_id: "workspace".to_string(), - doc_id: "workspace".to_string(), blob: snapshot, updated_at: Utc::now(), }), @@ -149,8 +171,6 @@ mod tests { trash.insert("trash".to_string(), Value::Any(Any::True)).unwrap(); let ids = workspace_live_doc_ids(Some(CurrentDoc { - workspace_id: "workspace".to_string(), - doc_id: "workspace".to_string(), blob: root.encode_update_v1().unwrap(), updated_at: Utc::now(), })) @@ -165,8 +185,6 @@ mod tests { .unwrap(); pages.remove(trash_index as u64, 1).unwrap(); let ids = workspace_live_doc_ids(Some(CurrentDoc { - workspace_id: "workspace".to_string(), - doc_id: "workspace".to_string(), blob: root.encode_update_v1().unwrap(), updated_at: Utc::now(), })) @@ -179,8 +197,6 @@ mod tests { assert!(workspace_live_doc_ids(None).is_err()); assert!( workspace_live_doc_ids(Some(CurrentDoc { - workspace_id: "workspace".to_string(), - doc_id: "workspace".to_string(), blob: vec![0xff], updated_at: Utc::now(), })) @@ -188,8 +204,6 @@ mod tests { ); assert!( workspace_live_doc_ids(Some(CurrentDoc { - workspace_id: "workspace".to_string(), - doc_id: "workspace".to_string(), blob: vec![ 1, 1, 1, 1, 40, 0, 1, 0, 11, 115, 117, 98, 95, 109, 97, 112, 95, 107, 101, 121, 1, 119, 13, 115, 117, 98, 95, 109, 97, 112, 95, 118, 97, 108, 117, 101, 0, diff --git a/packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs b/packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs index 68836348a8..0d4fea6bea 100644 --- a/packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs +++ b/packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs @@ -1,24 +1,53 @@ use affine_doc_loader as doc_loader; use chrono::{DateTime, Utc}; -use sqlx::PgPool; +use sqlx::{Executor, FromRow, PgPool, Postgres}; use super::{ - CurrentDoc, RuntimeDocBlobRefsResult, RuntimeError, RuntimeResult, StorageRuntime, load_current_doc, - load_workspace_live_doc_ids, napi_error, + CurrentDoc, RuntimeDocBlobRefsResult, RuntimeError, RuntimeResult, StorageRuntime, load_canonical_doc, + load_workspace_canonical_doc_ids, napi_error, }; -const PARSER_VERSION: i32 = 1; +pub(super) const PARSER_VERSION: i32 = 1; +const ERROR_SUMMARY_LIMIT: usize = 512; type ExtractedRef = doc_loader::BlobRef; +#[derive(FromRow)] +struct DocSource { + updated_at: DateTime, + has_pending_updates: bool, +} + #[derive(Default)] struct ProjectionState { cursor: Option, failed_docs: i64, } +#[derive(Default)] +struct ProjectionStats { + result: RuntimeDocBlobRefsResult, + pending_docs: i64, + missing_docs: i64, + shadow_mismatches: i64, +} + +#[derive(Default)] +struct ProjectionAttempt { + written: i64, + deleted: i64, + shadow_mismatch: bool, +} + +enum ProjectionOutcome { + Fresh(ProjectionAttempt), + Pending, + Missing, +} + async fn load_workspace_doc_ids(pool: &PgPool, workspace_id: &str) -> RuntimeResult> { - let mut ids = load_workspace_live_doc_ids(pool, workspace_id).await?; + let mut ids = load_workspace_canonical_doc_ids(pool, workspace_id).await?; + ids.push(workspace_id.to_string()); let retained = sqlx::query_scalar::<_, String>( "SELECT doc_id FROM document_cleanup_candidates WHERE workspace_id = $1 AND status IN ('marked', 'failed') ORDER \ BY doc_id", @@ -33,15 +62,127 @@ async fn load_workspace_doc_ids(pool: &PgPool, workspace_id: &str) -> RuntimeRes Ok(ids) } +async fn load_doc_source(pool: &PgPool, workspace_id: &str, doc_id: &str) -> RuntimeResult> { + sqlx::query_as::<_, DocSource>( + r#" + SELECT s.updated_at, + EXISTS( + SELECT 1 FROM updates u + WHERE u.workspace_id = s.workspace_id AND u.guid = s.guid + ) AS has_pending_updates + FROM snapshots s + WHERE s.workspace_id = $1 AND s.guid = $2 + "#, + ) + .bind(workspace_id) + .bind(doc_id) + .fetch_optional(pool) + .await + .map_err(|err| RuntimeError::database("Doc blob refs source load failed", err)) +} + +async fn projection_is_fresh( + pool: &PgPool, + workspace_id: &str, + doc_id: &str, + source_revision: DateTime, +) -> RuntimeResult { + sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS( + SELECT 1 FROM doc_blob_ref_projections + WHERE workspace_id = $1 + AND doc_id = $2 + AND source_revision = $3 + AND parser_version = $4 + AND status = 'fresh' + ) + "#, + ) + .bind(workspace_id) + .bind(doc_id) + .bind(source_revision) + .bind(PARSER_VERSION) + .fetch_one(pool) + .await + .map_err(|err| RuntimeError::database("Doc blob refs projection freshness load failed", err)) +} + +fn truncate_error_summary(error: &str) -> String { + let mut end = error.len().min(ERROR_SUMMARY_LIMIT); + while end > 0 && !error.is_char_boundary(end) { + end -= 1; + } + error[..end].to_string() +} + +async fn upsert_projection_state<'e, E>( + executor: E, + workspace_id: &str, + doc_id: &str, + source_revision: Option>, + status: &str, + error_code: Option<&str>, + error_summary: Option<&str>, +) -> RuntimeResult<()> +where + E: Executor<'e, Database = Postgres>, +{ + sqlx::query( + r#" + INSERT INTO doc_blob_ref_projections + (workspace_id, doc_id, source_revision, parser_version, status, indexed_at, error_code, error_summary, attempt_count) + VALUES ($1, $2, $3, $4, $5, CURRENT_TIMESTAMP, $6, $7, CASE WHEN $5 = 'fresh' THEN 0 ELSE 1 END) + ON CONFLICT (workspace_id, doc_id) DO UPDATE + SET source_revision = EXCLUDED.source_revision, + parser_version = EXCLUDED.parser_version, + status = EXCLUDED.status, + indexed_at = EXCLUDED.indexed_at, + error_code = EXCLUDED.error_code, + error_summary = EXCLUDED.error_summary, + attempt_count = CASE + WHEN EXCLUDED.status = 'fresh' THEN 0 + ELSE doc_blob_ref_projections.attempt_count + 1 + END, + updated_at = CURRENT_TIMESTAMP + WHERE ( + EXCLUDED.source_revision IS NULL + AND doc_blob_ref_projections.source_revision IS NULL + AND doc_blob_ref_projections.parser_version <= EXCLUDED.parser_version + ) OR ( + EXCLUDED.source_revision IS NOT NULL + AND doc_blob_ref_projections.parser_version <= EXCLUDED.parser_version + AND ( + doc_blob_ref_projections.source_revision IS NULL + OR EXCLUDED.source_revision >= doc_blob_ref_projections.source_revision + ) + ) + "#, + ) + .bind(workspace_id) + .bind(doc_id) + .bind(source_revision) + .bind(PARSER_VERSION) + .bind(status) + .bind(error_code) + .bind(error_summary.map(truncate_error_summary)) + .execute(executor) + .await + .map_err(|err| RuntimeError::database("Doc blob refs projection state write failed", err))?; + Ok(()) +} + async fn upsert_projection_checkpoint( pool: &PgPool, workspace_id: &str, result: &RuntimeDocBlobRefsResult, - failed_docs: i64, + pending_docs: i64, + missing_docs: i64, + shadow_mismatches: i64, ) -> RuntimeResult<()> { let status = if result.next_cursor.is_some() { "running" - } else if failed_docs > 0 { + } else if result.failed_docs > 0 { "failed" } else { "completed" @@ -66,7 +207,10 @@ async fn upsert_projection_checkpoint( .bind(completed) .bind(serde_json::json!({ "parserVersion": PARSER_VERSION, - "failedDocs": failed_docs, + "failedDocs": result.failed_docs, + "pendingDocs": pending_docs, + "missingDocs": missing_docs, + "shadowMismatches": shadow_mismatches, })) .execute(pool) .await @@ -74,7 +218,7 @@ async fn upsert_projection_checkpoint( Ok(()) } -async fn upsert_projection_failure_checkpoint(pool: &PgPool, workspace_id: &str, error: &str) -> RuntimeResult<()> { +async fn upsert_projection_failure_checkpoint(pool: &PgPool, workspace_id: &str) -> RuntimeResult<()> { sqlx::query( r#" INSERT INTO storage_reconciliation_checkpoints @@ -91,7 +235,7 @@ async fn upsert_projection_failure_checkpoint(pool: &PgPool, workspace_id: &str, .bind(workspace_id) .bind(serde_json::json!({ "parserVersion": PARSER_VERSION, - "error": error, + "errorCode": "root_projection_failed", })) .execute(pool) .await @@ -111,43 +255,46 @@ async fn load_projection_state(pool: &PgPool, workspace_id: &str) -> RuntimeResu let Some((status, cursor, metadata)) = checkpoint else { return Ok(ProjectionState::default()); }; - if status != "running" && status != "failed" { + if status != "running" && status != "failed" + || metadata.get("parserVersion").and_then(serde_json::Value::as_i64) != Some(i64::from(PARSER_VERSION)) + { return Ok(ProjectionState::default()); } - if metadata.get("parserVersion").and_then(serde_json::Value::as_i64) != Some(i64::from(PARSER_VERSION)) { - return Ok(ProjectionState::default()); - } - let cursor = cursor - .get("lastDocId") - .and_then(|value| value.as_str()) - .map(ToString::to_string); - let Some(cursor) = cursor else { - return Ok(ProjectionState::default()); - }; - let failed_docs = metadata - .get("failedDocs") - .and_then(serde_json::Value::as_i64) - .unwrap_or(i64::from(status == "failed")); Ok(ProjectionState { - cursor: Some(cursor), - failed_docs, + cursor: cursor + .get("lastDocId") + .and_then(|value| value.as_str()) + .map(ToString::to_string), + failed_docs: if status == "running" { + metadata + .get("failedDocs") + .and_then(serde_json::Value::as_i64) + .unwrap_or(0) + } else { + 0 + }, }) } -async fn purge_removed_doc_refs(pool: &PgPool, workspace_id: &str, current_doc_ids: &[String]) -> RuntimeResult { - let result = sqlx::query( - r#" - DELETE FROM doc_blob_refs - WHERE workspace_id = $1 - AND NOT (doc_id = ANY($2)) - "#, - ) - .bind(workspace_id) - .bind(current_doc_ids) - .execute(pool) - .await - .map_err(|err| RuntimeError::database("Doc blob refs purge removed docs failed", err))?; - Ok(result.rows_affected() as i64) +async fn purge_removed_doc_projections( + pool: &PgPool, + workspace_id: &str, + current_doc_ids: &[String], +) -> RuntimeResult { + let refs = sqlx::query("DELETE FROM doc_blob_refs WHERE workspace_id = $1 AND NOT (doc_id = ANY($2))") + .bind(workspace_id) + .bind(current_doc_ids) + .execute(pool) + .await + .map_err(|err| RuntimeError::database("Doc blob refs purge removed docs failed", err))? + .rows_affected() as i64; + sqlx::query("DELETE FROM doc_blob_ref_projections WHERE workspace_id = $1 AND NOT (doc_id = ANY($2))") + .bind(workspace_id) + .bind(current_doc_ids) + .execute(pool) + .await + .map_err(|err| RuntimeError::database("Doc blob ref projections purge removed docs failed", err))?; + Ok(refs) } fn extract_refs(blob: Vec) -> RuntimeResult> { @@ -155,6 +302,283 @@ fn extract_refs(blob: Vec) -> RuntimeResult> { .map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs parse failed: {err}"))) } +async fn replace_doc_refs_if_current( + pool: &PgPool, + workspace_id: &str, + doc_id: &str, + source_revision: DateTime, + refs: Vec, +) -> RuntimeResult { + let mut tx = pool + .begin() + .await + .map_err(|err| RuntimeError::database("Doc blob refs transaction failed", err))?; + let current = sqlx::query_as::<_, (DateTime, bool)>( + r#" + SELECT s.updated_at, + EXISTS( + SELECT 1 FROM updates u + WHERE u.workspace_id = s.workspace_id AND u.guid = s.guid + ) AS has_pending_updates + FROM snapshots s + WHERE s.workspace_id = $1 AND s.guid = $2 + "#, + ) + .bind(workspace_id) + .bind(doc_id) + .fetch_optional(&mut *tx) + .await + .map_err(|err| RuntimeError::database("Doc blob refs CAS source load failed", err))?; + let Some((current_revision, has_pending_updates)) = current else { + tx.rollback() + .await + .map_err(|err| RuntimeError::database("Doc blob refs CAS rollback failed", err))?; + upsert_projection_state( + pool, + workspace_id, + doc_id, + None, + "missing", + Some("snapshot_missing"), + None, + ) + .await?; + return Ok(ProjectionOutcome::Missing); + }; + if current_revision != source_revision || has_pending_updates { + tx.rollback() + .await + .map_err(|err| RuntimeError::database("Doc blob refs CAS rollback failed", err))?; + upsert_projection_state( + pool, + workspace_id, + doc_id, + Some(source_revision), + "pending", + Some("source_changed"), + None, + ) + .await?; + return Ok(ProjectionOutcome::Pending); + } + let projection = sqlx::query_as::<_, (i32, Option>)>( + "SELECT parser_version, source_revision FROM doc_blob_ref_projections WHERE workspace_id = $1 AND doc_id = $2", + ) + .bind(workspace_id) + .bind(doc_id) + .fetch_optional(&mut *tx) + .await + .map_err(|err| RuntimeError::database("Doc blob refs projection CAS load failed", err))?; + if projection.is_some_and(|(parser_version, projection_revision)| { + parser_version > PARSER_VERSION || projection_revision.is_some_and(|revision| revision > source_revision) + }) { + tx.rollback() + .await + .map_err(|err| RuntimeError::database("Doc blob refs projection CAS rollback failed", err))?; + upsert_projection_state( + pool, + workspace_id, + doc_id, + Some(source_revision), + "pending", + Some("projection_newer"), + None, + ) + .await?; + return Ok(ProjectionOutcome::Pending); + } + + let mut old_refs = sqlx::query_as::<_, (String, String, String)>( + "SELECT blob_key, block_id, flavour FROM doc_blob_refs WHERE workspace_id = $1 AND doc_id = $2", + ) + .bind(workspace_id) + .bind(doc_id) + .fetch_all(&mut *tx) + .await + .map_err(|err| RuntimeError::database("Doc blob refs shadow load failed", err))?; + old_refs.sort(); + let mut new_refs = refs + .iter() + .map(|reference| { + ( + reference.blob_key.clone(), + reference.block_id.clone(), + reference.flavour.clone(), + ) + }) + .collect::>(); + new_refs.sort(); + let shadow_mismatch = old_refs != new_refs; + + let deleted = sqlx::query("DELETE FROM doc_blob_refs WHERE workspace_id = $1 AND doc_id = $2") + .bind(workspace_id) + .bind(doc_id) + .execute(&mut *tx) + .await + .map_err(|err| RuntimeError::database("Doc blob refs delete failed", err))? + .rows_affected() as i64; + let mut written = 0; + for reference in refs { + written += sqlx::query( + r#" + INSERT INTO doc_blob_refs + (workspace_id, doc_id, blob_key, block_id, flavour, snapshot_updated_at, parser_version, status, error) + VALUES ($1, $2, $3, $4, $5, $6, $7, 'fresh', NULL) + ON CONFLICT (workspace_id, doc_id, blob_key, block_id) DO UPDATE + SET flavour = EXCLUDED.flavour, + snapshot_updated_at = EXCLUDED.snapshot_updated_at, + indexed_at = CURRENT_TIMESTAMP, + parser_version = EXCLUDED.parser_version, + status = 'fresh', + error = NULL + "#, + ) + .bind(workspace_id) + .bind(doc_id) + .bind(reference.blob_key) + .bind(reference.block_id) + .bind(reference.flavour) + .bind(source_revision) + .bind(PARSER_VERSION) + .execute(&mut *tx) + .await + .map_err(|err| RuntimeError::database("Doc blob refs insert failed", err))? + .rows_affected() as i64; + } + upsert_projection_state( + &mut *tx, + workspace_id, + doc_id, + Some(source_revision), + "fresh", + None, + None, + ) + .await?; + tx.commit() + .await + .map_err(|err| RuntimeError::database("Doc blob refs transaction commit failed", err))?; + Ok(ProjectionOutcome::Fresh(ProjectionAttempt { + written, + deleted, + shadow_mismatch, + })) +} + +async fn rebuild_doc_blob_refs_inner( + runtime: &StorageRuntime, + workspace_id: &str, + doc_id: &str, + expected_source_revision: Option, +) -> RuntimeResult { + let pool = runtime.pool().await?; + let mut stats = ProjectionStats::default(); + stats.result.scanned_docs = 1; + let Some(source) = load_doc_source(&pool, workspace_id, doc_id).await? else { + upsert_projection_state( + &pool, + workspace_id, + doc_id, + None, + "missing", + Some("snapshot_missing"), + None, + ) + .await?; + stats.result.failed_docs = 1; + stats.missing_docs = 1; + return Ok(stats); + }; + if expected_source_revision.is_some_and(|revision| source.updated_at.timestamp_millis() != revision) { + upsert_projection_state( + &pool, + workspace_id, + doc_id, + Some(source.updated_at), + "pending", + Some("source_changed"), + None, + ) + .await?; + stats.pending_docs = 1; + return Ok(stats); + } + if source.has_pending_updates { + upsert_projection_state( + &pool, + workspace_id, + doc_id, + Some(source.updated_at), + "pending", + Some("pending_updates"), + None, + ) + .await?; + stats.pending_docs = 1; + return Ok(stats); + } + if projection_is_fresh(&pool, workspace_id, doc_id, source.updated_at).await? { + return Ok(stats); + } + upsert_projection_state( + &pool, + workspace_id, + doc_id, + Some(source.updated_at), + "running", + None, + None, + ) + .await?; + let Some(snapshot) = load_canonical_doc(&pool, workspace_id, doc_id).await? else { + upsert_projection_state( + &pool, + workspace_id, + doc_id, + None, + "missing", + Some("snapshot_missing"), + None, + ) + .await?; + stats.result.failed_docs = 1; + stats.missing_docs = 1; + return Ok(stats); + }; + let CurrentDoc { blob, updated_at, .. } = snapshot; + let refs = match extract_refs(blob) { + Ok(refs) => refs, + Err(_) => { + upsert_projection_state( + &pool, + workspace_id, + doc_id, + Some(updated_at), + "failed", + Some("parse_failed"), + Some("canonical snapshot parser rejected the document"), + ) + .await?; + stats.result.failed_docs = 1; + return Ok(stats); + } + }; + match replace_doc_refs_if_current(&pool, workspace_id, doc_id, updated_at, refs).await? { + ProjectionOutcome::Fresh(attempt) => { + stats.result.parsed_docs = 1; + stats.result.refs_written = attempt.written; + stats.result.refs_deleted = attempt.deleted; + stats.shadow_mismatches = i64::from(attempt.shadow_mismatch); + } + ProjectionOutcome::Pending => stats.pending_docs = 1, + ProjectionOutcome::Missing => { + stats.result.failed_docs = 1; + stats.missing_docs = 1; + } + } + Ok(stats) +} + #[cfg(test)] mod tests { use chrono::Utc; @@ -168,14 +592,10 @@ mod tests { let blob = doc_loader::build_full_doc("Doc", "![Alt](blob://image-blob-key)", &doc_id).expect("doc fixture should build"); let snapshot = CurrentDoc { - workspace_id: "workspace".to_string(), - doc_id, blob, updated_at: Utc::now(), }; - let refs = extract_refs(snapshot.blob).expect("refs should parse"); - assert!( refs .iter() @@ -201,144 +621,25 @@ mod tests { meta .insert("pages".to_string(), pages) .expect("root pages should insert"); - let root = root.encode_update_v1().expect("root doc should encode"); let ids = doc_loader::get_doc_ids_from_binary(root, true).expect("root doc ids should parse"); assert_eq!(ids, vec!["active-doc", "trashed-doc"]); } #[test] - fn doc_blob_refs_rejects_corrupt_docs() { + fn doc_blob_refs_rejects_corrupt_docs_without_a_failure_ref() { let snapshot = CurrentDoc { - workspace_id: "workspace".to_string(), - doc_id: "corrupt".to_string(), blob: vec![0xff], updated_at: Utc::now(), }; - assert!(extract_refs(snapshot.blob).is_err()); } -} -async fn replace_doc_refs( - pool: &PgPool, - workspace_id: &str, - doc_id: &str, - updated_at: DateTime, - refs: Vec, -) -> RuntimeResult<(i64, i64)> { - let mut tx = pool - .begin() - .await - .map_err(|err| RuntimeError::database("Doc blob refs transaction failed", err))?; - - let deleted = sqlx::query("DELETE FROM doc_blob_refs WHERE workspace_id = $1 AND doc_id = $2") - .bind(workspace_id) - .bind(doc_id) - .execute(&mut *tx) - .await - .map_err(|err| RuntimeError::database("Doc blob refs delete failed", err))? - .rows_affected() as i64; - - let mut written = 0; - for reference in refs { - let affected = sqlx::query( - r#" - INSERT INTO doc_blob_refs - (workspace_id, doc_id, blob_key, block_id, flavour, snapshot_updated_at, parser_version, status) - VALUES ($1, $2, $3, $4, $5, $6, $7, 'fresh') - ON CONFLICT (workspace_id, doc_id, blob_key, block_id) DO UPDATE - SET flavour = EXCLUDED.flavour, - snapshot_updated_at = EXCLUDED.snapshot_updated_at, - indexed_at = CURRENT_TIMESTAMP, - parser_version = EXCLUDED.parser_version, - status = 'fresh', - error = NULL - "#, - ) - .bind(workspace_id) - .bind(doc_id) - .bind(reference.blob_key) - .bind(reference.block_id) - .bind(reference.flavour) - .bind(updated_at) - .bind(PARSER_VERSION) - .execute(&mut *tx) - .await - .map_err(|err| RuntimeError::database("Doc blob refs insert failed", err))? - .rows_affected() as i64; - written += affected; + #[test] + fn error_summary_is_bounded() { + let error = "x".repeat(ERROR_SUMMARY_LIMIT + 20); + assert_eq!(truncate_error_summary(&error).len(), ERROR_SUMMARY_LIMIT); } - - tx.commit() - .await - .map_err(|err| RuntimeError::database("Doc blob refs transaction commit failed", err))?; - Ok((written, deleted)) -} - -async fn mark_doc_failed(pool: &PgPool, workspace_id: &str, doc_id: &str, error: &str) -> RuntimeResult<()> { - sqlx::query( - r#" - INSERT INTO doc_blob_refs - (workspace_id, doc_id, blob_key, block_id, flavour, snapshot_updated_at, parser_version, status, error) - VALUES ($1, $2, '__parse_failed__', '__parse_failed__', '__parse_failed__', CURRENT_TIMESTAMP, $3, 'failed', $4) - ON CONFLICT (workspace_id, doc_id, blob_key, block_id) DO UPDATE - SET indexed_at = CURRENT_TIMESTAMP, - status = 'failed', - error = EXCLUDED.error - "#, - ) - .bind(workspace_id) - .bind(doc_id) - .bind(PARSER_VERSION) - .bind(error) - .execute(pool) - .await - .map_err(|err| RuntimeError::database("Doc blob refs mark failure failed", err))?; - Ok(()) -} - -async fn rebuild_doc_blob_refs_inner( - runtime: &StorageRuntime, - workspace_id: String, - doc_id: String, -) -> RuntimeResult { - let pool = runtime.pool().await?; - let mut result = RuntimeDocBlobRefsResult { - scanned_docs: 1, - parsed_docs: 0, - refs_written: 0, - refs_deleted: 0, - failed_docs: 0, - next_cursor: None, - }; - - let Some(snapshot) = load_current_doc(&pool, &workspace_id, &doc_id).await? else { - result.failed_docs = 1; - mark_doc_failed(&pool, &workspace_id, &doc_id, "snapshot_missing").await?; - return Ok(result); - }; - - let CurrentDoc { - workspace_id, - doc_id, - blob, - updated_at, - } = snapshot; - match extract_refs(blob) { - Ok(refs) => { - let (written, deleted) = replace_doc_refs(&pool, &workspace_id, &doc_id, updated_at, refs).await?; - result.parsed_docs = 1; - result.refs_written = written; - result.refs_deleted = deleted; - } - Err(err) => { - result.failed_docs = 1; - mark_doc_failed(&pool, &workspace_id, &doc_id, &err.to_string()).await?; - } - } - - Ok(result) } #[napi_derive::napi] @@ -348,8 +649,13 @@ impl StorageRuntime { &self, workspace_id: String, doc_id: String, + source_revision: i64, ) -> napi::Result { - Ok(rebuild_doc_blob_refs_inner(self, workspace_id, doc_id).await?) + Ok( + rebuild_doc_blob_refs_inner(self, &workspace_id, &doc_id, Some(source_revision)) + .await? + .result, + ) } #[napi] @@ -361,12 +667,11 @@ impl StorageRuntime { if limit <= 0 { return Err(napi_error("doc blob refs rebuild limit must be positive")); } - let pool = self.pool().await?; let doc_ids = match load_workspace_doc_ids(&pool, &workspace_id).await { Ok(doc_ids) => doc_ids, Err(err) => { - upsert_projection_failure_checkpoint(&pool, &workspace_id, &err.to_string()).await?; + upsert_projection_failure_checkpoint(&pool, &workspace_id).await?; return Err(err.into()); } }; @@ -377,34 +682,35 @@ impl StorageRuntime { .filter(|doc_id| state.cursor.as_ref().is_none_or(|cursor| doc_id > cursor)) .collect::>(); let has_more = doc_ids.len() > limit as usize; - let mut total = RuntimeDocBlobRefsResult { - scanned_docs: 0, - parsed_docs: 0, - refs_written: 0, - refs_deleted: 0, - failed_docs: 0, - next_cursor: None, - }; - + let mut total = ProjectionStats::default(); + total.result.failed_docs = state.failed_docs; let mut last_doc_id = None; for doc_id in doc_ids.into_iter().take(limit as usize) { last_doc_id = Some(doc_id.clone()); - let result = rebuild_doc_blob_refs_inner(self, workspace_id.clone(), doc_id).await?; - total.scanned_docs += result.scanned_docs; - total.parsed_docs += result.parsed_docs; - total.refs_written += result.refs_written; - total.refs_deleted += result.refs_deleted; - total.failed_docs += result.failed_docs; + let stats = rebuild_doc_blob_refs_inner(self, &workspace_id, &doc_id, None).await?; + total.result.scanned_docs += stats.result.scanned_docs; + total.result.parsed_docs += stats.result.parsed_docs; + total.result.refs_written += stats.result.refs_written; + total.result.refs_deleted += stats.result.refs_deleted; + total.result.failed_docs += stats.result.failed_docs; + total.pending_docs += stats.pending_docs; + total.missing_docs += stats.missing_docs; + total.shadow_mismatches += stats.shadow_mismatches; } - let failed_docs = state.failed_docs + total.failed_docs; if has_more { - total.next_cursor = last_doc_id; - } else if failed_docs == 0 { - total.refs_deleted += purge_removed_doc_refs(&pool, &workspace_id, ¤t_doc_ids).await?; + total.result.next_cursor = last_doc_id; + } else if total.result.failed_docs == 0 && total.pending_docs == 0 { + total.result.refs_deleted += purge_removed_doc_projections(&pool, &workspace_id, ¤t_doc_ids).await?; } - - upsert_projection_checkpoint(&pool, &workspace_id, &total, failed_docs).await?; - - Ok(total) + upsert_projection_checkpoint( + &pool, + &workspace_id, + &total.result, + total.pending_docs, + total.missing_docs, + total.shadow_mismatches, + ) + .await?; + Ok(total.result) } } diff --git a/packages/backend/native/src/runtime/storage_runtime/document_cleanup.rs b/packages/backend/native/src/runtime/storage_runtime/document_cleanup.rs index 54fc1c203b..ae472ab05d 100644 --- a/packages/backend/native/src/runtime/storage_runtime/document_cleanup.rs +++ b/packages/backend/native/src/runtime/storage_runtime/document_cleanup.rs @@ -269,14 +269,13 @@ async fn load_current_doc_for_update( workspace_id: &str, doc_id: &str, ) -> RuntimeResult> { - let snapshot = sqlx::query_as::<_, CurrentDoc>( - "SELECT workspace_id, guid AS doc_id, blob, updated_at FROM snapshots WHERE workspace_id = $1 AND guid = $2", - ) - .bind(workspace_id) - .bind(doc_id) - .fetch_optional(&mut **tx) - .await - .map_err(|err| RuntimeError::database("Document cleanup current snapshot load failed", err))?; + let snapshot = + sqlx::query_as::<_, CurrentDoc>("SELECT blob, updated_at FROM snapshots WHERE workspace_id = $1 AND guid = $2") + .bind(workspace_id) + .bind(doc_id) + .fetch_optional(&mut **tx) + .await + .map_err(|err| RuntimeError::database("Document cleanup current snapshot load failed", err))?; let updates = sqlx::query_as::<_, CurrentDocUpdate>( "SELECT blob, created_at FROM updates WHERE workspace_id = $1 AND guid = $2 ORDER BY created_at ASC", ) @@ -285,7 +284,7 @@ async fn load_current_doc_for_update( .fetch_all(&mut **tx) .await .map_err(|err| RuntimeError::database("Document cleanup current updates load failed", err))?; - merge_current_doc(workspace_id, doc_id, snapshot, updates) + merge_current_doc(snapshot, updates) } async fn current_activity( @@ -358,6 +357,7 @@ async fn delete_doc_rows(tx: &mut Transaction<'_, Postgres>, candidate: &Candida ("doc_access_policies", "doc_id"), ("doc_grants", "doc_id"), ("doc_blob_refs", "doc_id"), + ("doc_blob_ref_projections", "doc_id"), ("ai_workspace_ignored_docs", "doc_id"), ("comments", "doc_id"), ("comment_attachments", "doc_id"), @@ -937,6 +937,7 @@ mod tests { "blob_cleanup_candidates", "document_cleanup_candidates", "doc_blob_refs", + "doc_blob_ref_projections", ] { sqlx::query(&format!("DELETE FROM {table} WHERE workspace_id = $1")) .bind(workspace_id) @@ -1001,6 +1002,59 @@ mod tests { .await .map_err(|err| anyhow::anyhow!(err.to_string()))?; assert_eq!(projection.failed_docs, 0); + assert_eq!(projection.parsed_docs, 3); + let projection_checkpoint = sqlx::query( + "SELECT metadata FROM storage_reconciliation_checkpoints WHERE kind = 'doc_blob_refs' AND scope = $1", + ) + .bind(&workspace_id) + .fetch_one(&pool) + .await?; + assert_eq!(projection_checkpoint.get::("metadata")["shadowMismatches"], 1); + assert_eq!( + sqlx::query_scalar::<_, String>( + "SELECT status FROM doc_blob_ref_projections WHERE workspace_id = $1 AND doc_id = 'live-doc'", + ) + .bind(&workspace_id) + .fetch_one(&pool) + .await?, + "fresh" + ); + let unchanged = runtime + .rebuild_workspace_doc_blob_refs(workspace_id.clone(), 100) + .await + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + assert_eq!(unchanged.parsed_docs, 0); + + sqlx::query( + "INSERT INTO updates (workspace_id, guid, blob, created_at) VALUES ($1, 'live-doc', $2, CURRENT_TIMESTAMP)", + ) + .bind(&workspace_id) + .bind(affine_doc_loader::build_full_doc("Live pending", "", "live-doc")?) + .execute(&pool) + .await?; + let pending = runtime + .rebuild_workspace_doc_blob_refs(workspace_id.clone(), 100) + .await + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + assert_eq!(pending.failed_docs, 0); + assert_eq!( + sqlx::query_scalar::<_, String>( + "SELECT status FROM doc_blob_ref_projections WHERE workspace_id = $1 AND doc_id = 'live-doc'", + ) + .bind(&workspace_id) + .fetch_one(&pool) + .await?, + "pending" + ); + sqlx::query("DELETE FROM updates WHERE workspace_id = $1 AND guid = 'live-doc'") + .bind(&workspace_id) + .execute(&pool) + .await?; + let repaired = runtime + .rebuild_workspace_doc_blob_refs(workspace_id.clone(), 100) + .await + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + assert_eq!(repaired.parsed_docs, 1); assert_eq!( sqlx::query_scalar::<_, i64>( "SELECT COUNT(*) FROM doc_blob_refs WHERE workspace_id = $1 AND doc_id = $2 AND blob_key = 'candidate-blob'", @@ -1023,7 +1077,7 @@ mod tests { .map_err(|err| anyhow::anyhow!(err.to_string()))?; assert_eq!( (partial.failed_docs, partial.next_cursor.as_deref()), - (1, Some("live-doc")) + (0, Some("live-doc")) ); let partial_checkpoint = sqlx::query( "SELECT status, metadata FROM storage_reconciliation_checkpoints WHERE kind = 'doc_blob_refs' AND scope = $1", @@ -1032,11 +1086,11 @@ mod tests { .fetch_one(&pool) .await?; assert_eq!(partial_checkpoint.get::("status"), "running"); - assert_eq!(partial_checkpoint.get::("metadata")["failedDocs"], 1); + assert_eq!(partial_checkpoint.get::("metadata")["failedDocs"], 0); sqlx::query( - "UPDATE storage_reconciliation_checkpoints SET status = 'failed', metadata = '{\"parserVersion\":1}' WHERE kind \ - = 'doc_blob_refs' AND scope = $1", + "UPDATE storage_reconciliation_checkpoints SET status = 'failed', metadata = \ + '{\"parserVersion\":1,\"failedDocs\":99}' WHERE kind = 'doc_blob_refs' AND scope = $1", ) .bind(&workspace_id) .execute(&pool) @@ -1045,15 +1099,18 @@ mod tests { .rebuild_workspace_doc_blob_refs(workspace_id.clone(), 1) .await .map_err(|err| anyhow::anyhow!(err.to_string()))?; - assert_eq!((resumed.failed_docs, resumed.next_cursor), (0, None)); + assert_eq!( + (resumed.failed_docs, resumed.next_cursor.as_deref()), + (0, Some("missing-doc")) + ); let resumed_checkpoint = sqlx::query( "SELECT status, metadata FROM storage_reconciliation_checkpoints WHERE kind = 'doc_blob_refs' AND scope = $1", ) .bind(&workspace_id) .fetch_one(&pool) .await?; - assert_eq!(resumed_checkpoint.get::("status"), "failed"); - assert_eq!(resumed_checkpoint.get::("metadata")["failedDocs"], 1); + assert_eq!(resumed_checkpoint.get::("status"), "running"); + assert_eq!(resumed_checkpoint.get::("metadata")["failedDocs"], 0); sqlx::query("UPDATE snapshots SET blob = $2 WHERE workspace_id = $1 AND guid = 'live-doc'") .bind(&workspace_id) @@ -1086,7 +1143,7 @@ mod tests { .rebuild_workspace_doc_blob_refs(workspace_id.clone(), 100) .await .map_err(|err| anyhow::anyhow!(err.to_string()))?; - assert_eq!((parser_upgrade.scanned_docs, parser_upgrade.failed_docs), (2, 0)); + assert_eq!((parser_upgrade.scanned_docs, parser_upgrade.failed_docs), (3, 0)); assert_eq!( sqlx::query_scalar::<_, String>( "SELECT status FROM storage_reconciliation_checkpoints WHERE kind = 'doc_blob_refs' AND scope = $1", @@ -1159,6 +1216,10 @@ mod tests { .bind(&workspace_id) .execute(&pool) .await?; + sqlx::query("DELETE FROM doc_blob_ref_projections WHERE workspace_id = $1") + .bind(&workspace_id) + .execute(&pool) + .await?; sqlx::query("DELETE FROM document_cleanup_candidates WHERE workspace_id = $1") .bind(&workspace_id) .execute(&pool) @@ -1259,7 +1320,17 @@ mod tests { .bind(doc_id) .fetch_one(&pool) .await?; - assert_eq!(ref_count, 1); + assert_eq!(ref_count, 0); + assert_eq!( + sqlx::query_scalar::<_, String>( + "SELECT status FROM doc_blob_ref_projections WHERE workspace_id = $1 AND doc_id = $2", + ) + .bind(&workspace_id) + .bind(doc_id) + .fetch_one(&pool) + .await?, + "pending" + ); let not_due = execute_one(&pool, Some(&workspace_id), 30).await?; assert!(not_due.is_none()); @@ -1416,6 +1487,7 @@ mod tests { ("doc_access_policies", "doc_id"), ("doc_grants", "doc_id"), ("doc_blob_refs", "doc_id"), + ("doc_blob_ref_projections", "doc_id"), ("ai_workspace_ignored_docs", "doc_id"), ("comments", "doc_id"), ("comment_attachments", "doc_id"), diff --git a/packages/backend/native/src/runtime/storage_runtime/mod.rs b/packages/backend/native/src/runtime/storage_runtime/mod.rs index 4c4595a1c5..9b4685c00d 100644 --- a/packages/backend/native/src/runtime/storage_runtime/mod.rs +++ b/packages/backend/native/src/runtime/storage_runtime/mod.rs @@ -17,7 +17,10 @@ pub use capabilities::StorageProviderCapabilities; use capabilities::storage_provider_capabilities; use config::StorageRuntimeConfig; pub(super) use current_doc::load_current_doc; -use current_doc::{CurrentDoc, CurrentDocUpdate, load_workspace_live_doc_ids, merge_current_doc}; +use current_doc::{ + CurrentDoc, CurrentDocUpdate, load_canonical_doc, load_workspace_canonical_doc_ids, load_workspace_live_doc_ids, + merge_current_doc, +}; use super::object_storage::{ self, ObjectStorageService, StorageBackendConfig, diff --git a/packages/backend/native/src/runtime/types.rs b/packages/backend/native/src/runtime/types.rs index 548b90692d..fff6a6c664 100644 --- a/packages/backend/native/src/runtime/types.rs +++ b/packages/backend/native/src/runtime/types.rs @@ -444,6 +444,7 @@ pub struct RuntimeBlobMetadataBackfillResult { pub workspace_ids: Vec, } +#[derive(Default)] #[napi_derive::napi(object)] pub struct RuntimeDocBlobRefsResult { pub scanned_docs: i64, @@ -552,3 +553,10 @@ pub struct RuntimeEmbeddingProgress { pub total: i64, pub embedded: i64, } + +#[napi_derive::napi(object)] +pub struct SearchOperationOutput { + pub ok: bool, + pub value: Option, + pub error_code: Option, +} diff --git a/packages/backend/native/src/search_index/document.rs b/packages/backend/native/src/search_index/document.rs new file mode 100644 index 0000000000..b901573959 --- /dev/null +++ b/packages/backend/native/src/search_index/document.rs @@ -0,0 +1,63 @@ +use memory_indexer::{Document, FieldType, Value}; +use serde_json::Value as JsonValue; + +use super::{IndexError, Result, schema::TableSchema}; + +pub(super) fn compile_document(table: &TableSchema, value: JsonValue) -> Result { + let mut object = value + .as_object() + .cloned() + .ok_or_else(|| IndexError::InvalidInput("index document must be an object".into()))?; + let explicit_id = object + .remove("_id") + .map(|value| { + value + .as_str() + .map(str::to_owned) + .ok_or_else(|| IndexError::InvalidInput("index document _id must be a string".into())) + }) + .transpose()?; + let id = match explicit_id { + Some(id) => id, + None => table.document_id(&object)?, + }; + let mut document = Document::new(id); + for (name, value) in object { + if value.is_null() { + continue; + } + let field = table.field(&name)?; + let values = match value { + JsonValue::Array(values) => values.into_iter().filter(|value| !value.is_null()).collect(), + value => vec![value], + }; + if values.is_empty() { + continue; + } + document.add_values( + field, + values + .into_iter() + .map(|value| compile_value(table.field_type(field), value)) + .collect::>>()?, + ); + } + Ok(document) +} + +fn compile_value(field_type: &FieldType, value: JsonValue) -> Result { + match field_type { + FieldType::Text(_) | FieldType::Keyword => value + .as_str() + .map(|value| Value::String(value.into())) + .ok_or_else(|| IndexError::InvalidInput("string index value required".into())), + FieldType::I64 => value + .as_i64() + .map(Value::I64) + .ok_or_else(|| IndexError::InvalidInput("integer index value required".into())), + FieldType::Bool => value + .as_bool() + .map(Value::Bool) + .ok_or_else(|| IndexError::InvalidInput("boolean index value required".into())), + } +} diff --git a/packages/backend/native/src/search_index/mod.rs b/packages/backend/native/src/search_index/mod.rs new file mode 100644 index 0000000000..c3892ed8fa --- /dev/null +++ b/packages/backend/native/src/search_index/mod.rs @@ -0,0 +1,371 @@ +mod document; +mod query; +mod result; +mod schema; + +use std::sync::Arc; + +use memory_indexer::{MemoryIndex, Mutation, TermsAggregation}; +use napi::{Status, bindgen_prelude::Buffer}; +use serde_json::Value as JsonValue; +use tokio::sync::RwLock; + +use self::{ + document::compile_document, + query::{compile_options, compile_query}, + result::{HighlightTags, aggregate_result, search_result}, + schema::TableSchema, +}; + +type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +enum IndexError { + #[error("Invalid index input: {0}")] + InvalidInput(String), + #[error(transparent)] + Memory(#[from] memory_indexer::Error), + #[error(transparent)] + Json(#[from] serde_json::Error), +} + +impl From for napi::Error { + fn from(error: IndexError) -> Self { + napi::Error::new(Status::InvalidArg, error.to_string()) + } +} + +struct TableIndex { + schema: TableSchema, + index: RwLock, +} + +impl TableIndex { + fn new(schema: TableSchema) -> Self { + Self { + index: RwLock::new(MemoryIndex::new(schema.schema.clone())), + schema, + } + } +} + +struct IndexManager { + doc: TableIndex, + block: TableIndex, +} + +impl IndexManager { + fn new() -> Self { + Self { + doc: TableIndex::new(TableSchema::doc()), + block: TableIndex::new(TableSchema::block()), + } + } + + fn table(&self, name: &str) -> Result<&TableIndex> { + match name { + "doc" => Ok(&self.doc), + "block" => Ok(&self.block), + _ => Err(IndexError::InvalidInput(format!("unknown index table {name}"))), + } + } +} + +pub(crate) struct EmbeddedIndexCheckpoint { + pub sequence: i64, + pub data: Buffer, +} + +pub(crate) struct EmbeddedSearchIndex { + manager: Arc, +} + +impl EmbeddedSearchIndex { + pub(crate) fn new() -> Self { + Self { + manager: Arc::new(IndexManager::new()), + } + } + + pub(crate) async fn restore(&self, table: String, checkpoint: Buffer) -> napi::Result<()> { + let table = self.manager.table(&table)?; + let index = + MemoryIndex::from_checkpoint(table.schema.schema.clone(), checkpoint.as_ref()).map_err(IndexError::from)?; + *table.index.write().await = index; + Ok(()) + } + + pub(crate) async fn reset(&self, table: String) -> napi::Result<()> { + let table = self.manager.table(&table)?; + *table.index.write().await = MemoryIndex::new(table.schema.schema.clone()); + Ok(()) + } + + pub(crate) async fn write(&self, table: String, documents_json: String) -> napi::Result<()> { + let table = self.manager.table(&table)?; + let documents: Vec = serde_json::from_str(&documents_json)?; + let documents = documents + .into_iter() + .map(|document| compile_document(&table.schema, document)) + .collect::>>()?; + table + .index + .write() + .await + .apply_batch(documents.into_iter().map(Mutation::Upsert).collect()) + .map_err(IndexError::from)?; + Ok(()) + } + + pub(crate) async fn delete(&self, table: String, id: String) -> napi::Result<()> { + self.manager.table(&table)?.index.write().await.delete(&id); + Ok(()) + } + + pub(crate) async fn search(&self, table: String, dsl_json: String) -> napi::Result { + let table = self.manager.table(&table)?; + let dsl: JsonValue = serde_json::from_str(&dsl_json)?; + let query = compile_query( + &table.schema, + dsl + .get("query") + .ok_or_else(|| IndexError::InvalidInput("search query is required".into()))?, + )?; + let result = table + .index + .read() + .await + .search(&query, compile_options(&table.schema, &dsl)?) + .map_err(IndexError::from)?; + Ok(serde_json::to_string(&search_result( + &table.schema, + result, + &highlight_tags(&dsl), + ))?) + } + + pub(crate) async fn aggregate(&self, table: String, dsl_json: String) -> napi::Result { + let table = self.manager.table(&table)?; + let dsl: JsonValue = serde_json::from_str(&dsl_json)?; + let query = compile_query( + &table.schema, + dsl + .get("query") + .ok_or_else(|| IndexError::InvalidInput("aggregate query is required".into()))?, + )?; + let terms = dsl + .pointer("/aggs/result/terms") + .ok_or_else(|| IndexError::InvalidInput("terms aggregation is required".into()))?; + let top_hits = dsl + .pointer("/aggs/result/aggs/result/top_hits") + .map(|options| compile_options(&table.schema, options)) + .transpose()?; + let limit = terms.get("size").and_then(JsonValue::as_u64).unwrap_or(10) as usize; + let result = table + .index + .read() + .await + .aggregate( + &query, + TermsAggregation { + field: table.schema.field( + terms + .get("field") + .and_then(JsonValue::as_str) + .ok_or_else(|| IndexError::InvalidInput("aggregation field is required".into()))?, + )?, + limit: limit.saturating_add(1), + offset: dsl.get("from").and_then(JsonValue::as_u64).unwrap_or(0) as usize, + top_hits, + }, + ) + .map_err(IndexError::from)?; + Ok(serde_json::to_string(&aggregate_result( + &table.schema, + result, + limit, + &highlight_tags(dsl.pointer("/aggs/result/aggs/result/top_hits").unwrap_or(&dsl)), + ))?) + } + + pub(crate) async fn checkpoint(&self, table: String) -> napi::Result { + let checkpoint = self + .manager + .table(&table)? + .index + .read() + .await + .checkpoint() + .map_err(IndexError::from)?; + Ok(EmbeddedIndexCheckpoint { + sequence: checkpoint.sequence as i64, + data: checkpoint.bytes.into(), + }) + } + + pub(crate) async fn optimize(&self, table: String) -> napi::Result<()> { + self.manager.table(&table)?.index.write().await.optimize(); + Ok(()) + } + + pub(crate) async fn mark_checkpoint_persisted(&self, table: String, sequence: i64) -> napi::Result<()> { + self + .manager + .table(&table)? + .index + .write() + .await + .mark_checkpoint_persisted(sequence as u64) + .map_err(IndexError::from)?; + Ok(()) + } +} + +fn highlight_tags(dsl: &JsonValue) -> HighlightTags { + dsl + .pointer("/highlight/fields") + .and_then(JsonValue::as_object) + .into_iter() + .flatten() + .filter_map(|(field, options)| { + Some(( + field.clone(), + ( + options.get("pre_tags")?.as_array()?.first()?.as_str()?.to_string(), + options.get("post_tags")?.as_array()?.first()?.as_str()?.to_string(), + ), + )) + }) + .collect() +} + +impl Default for EmbeddedSearchIndex { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + + use super::EmbeddedSearchIndex; + + fn doc(workspace: &str, id: &str, title: &str, updated_at: i64) -> Value { + json!({ + "workspace_id": workspace, + "doc_id": id, + "title": title, + "summary": title, + "created_by_user_id": "user", + "updated_by_user_id": "user", + "created_at": updated_at, + "updated_at": updated_at + }) + } + + fn search(query: Value, cursor: Option<&str>) -> String { + json!({ + "query": query, + "fields": ["doc_id", "title"], + "_source": ["doc_id"], + "sort": ["_score", { "updated_at": "desc" }, "doc_id"], + "size": 1, + "cursor": cursor + }) + .to_string() + } + + #[tokio::test] + async fn exact_search_cursor_and_checkpoint_roundtrip() { + let index = EmbeddedSearchIndex::new(); + index + .write( + "doc".into(), + json!([ + doc("workspace-1", "one", "设计文档", 1), + doc("workspace-1", "two", "设计方案", 2), + doc("workspace-2", "three", "设计文档", 3) + ]) + .to_string(), + ) + .await + .unwrap(); + + let query = json!({ "term": { "workspace_id": { "value": "workspace-1" } } }); + let first: Value = + serde_json::from_str(&index.search("doc".into(), search(query.clone(), None)).await.unwrap()).unwrap(); + assert_eq!(first["total"], 2); + assert_eq!(first["nodes"][0]["id"], "workspace-1/two"); + assert_eq!(first["nodes"][0]["fields"]["doc_id"], json!(["two"])); + let cursor = first["nextCursor"].as_str().unwrap(); + + let second: Value = serde_json::from_str( + &index + .search("doc".into(), search(query.clone(), Some(cursor))) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(second["nodes"][0]["id"], "workspace-1/one"); + + let checkpoint = index.checkpoint("doc".into()).await.unwrap(); + index.reset("doc".into()).await.unwrap(); + let empty: Value = + serde_json::from_str(&index.search("doc".into(), search(query.clone(), None)).await.unwrap()).unwrap(); + assert_eq!(empty["total"], 0); + index.restore("doc".into(), checkpoint.data).await.unwrap(); + let restored: Value = + serde_json::from_str(&index.search("doc".into(), search(query, None)).await.unwrap()).unwrap(); + assert_eq!(restored["total"], 2); + + let aggregate: Value = serde_json::from_str( + &index + .aggregate( + "doc".into(), + json!({ + "query":{"match_all":{}}, + "from":0, + "aggs":{"result":{"terms":{"field":"workspace_id","size":10},"aggs":{"result":{"top_hits":{ + "size":1,"fields":["doc_id","title"],"sort":["updated_at","doc_id"] + }}}}} + }) + .to_string(), + ) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(aggregate["total"], 2); + assert_eq!(aggregate["buckets"][0]["key"], "workspace-1"); + assert_eq!(aggregate["buckets"][0]["count"], 2); + assert!(aggregate["buckets"][0]["hits"][0]["fields"]["doc_id"].is_array()); + } + + #[tokio::test] + async fn write_is_atomic_and_corrupt_checkpoint_is_rejected() { + let index = EmbeddedSearchIndex::new(); + index + .write( + "doc".into(), + json!([{ "workspace_id": "workspace", "doc_id": "null-values", "summary": [null] }]).to_string(), + ) + .await + .unwrap(); + let all = json!({ "match_all": {} }); + let result: Value = + serde_json::from_str(&index.search("doc".into(), search(all.clone(), None)).await.unwrap()).unwrap(); + assert_eq!(result["total"], 1); + + index.reset("doc".into()).await.unwrap(); + let documents = json!([ + doc("workspace", "valid", "hello", 1), + { "workspace_id": "workspace", "doc_id": "invalid", "unknown": true } + ]); + assert!(index.write("doc".into(), documents.to_string()).await.is_err()); + + let result: Value = serde_json::from_str(&index.search("doc".into(), search(all, None)).await.unwrap()).unwrap(); + assert_eq!(result["total"], 0); + assert!(index.restore("doc".into(), vec![1, 2, 3].into()).await.is_err()); + } +} diff --git a/packages/backend/native/src/search_index/query.rs b/packages/backend/native/src/search_index/query.rs new file mode 100644 index 0000000000..04f4e674de --- /dev/null +++ b/packages/backend/native/src/search_index/query.rs @@ -0,0 +1,215 @@ +use memory_indexer::{FieldType, Query, SearchMode, SearchOptions, Sort, SortOrder, SortValue, Value}; +use serde_json::Value as JsonValue; + +use super::{IndexError, Result, schema::TableSchema}; + +pub(super) fn compile_query(table: &TableSchema, value: &JsonValue) -> Result { + let query = if let Some(node) = value.get("match") { + let (field, options) = first_entry(node, "match")?; + Query::text( + table.field(field)?, + required_string(options, "query")?, + SearchMode::Auto, + ) + } else if let Some(node) = value.get("term") { + let (field, options) = first_entry(node, "term")?; + let field_id = table.field(field)?; + Query::term(field_id, parse_term(table.field_type(field_id), options.get("value"))?) + } else if let Some(node) = value.get("exists") { + Query::Exists(table.field(required_string(node, "field")?)?) + } else if value.get("match_all").is_some() { + Query::All + } else if let Some(node) = value.get("bool") { + Query::boolean( + compile_clauses(table, node.get("must"))?, + compile_clauses(table, node.get("should"))?, + compile_clauses(table, node.get("must_not"))?, + ) + } else { + return Err(IndexError::InvalidInput("unsupported search query".into())); + }; + let boost = query_boost(value); + Ok(if boost == 1.0 { + query + } else { + Query::Boost { + query: Box::new(query), + factor: boost, + } + }) +} + +pub(super) fn compile_options(table: &TableSchema, dsl: &JsonValue) -> Result { + let limit = dsl.get("size").and_then(JsonValue::as_u64).unwrap_or(10) as usize; + let offset = dsl.get("from").and_then(JsonValue::as_u64).unwrap_or(0) as usize; + let mut stored_fields = string_array(dsl.get("fields")) + .into_iter() + .chain(string_array(dsl.get("_source"))) + .map(|field| table.field(field)) + .collect::>>()?; + let mut seen = std::collections::HashSet::new(); + stored_fields.retain(|field| seen.insert(*field)); + let highlight_fields = dsl + .pointer("/highlight/fields") + .and_then(JsonValue::as_object) + .map(|fields| { + fields + .keys() + .map(|field| table.field(field)) + .collect::>>() + }) + .transpose()? + .unwrap_or_default(); + let sort = compile_sort(table, dsl.get("sort"))?; + let after = dsl + .get("cursor") + .and_then(JsonValue::as_str) + .map(|cursor| parse_cursor(cursor, &sort, table)) + .transpose()?; + Ok(SearchOptions { + limit, + offset, + after, + sort, + stored_fields, + highlight_fields, + }) +} + +fn compile_sort(table: &TableSchema, value: Option<&JsonValue>) -> Result> { + let mut sorts = Vec::new(); + for item in value.and_then(JsonValue::as_array).into_iter().flatten() { + if let Some(field) = item.as_str() { + match field { + "_score" => sorts.push(Sort::ScoreDesc), + "id" | "_id" => sorts.push(Sort::DocumentId), + field => sorts.push(Sort::Field { + field: table.field(field)?, + order: SortOrder::Asc, + }), + } + } else if let Some((field, order)) = item.as_object().and_then(|value| value.iter().next()) { + sorts.push(Sort::Field { + field: table.field(field)?, + order: if order.as_str() == Some("desc") { + SortOrder::Desc + } else { + SortOrder::Asc + }, + }); + } + } + Ok(sorts) +} + +fn parse_cursor(cursor: &str, sorts: &[Sort], table: &TableSchema) -> Result> { + let values: Vec = serde_json::from_str(cursor)?; + let mut effective = sorts.to_vec(); + if !effective.iter().any(|sort| matches!(sort, Sort::DocumentId)) { + effective.push(Sort::DocumentId); + } + if values.len() != effective.len() { + return Err(IndexError::InvalidInput("invalid search cursor".into())); + } + values + .into_iter() + .zip(effective) + .map(|(value, sort)| match sort { + _ if value.is_null() => Ok(SortValue::Missing), + Sort::ScoreDesc => value + .as_f64() + .map(|value| SortValue::Score(value as f32)) + .ok_or_else(|| IndexError::InvalidInput("invalid score cursor".into())), + Sort::DocumentId => value + .as_str() + .map(|value| SortValue::String(value.into())) + .ok_or_else(|| IndexError::InvalidInput("invalid document cursor".into())), + Sort::Field { field, .. } => match table.field_type(field) { + FieldType::Keyword => value + .as_str() + .map(|value| SortValue::String(value.into())) + .ok_or_else(|| IndexError::InvalidInput("invalid keyword cursor".into())), + FieldType::I64 => value + .as_i64() + .map(SortValue::I64) + .ok_or_else(|| IndexError::InvalidInput("invalid integer cursor".into())), + FieldType::Bool => value + .as_bool() + .map(SortValue::Bool) + .ok_or_else(|| IndexError::InvalidInput("invalid boolean cursor".into())), + FieldType::Text(_) => Err(IndexError::InvalidInput("text fields are not sortable".into())), + }, + }) + .collect() +} + +fn compile_clauses(table: &TableSchema, value: Option<&JsonValue>) -> Result> { + value + .and_then(JsonValue::as_array) + .into_iter() + .flatten() + .map(|query| compile_query(table, query)) + .collect() +} + +fn query_boost(value: &JsonValue) -> f32 { + for operator in ["match", "term", "exists", "match_all", "bool"] { + let Some(node) = value.get(operator) else { + continue; + }; + if let Some(boost) = node.get("boost").and_then(JsonValue::as_f64) { + return boost as f32; + } + if let Some((_, options)) = node.as_object().and_then(|value| value.iter().next()) + && let Some(boost) = options.get("boost").and_then(JsonValue::as_f64) + { + return boost as f32; + } + } + 1.0 +} + +fn first_entry<'a>(value: &'a JsonValue, operator: &str) -> Result<(&'a str, &'a JsonValue)> { + value + .as_object() + .and_then(|value| value.iter().next()) + .map(|(field, value)| (field.as_str(), value)) + .ok_or_else(|| IndexError::InvalidInput(format!("invalid {operator} query"))) +} + +fn required_string<'a>(value: &'a JsonValue, field: &str) -> Result<&'a str> { + value + .get(field) + .and_then(JsonValue::as_str) + .ok_or_else(|| IndexError::InvalidInput(format!("{field} must be a string"))) +} + +fn parse_term(field_type: &FieldType, value: Option<&JsonValue>) -> Result { + let value = value.ok_or_else(|| IndexError::InvalidInput("term value is required".into()))?; + match field_type { + FieldType::Keyword => value + .as_str() + .map(|value| Value::String(value.into())) + .ok_or_else(|| IndexError::InvalidInput("keyword term must be a string".into())), + FieldType::I64 => value + .as_i64() + .map(Value::I64) + .ok_or_else(|| IndexError::InvalidInput("integer term must be an integer".into())), + FieldType::Bool => value + .as_bool() + .map(Value::Bool) + .ok_or_else(|| IndexError::InvalidInput("boolean term must be a boolean".into())), + FieldType::Text(_) => Err(IndexError::InvalidInput( + "term query does not accept text fields".into(), + )), + } +} + +fn string_array(value: Option<&JsonValue>) -> Vec<&str> { + value + .and_then(JsonValue::as_array) + .into_iter() + .flatten() + .filter_map(JsonValue::as_str) + .collect() +} diff --git a/packages/backend/native/src/search_index/result.rs b/packages/backend/native/src/search_index/result.rs new file mode 100644 index 0000000000..4cd59475cd --- /dev/null +++ b/packages/backend/native/src/search_index/result.rs @@ -0,0 +1,173 @@ +use std::collections::HashMap; + +use memory_indexer::{AggregationResult, SearchHit, SearchResult, SortValue, Value}; +use serde::Serialize; + +use super::schema::TableSchema; + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct NativeSearchResult { + pub total: usize, + pub nodes: Vec, + pub next_cursor: Option, +} + +#[derive(Serialize)] +pub(super) struct NativeAggregateResult { + pub total: usize, + #[serde(rename = "hasMore")] + pub has_more: bool, + pub buckets: Vec, +} + +#[derive(Serialize)] +pub(super) struct NativeBucket { + pub key: serde_json::Value, + pub count: u64, + pub hits: Vec, +} + +#[derive(Serialize)] +pub(super) struct NativeHit { + pub id: String, + pub score: f32, + pub fields: serde_json::Map, + pub highlights: serde_json::Map, +} + +pub(super) type HighlightTags = HashMap; + +pub(super) fn search_result( + table: &TableSchema, + result: SearchResult, + highlight_tags: &HighlightTags, +) -> NativeSearchResult { + let next_cursor = result.hits.last().map(|hit| cursor(&hit.sort_values)); + NativeSearchResult { + total: result.total, + nodes: result + .hits + .into_iter() + .map(|hit| native_hit(table, hit, highlight_tags)) + .collect(), + next_cursor, + } +} + +pub(super) fn aggregate_result( + table: &TableSchema, + mut result: AggregationResult, + limit: usize, + highlight_tags: &HighlightTags, +) -> NativeAggregateResult { + let total = result.buckets.len(); + let has_more = result.buckets.len() > limit; + result.buckets.truncate(limit); + NativeAggregateResult { + total, + has_more, + buckets: result + .buckets + .into_iter() + .map(|bucket| NativeBucket { + key: json_value(bucket.key), + count: bucket.count, + hits: bucket + .hits + .into_iter() + .map(|hit| native_hit(table, hit, highlight_tags)) + .collect(), + }) + .collect(), + } +} + +fn native_hit(table: &TableSchema, hit: SearchHit, highlight_tags: &HighlightTags) -> NativeHit { + let mut fields = serde_json::Map::new(); + for (field, values) in hit.fields { + let name = table.field_name(field).to_string(); + let values = values.into_iter().map(json_value).collect::>(); + fields.insert(name, serde_json::Value::Array(values)); + } + let mut highlights: serde_json::Map = serde_json::Map::new(); + for highlight in hit.highlights { + let name = table.field_name(highlight.field).to_string(); + let Some((before, after)) = highlight_tags.get(&name) else { + continue; + }; + let Some(text) = fields + .get(&name) + .and_then(serde_json::Value::as_array) + .and_then(|values| values.get(highlight.value_index as usize)) + .and_then(serde_json::Value::as_str) + else { + continue; + }; + let value = render_highlight(text, &highlight.spans, before, after); + highlights + .entry(name) + .or_insert_with(|| serde_json::Value::Array(Vec::new())) + .as_array_mut() + .expect("highlight value is an array") + .push(serde_json::Value::String(value)); + } + NativeHit { + id: hit.id, + score: hit.score, + fields, + highlights, + } +} + +fn render_highlight(text: &str, spans: &[(u32, u32)], before: &str, after: &str) -> String { + let mut output = String::new(); + let mut cursor = 0; + for &(start, end) in spans { + let start = utf16_to_byte(text, start as usize); + let end = utf16_to_byte(text, end as usize); + if start < cursor || end < start || end > text.len() { + continue; + } + output.push_str(&text[cursor..start]); + output.push_str(before); + output.push_str(&text[start..end]); + output.push_str(after); + cursor = end; + } + output.push_str(&text[cursor..]); + output +} + +fn utf16_to_byte(text: &str, offset: usize) -> usize { + let mut units = 0; + for (byte, character) in text.char_indices() { + if units >= offset { + return byte; + } + units += character.len_utf16(); + } + text.len() +} + +fn cursor(values: &[SortValue]) -> String { + serde_json::to_string(&values.iter().map(sort_value).collect::>()).expect("cursor values serialize") +} + +fn sort_value(value: &SortValue) -> serde_json::Value { + match value { + SortValue::Score(value) => serde_json::json!(value), + SortValue::String(value) => serde_json::json!(value), + SortValue::I64(value) => serde_json::json!(value), + SortValue::Bool(value) => serde_json::json!(value), + SortValue::Missing => serde_json::Value::Null, + } +} + +fn json_value(value: Value) -> serde_json::Value { + match value { + Value::String(value) => serde_json::Value::String(value), + Value::I64(value) => serde_json::json!(value), + Value::Bool(value) => serde_json::json!(value), + } +} diff --git a/packages/backend/native/src/search_index/schema.rs b/packages/backend/native/src/search_index/schema.rs new file mode 100644 index 0000000000..0c1aaf31c9 --- /dev/null +++ b/packages/backend/native/src/search_index/schema.rs @@ -0,0 +1,175 @@ +use std::collections::HashMap; + +use memory_indexer::{FieldId, FieldOptions, FieldType, PositionEncoding, Schema, TextOptions}; + +use super::{IndexError, Result}; + +pub(super) struct TableSchema { + pub schema: Schema, + fields: HashMap, + id_fields: &'static [&'static str], +} + +impl TableSchema { + pub fn doc() -> Self { + let mut builder = Schema::builder().position_encoding(PositionEncoding::Utf16); + let mut fields = HashMap::new(); + keyword(&mut builder, &mut fields, "workspace_id", true, false); + keyword(&mut builder, &mut fields, "workspace_token", true, false); + keyword(&mut builder, &mut fields, "doc_id", true, true); + keyword(&mut builder, &mut fields, "doc_token", true, false); + fields.insert( + "title".into(), + builder.text("title", text_options(), FieldOptions::indexed_stored()), + ); + keyword(&mut builder, &mut fields, "summary", false, false); + keyword(&mut builder, &mut fields, "journal", false, false); + keyword(&mut builder, &mut fields, "created_by_user_id", true, false); + keyword(&mut builder, &mut fields, "updated_by_user_id", true, false); + keyword(&mut builder, &mut fields, "acl_read_tokens", true, false); + boolean(&mut builder, &mut fields, "acl_public_readable"); + boolean(&mut builder, &mut fields, "acl_member_default_readable"); + integer(&mut builder, &mut fields, "acl_revision", true, false); + integer(&mut builder, &mut fields, "created_at", true, true); + integer(&mut builder, &mut fields, "updated_at", true, true); + Self::finish(builder, fields, &["workspace_id", "doc_id"]) + } + + pub fn block() -> Self { + let mut builder = Schema::builder().position_encoding(PositionEncoding::Utf16); + let mut fields = HashMap::new(); + for field in [ + "workspace_id", + "unit_id", + "source_hash", + "visibility", + "element_id", + "frame_id", + "source_block_id", + "flavour", + "blob", + "ref_doc_id", + "parent_flavour", + "parent_block_id", + "created_by_user_id", + "updated_by_user_id", + ] { + keyword(&mut builder, &mut fields, field, true, false); + } + keyword(&mut builder, &mut fields, "doc_id", true, true); + keyword(&mut builder, &mut fields, "workspace_token", true, false); + keyword(&mut builder, &mut fields, "doc_token", true, false); + keyword(&mut builder, &mut fields, "block_id", true, true); + keyword(&mut builder, &mut fields, "block_token", true, false); + fields.insert( + "content".into(), + builder.text("content", text_options(), FieldOptions::indexed_stored().multi_value()), + ); + for field in ["ref", "additional", "markdown_preview"] { + keyword(&mut builder, &mut fields, field, false, false); + } + integer(&mut builder, &mut fields, "projection_version", true, false); + integer(&mut builder, &mut fields, "created_at", true, true); + integer(&mut builder, &mut fields, "updated_at", true, true); + keyword(&mut builder, &mut fields, "acl_read_tokens", true, false); + boolean(&mut builder, &mut fields, "acl_public_readable"); + boolean(&mut builder, &mut fields, "acl_member_default_readable"); + integer(&mut builder, &mut fields, "acl_revision", true, false); + Self::finish(builder, fields, &["workspace_id", "doc_id", "block_id"]) + } + + fn finish( + builder: memory_indexer::SchemaBuilder, + fields: HashMap, + id_fields: &'static [&'static str], + ) -> Self { + Self { + schema: builder.build().expect("static server index schema must be valid"), + fields, + id_fields, + } + } + + pub fn document_id(&self, document: &serde_json::Map) -> Result { + self + .id_fields + .iter() + .map(|field| { + document + .get(*field) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| IndexError::InvalidInput(format!("index document {field} is required"))) + }) + .collect::>>() + .map(|parts| parts.join("/")) + } + + pub fn field(&self, name: &str) -> Result { + self + .fields + .get(name) + .copied() + .ok_or_else(|| IndexError::InvalidInput(format!("unknown index field {name}"))) + } + + pub fn field_name(&self, field: FieldId) -> &str { + &self.schema.field(field).expect("field belongs to table schema").name + } + + pub fn field_type(&self, field: FieldId) -> &FieldType { + &self + .schema + .field(field) + .expect("field belongs to table schema") + .field_type + } +} + +fn keyword( + builder: &mut memory_indexer::SchemaBuilder, + fields: &mut HashMap, + name: &str, + indexed: bool, + sortable: bool, +) { + let mut options = FieldOptions::new().stored(); + if !sortable { + options = options.multi_value(); + } + if indexed { + options = options.indexed(); + } + if sortable { + options = options.sortable(); + } + fields.insert(name.into(), builder.keyword(name, options)); +} + +fn integer( + builder: &mut memory_indexer::SchemaBuilder, + fields: &mut HashMap, + name: &str, + indexed: bool, + sortable: bool, +) { + let mut options = FieldOptions::new().stored(); + if indexed { + options = options.indexed(); + } + if sortable { + options = options.sortable(); + } + fields.insert(name.into(), builder.i64(name, options)); +} + +fn boolean(builder: &mut memory_indexer::SchemaBuilder, fields: &mut HashMap, name: &str) { + fields.insert(name.into(), builder.bool(name, FieldOptions::indexed_stored())); +} + +fn text_options() -> TextOptions { + TextOptions::multilingual() + .with_pinyin() + .with_prefix() + .with_fuzzy() + .with_positions() +} diff --git a/packages/backend/server/migrations/20260820120000_cleanup_legacy_copilot_runtime/migration.sql b/packages/backend/server/migrations/20260820120000_cleanup_legacy_copilot_runtime/migration.sql new file mode 100644 index 0000000000..febc82dba7 --- /dev/null +++ b/packages/backend/server/migrations/20260820120000_cleanup_legacy_copilot_runtime/migration.sql @@ -0,0 +1,46 @@ +-- This migration is intentionally fail-closed. The data migration with the +-- same release must have admitted every live legacy context blob through the +-- artifact runtime before these product-owned tables are removed. +DO $$ +BEGIN + IF to_regclass('public.ai_contexts') IS NOT NULL AND EXISTS ( + SELECT 1 + FROM ai_contexts context + JOIN ai_sessions_metadata session ON session.id = context.session_id + JOIN blobs blob + ON blob.workspace_id = session.workspace_id + AND blob.deleted_at IS NULL + AND blob.status = 'completed' + WHERE jsonb_path_exists( + context.config::jsonb, + '$.** ? (@ == $blobKey)', + jsonb_build_object('blobKey', to_jsonb(blob.key::text)) + ) + AND NOT EXISTS ( + SELECT 1 + FROM workspace_artifacts artifact + WHERE artifact.workspace_id = session.workspace_id + AND artifact.status = 'ready' + AND artifact.storage_scope = 'blob' + AND artifact.storage_key = concat(session.workspace_id, '/', blob.key) + ) + ) THEN + RAISE EXCEPTION + 'legacy context blob artifact admission is incomplete; run the data migration before cleanup'; + END IF; +END $$; + +DELETE FROM app_configs WHERE id = 'copilot.providers.defaults'; + +ALTER TABLE ai_workspace_byok_configs + ALTER COLUMN definition DROP DEFAULT, + DROP COLUMN IF EXISTS endpoint, + DROP COLUMN IF EXISTS disabled_reason, + DROP COLUMN IF EXISTS last_validated_at, + DROP COLUMN IF EXISTS last_validation_error; + +DELETE FROM ai_workspace_byok_configs WHERE definition = '{}'::jsonb; + +DROP TABLE IF EXISTS ai_context_embeddings; +DROP TABLE IF EXISTS ai_workspace_embeddings; +DROP TABLE IF EXISTS ai_contexts; diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index 127183d0d2..6ceb10e6f8 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -19,7 +19,7 @@ "seed": "r ./src/seed/index.ts", "genconfig": "r ./scripts/genconfig.ts", "cli": "cross-env SERVER_FLAVOR=script node ./dist/main.js", - "predeploy": "yarn prisma migrate deploy && yarn cli run", + "predeploy": "yarn cli admit-legacy-context-blobs && yarn prisma migrate deploy && yarn cli run", "postinstall": "prisma generate" }, "dependencies": { diff --git a/packages/backend/server/src/__tests__/app/selfhost.e2e.ts b/packages/backend/server/src/__tests__/app/selfhost.e2e.ts index bf7a74bd0b..aef0a20f98 100644 --- a/packages/backend/server/src/__tests__/app/selfhost.e2e.ts +++ b/packages/backend/server/src/__tests__/app/selfhost.e2e.ts @@ -16,6 +16,7 @@ const test = ava as TestFn<{ app: TestingApp; db: PrismaClient; }>; +let originalDeploymentType: typeof env.DEPLOYMENT_TYPE; const mobileUAString = 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36'; @@ -47,6 +48,7 @@ export class TestResolver { } test.before('init selfhost server', async t => { + originalDeploymentType = globalThis.env.DEPLOYMENT_TYPE; // @ts-expect-error override globalThis.env.DEPLOYMENT_TYPE = 'selfhosted'; const app = await createTestingApp({ @@ -69,7 +71,12 @@ test.beforeEach(async t => { }); test.after.always(async t => { - await t.context.app.close(); + try { + await t.context.app.close(); + } finally { + // @ts-expect-error restore mutable test env singleton + globalThis.env.DEPLOYMENT_TYPE = originalDeploymentType; + } }); test('do not allow visit index.html directly', async t => { diff --git a/packages/backend/server/src/__tests__/copilot/runtime-boundaries.spec.ts b/packages/backend/server/src/__tests__/copilot/runtime-boundaries.spec.ts index 27fc2c1773..18142a8a93 100644 --- a/packages/backend/server/src/__tests__/copilot/runtime-boundaries.spec.ts +++ b/packages/backend/server/src/__tests__/copilot/runtime-boundaries.spec.ts @@ -10,6 +10,7 @@ import { type Config, type EventBus, type JobQueue, + SearchProviderNotFound, } from '../../base'; import { ServerFeature, type ServerService } from '../../core'; import type { DocReader } from '../../core/doc'; @@ -417,7 +418,6 @@ test('document tools enforce the user-selected hard scope', async t => { ) => candidates, }; const hybrid = new DocumentRetrievalService( - { indexer: { enabled: true } } as Config, readableAc, lexicalIndexer, vectorSearch, @@ -432,7 +432,6 @@ test('document tools enforce the user-selected hard scope', async t => { t.true(hybridResult.hits[0].score > 1 / 61); const lexicalOnly = new DocumentRetrievalService( - { indexer: { enabled: true } } as Config, readableAc, lexicalIndexer, { ...vectorSearch, canEmbedding: false }, @@ -448,9 +447,12 @@ test('document tools enforce the user-selected hard scope', async t => { t.is(lexicalResult.degradedReason, 'VECTOR_UNAVAILABLE'); const vectorOnly = new DocumentRetrievalService( - { indexer: { enabled: false } } as Config, readableAc, - lexicalIndexer, + { + searchDocsByKeyword: async () => { + throw new SearchProviderNotFound(); + }, + } as unknown as IndexerService, vectorSearch, documentModels ); diff --git a/packages/backend/server/src/__tests__/create-module.ts b/packages/backend/server/src/__tests__/create-module.ts index d17881cbea..39454de4c9 100644 --- a/packages/backend/server/src/__tests__/create-module.ts +++ b/packages/backend/server/src/__tests__/create-module.ts @@ -7,7 +7,18 @@ import { import { PrismaClient } from '@prisma/client'; import { FunctionalityModules } from '../app.module'; -import { AFFiNELogger, EventBus, JobModule, JobQueue } from '../base'; +import { + AFFiNELogger, + ConfigFactory, + EventBus, + JobModule, + JobQueue, +} from '../base'; +import { + BACKEND_RUNTIME_CONFIG_PATHS, + BackendRuntimeProvider, +} from '../core/backend-runtime'; +import { StorageRuntimeProvider } from '../core/storage-runtime'; import { createFactory, MockEventBus, @@ -15,6 +26,7 @@ import { MockJobQueue, } from './mocks'; import { TEST_LOG_LEVEL } from './utils'; +import { createTestRuntimeConfig } from './utils/runtime-config'; interface TestingModuleMetadata extends ModuleMetadata { tapModule?(m: TestingModuleBuilder): void; @@ -30,6 +42,11 @@ export interface TestingModule extends NestjsTestingModule { export async function createModule( metadata: TestingModuleMetadata = {} ): Promise { + const config = new ConfigFactory().config; + const runtimeConfig = await createTestRuntimeConfig( + config.db.datasourceUrl, + config.indexer + ); const { tapModule, ...meta } = metadata; const functionalityModules = [ ...FunctionalityModules.filter(module => { @@ -48,14 +65,22 @@ export async function createModule( .overrideProvider(JobQueue) .useValue(new MockJobQueue()) .overrideProvider(EventBus) - .useValue(new MockEventBus()); + .useValue(new MockEventBus()) + .overrideProvider(BACKEND_RUNTIME_CONFIG_PATHS) + .useValue([runtimeConfig.configPath]); // when custom override happens if (tapModule) { tapModule(builder); } - const module = (await builder.compile()) as TestingModule; + let module: TestingModule; + try { + module = (await builder.compile()) as TestingModule; + } catch (error) { + await runtimeConfig.cleanup(); + throw error; + } const logger = new AFFiNELogger(); // we got a lot smoking tests try to break nestjs @@ -63,7 +88,33 @@ export async function createModule( logger.setLogLevels([TEST_LOG_LEVEL]); module.useLogger(logger); - await module.init(); + const close = module.close.bind(module); + let closePromise: Promise | undefined; + module.close = () => { + return (closePromise ??= (async () => { + try { + await close(); + } finally { + await runtimeConfig.cleanup(); + } + })()); + }; + + try { + await module.init(); + } catch (error) { + await module.close(); + throw error; + } + const backendRuntime = module.get(BackendRuntimeProvider); + if (backendRuntime instanceof BackendRuntimeProvider) { + await backendRuntime.runMigrations(); + await backendRuntime.onConfigChanged({ updates: { indexer: {} } }); + } + const storageRuntime = module.get(StorageRuntimeProvider); + if (storageRuntime instanceof StorageRuntimeProvider) { + await storageRuntime.runMigrations(); + } module[Symbol.asyncDispose] = async () => { await module.close(); }; diff --git a/packages/backend/server/src/__tests__/doc/cron.spec.ts b/packages/backend/server/src/__tests__/doc/cron.spec.ts index ea3f5e8cd5..83050c7c55 100644 --- a/packages/backend/server/src/__tests__/doc/cron.spec.ts +++ b/packages/backend/server/src/__tests__/doc/cron.spec.ts @@ -4,7 +4,7 @@ import ava, { TestFn } from 'ava'; import Sinon from 'sinon'; import { BackendRuntimeProvider } from '../../core/backend-runtime'; -import { DocStorageModule } from '../../core/doc'; +import { DocStorageModule, DocStorageWorkerModule } from '../../core/doc'; import { DocStorageCronJob } from '../../core/doc/job'; import { createTestingModule, type TestingModule } from '../utils'; @@ -23,7 +23,11 @@ test.before(async t => { cleanupExpiredSnapshotHistories: Sinon.stub(), }; t.context.module = await createTestingModule({ - imports: [ScheduleModule.forRoot(), DocStorageModule], + imports: [ + ScheduleModule.forRoot(), + DocStorageModule, + DocStorageWorkerModule, + ], tapModule: builder => { builder .overrideProvider(BackendRuntimeProvider) diff --git a/packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts b/packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts index 8603eb45cd..cf1394c898 100644 --- a/packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts +++ b/packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts @@ -1,66 +1,93 @@ import { getCurrentUserQuery } from '@affine/graphql'; import { JobExecutor } from '../../../base/job/queue/executor'; +import { JobHandlerScanner } from '../../../base/job/queue/scanner'; import { DatabaseDocReader, DocReader } from '../../../core/doc'; import { createApp } from '../create-app'; import { e2e } from '../test'; -type TestFlavor = 'doc' | 'graphql' | 'sync' | 'renderer' | 'front'; +type TestFlavor = + | 'allinone' + | 'worker' + | 'graphql' + | 'sync' + | 'renderer' + | 'front'; -const createFlavorApp = async (flavor: TestFlavor) => { +const withFlavor = async ( + flavor: TestFlavor, + run: (app: Awaited>) => Promise +) => { + const mutableEnv = globalThis.env as unknown as { FLAVOR: string }; + const previousFlavor = mutableEnv.FLAVOR; // @ts-expect-error override globalThis.env.FLAVOR = flavor; - return await createApp({ - tapModule(module) { - module.overrideProvider(JobExecutor).useValue({ - onConfigInit: async () => {}, - onConfigChanged: async () => {}, - onModuleDestroy: async () => {}, - }); - }, - }); + try { + await using app = await createApp({ + tapModule(module) { + module.overrideProvider(JobExecutor).useValue({ + onConfigInit: async () => {}, + onConfigChanged: async () => {}, + onModuleDestroy: async () => {}, + }); + }, + }); + return await run(app); + } finally { + mutableEnv.FLAVOR = previousFlavor; + } }; -e2e('should init doc service', async t => { - await using app = await createFlavorApp('doc'); +e2e('should init worker service', async t => { + await withFlavor('worker', async app => { + const res = await app.GET('/info').expect(200); + t.is(res.body.flavor, 'worker'); + t.truthy(app.get(JobHandlerScanner).getHandler('indexer.indexDoc')); - const res = await app.GET('/info').expect(200); - t.is(res.body.flavor, 'doc'); + await t.throwsAsync(app.gql({ query: getCurrentUserQuery })); + await app.PUT('/api/storage/upload').expect(404); + }); +}); - await t.throwsAsync(app.gql({ query: getCurrentUserQuery })); +e2e('should init allinone service with worker handlers', async t => { + await withFlavor('allinone', async app => { + const res = await app.GET('/info').expect(200); + t.is(res.body.flavor, 'allinone'); + t.truthy(app.get(JobHandlerScanner).getHandler('indexer.indexDoc')); + }); }); e2e('should init graphql service', async t => { - await using app = await createFlavorApp('graphql'); + await withFlavor('graphql', async app => { + const res = await app.GET('/info').expect(200); - const res = await app.GET('/info').expect(200); + t.is(res.body.flavor, 'graphql'); - t.is(res.body.flavor, 'graphql'); - - const user = await app.gql({ query: getCurrentUserQuery }); - t.is(user.currentUser, null); + const user = await app.gql({ query: getCurrentUserQuery }); + t.is(user.currentUser, null); + }); }); e2e('should init sync service', async t => { - await using app = await createFlavorApp('sync'); - - const res = await app.GET('/info').expect(200); - t.is(res.body.flavor, 'sync'); + await withFlavor('sync', async app => { + const res = await app.GET('/info').expect(200); + t.is(res.body.flavor, 'sync'); + }); }); e2e('should init renderer service', async t => { - await using app = await createFlavorApp('renderer'); - - const res = await app.GET('/info').expect(200); - t.is(res.body.flavor, 'renderer'); + await withFlavor('renderer', async app => { + const res = await app.GET('/info').expect(200); + t.is(res.body.flavor, 'renderer'); + }); }); e2e('should init front service', async t => { - await using app = await createFlavorApp('front'); + await withFlavor('front', async app => { + const res = await app.GET('/info').expect(200); + t.is(res.body.flavor, 'front'); - const res = await app.GET('/info').expect(200); - t.is(res.body.flavor, 'front'); - - const docReader = app.get(DocReader); - t.true(docReader instanceof DatabaseDocReader); + const docReader = app.get(DocReader); + t.true(docReader instanceof DatabaseDocReader); + }); }); diff --git a/packages/backend/server/src/__tests__/e2e/create-app.ts b/packages/backend/server/src/__tests__/e2e/create-app.ts index cb74a781b6..e9fc4908ff 100644 --- a/packages/backend/server/src/__tests__/e2e/create-app.ts +++ b/packages/backend/server/src/__tests__/e2e/create-app.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { gqlFetcherFactory } from '@affine/graphql'; -import { INestApplication, ModuleMetadata } from '@nestjs/common'; +import { INestApplication, ModuleMetadata, Type } from '@nestjs/common'; import { NestApplication } from '@nestjs/core'; import { Test, @@ -26,9 +26,15 @@ import { import { ThrottlerStorage } from '../../base/throttler'; import { SocketIoAdapter } from '../../base/websocket'; import { AuthGuard, AuthService } from '../../core/auth'; -import { BACKEND_RUNTIME_CONFIG_PATHS } from '../../core/backend-runtime'; +import { + BACKEND_RUNTIME_CONFIG_PATHS, + BackendRuntimeProvider, +} from '../../core/backend-runtime'; import { Mailer } from '../../core/mail'; +import { StorageRuntimeProvider } from '../../core/storage-runtime'; +import { ServerRole } from '../../env'; import { Models } from '../../models'; +import { IndexerService } from '../../plugins/indexer/service'; import { createFactory, MockedUser, @@ -52,8 +58,16 @@ export class TestingApp extends NestApplication { private csrfCookie: string | null = null; private readonly userCookies: Set = new Set(); + private getOptional(token: Type) { + try { + return this.get(token, { strict: false }); + } catch { + return undefined; + } + } + create = createFactory(this.get(PrismaClient, { strict: false })); - mails = this.get(Mailer, { strict: false }) as MockMailer; + mails = this.getOptional(Mailer) as unknown as MockMailer; queue = this.get(JobQueue, { strict: false }) as MockJobQueue; eventBus = this.get(EventBus, { strict: false }); models = this.get(Models, { strict: false }); @@ -241,8 +255,10 @@ export class TestingApp extends NestApplication { export async function createApp( metadata: TestingAppMetadata = {} ): Promise { + const config = new ConfigFactory().config; const runtimeConfig = await createTestRuntimeConfig( - new ConfigFactory().config.db.datasourceUrl + config.db.datasourceUrl, + config.indexer ); const { buildAppModule } = await import('../../app.module'); const { tapModule, tapApp } = metadata; @@ -326,7 +342,11 @@ export async function createApp( }) ); - app.useGlobalGuards(app.get(AuthGuard), app.get(CloudThrottlerGuard)); + if (globalThis.env.role === ServerRole.Worker) { + app.useGlobalGuards(app.get(CloudThrottlerGuard)); + } else { + app.useGlobalGuards(app.get(AuthGuard), app.get(CloudThrottlerGuard)); + } app.useGlobalInterceptors(app.get(CacheInterceptor)); app.useGlobalFilters(new GlobalExceptionFilter(app.getHttpAdapter())); @@ -340,6 +360,9 @@ export async function createApp( try { await app.init(); + await app.get(BackendRuntimeProvider, { strict: false }).runMigrations(); + await app.get(StorageRuntimeProvider, { strict: false }).runMigrations(); + await app.get(IndexerService, { strict: false }).onApplicationBootstrap(); } catch (error) { await app.close(); throw error; diff --git a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md b/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md deleted file mode 100644 index e5b8f7fee8..0000000000 --- a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md +++ /dev/null @@ -1,98 +0,0 @@ -# Snapshot report for `src/__tests__/e2e/doc-service/controller.spec.ts` - -The actual snapshot is saved in `controller.spec.ts.snap`. - -Generated by [AVA](https://avajs.dev). - -## should get doc markdown success - -> Snapshot 1 - - { - knownUnsupportedBlocks: [ - 'RX4CG2zsBk:affine:note', - 'S1mkc8zUoU:affine:note', - 'yGlBdshAqN:affine:note', - '6lDiuDqZGL:affine:note', - 'cauvaHOQmh:affine:note', - '2jwCeO8Yot:affine:note', - 'c9MF_JiRgx:affine:note', - '6x7ALjUDjj:affine:surface', - ], - markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊ - ␊ - ␊ - ␊ - # You own your data, with no compromises␊ - ␊ - ## Local-first & Real-time collaborative␊ - ␊ - We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊ - ␊ - AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊ - ␊ - ␊ - ␊ - ### Blocks that assemble your next docs, tasks kanban or whiteboard␊ - ␊ - There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊ - ␊ - We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊ - ␊ - If you want to learn more about the product design of AFFiNE, here goes the concepts:␊ - ␊ - To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊ - ␊ - ## A true canvas for blocks in any form␊ - ␊ - [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊ - ␊ - ␊ - ␊ - "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊ - ␊ - * Quip & Notion with their great concept of "everything is a block"␊ - * Trello with their Kanban␊ - * Airtable & Miro with their no-code programable datasheets␊ - * Miro & Whimiscal with their edgeless visual whiteboard␊ - * Remnote & Capacities with their object-based tag system␊ - For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊ - ␊ - ## Self Host␊ - ␊ - Self host AFFiNE␊ - ␊ - ␊ - ### Learning From␊ - ||Title|Tag|␊ - |---|---|---|␊ - |Affine Development|Affine Development|AFFiNE|␊ - |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc|Developers|␊ - |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"|Reference|␊ - |Trello with their Kanban|Trello with their Kanban|Reference|␊ - |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets|Reference|␊ - |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|Reference|␊ - |Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊ - ␊ - ## Affine Development␊ - ␊ - For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊ - ␊ - ␊ - ␊ - `, - title: 'Write, Draw, Plan all at Once.', - unknownBlocks: [], - } - -## should get doc markdown return null when doc not exists - -> Snapshot 1 - - { - code: 'Not Found', - message: 'Doc not found', - name: 'NOT_FOUND', - status: 404, - type: 'RESOURCE_NOT_FOUND', - } diff --git a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap b/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap deleted file mode 100644 index 702fef10e3..0000000000 Binary files a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap and /dev/null differ diff --git a/packages/backend/server/src/__tests__/e2e/doc-service/controller.spec.ts b/packages/backend/server/src/__tests__/e2e/doc-service/controller.spec.ts deleted file mode 100644 index 4956dcc866..0000000000 --- a/packages/backend/server/src/__tests__/e2e/doc-service/controller.spec.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { randomUUID } from 'node:crypto'; - -import { CryptoHelper } from '../../../base'; -import { app, e2e, Mockers } from '../test'; - -const crypto = app.get(CryptoHelper); - -e2e('should get doc markdown success', async t => { - const owner = await app.signup(); - const workspace = await app.create(Mockers.Workspace, { - owner, - }); - - const docSnapshot = await app.create(Mockers.DocSnapshot, { - workspaceId: workspace.id, - user: owner, - }); - - const path = `/rpc/workspaces/${workspace.id}/docs/${docSnapshot.id}/markdown`; - const res = await app - .GET(path) - .set( - 'x-access-token', - crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect(200) - .expect('Content-Type', 'application/json; charset=utf-8'); - - const { revision, ...body } = res.body; - t.regex(revision, /^\d+$/); - t.snapshot(body); -}); - -e2e('should get doc markdown return null when doc not exists', async t => { - const owner = await app.signup(); - const workspace = await app.create(Mockers.Workspace, { - owner, - }); - - const docId = randomUUID(); - const path = `/rpc/workspaces/${workspace.id}/docs/${docId}/markdown`; - const res = await app - .GET(path) - .set( - 'x-access-token', - crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect(404) - .expect('Content-Type', 'application/json; charset=utf-8'); - - t.snapshot(res.body); -}); diff --git a/packages/backend/server/src/__tests__/e2e/indexer/aggregate.spec.ts b/packages/backend/server/src/__tests__/e2e/indexer/aggregate.spec.ts index a993fa3473..e0791cca99 100644 --- a/packages/backend/server/src/__tests__/e2e/indexer/aggregate.spec.ts +++ b/packages/backend/server/src/__tests__/e2e/indexer/aggregate.spec.ts @@ -1,85 +1,30 @@ -import { indexerAggregateQuery, SearchTable } from '@affine/graphql'; +import { + indexerAggregateQuery, + SearchQueryType, + SearchTable, +} from '@affine/graphql'; +import { createDocWithMarkdown } from '../../../native'; import { IndexerService } from '../../../plugins/indexer/service'; import { Mockers } from '../../mocks'; import { app, e2e } from '../test'; e2e('should aggregate by docId', async t => { const owner = await app.signup(); - - const workspace = await app.create(Mockers.Workspace, { - owner: { id: owner.id }, - }); - - const indexerService = app.get(IndexerService); - - await indexerService.write( - SearchTable.block, - [ - { - docId: 'doc-0', - workspaceId: workspace.id, - content: 'test1 hello world top2', - flavour: 'affine:text', - blockId: 'block-0', - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - docId: 'doc-0', - workspaceId: workspace.id, - content: 'test2 hello hello top3', - flavour: 'affine:text', - blockId: 'block-1', - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - docId: 'doc-0', - workspaceId: workspace.id, - content: 'test3 hello title top1', - flavour: 'affine:page', - blockId: 'block-2', - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - docId: 'doc-1', - workspaceId: workspace.id, - content: 'test4 hello world', - flavour: 'affine:text', - blockId: 'block-3', - refDocId: 'doc-0', - ref: ['{"foo": "bar1"}'], - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - docId: 'doc-2', - workspaceId: workspace.id, - content: 'test5 hello', - flavour: 'affine:text', - blockId: 'block-4', - refDocId: 'doc-0', - ref: ['{"foo": "bar2"}'], - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); + const workspace = await app.create(Mockers.Workspace, { owner }); + for (const [docId, markdown] of [ + ['doc-0', 'hello world\n\nhello again'], + ['doc-1', 'hello world'], + ] as const) { + await app.create(Mockers.DocMeta, { workspaceId: workspace.id, docId }); + await app.create(Mockers.DocSnapshot, { + workspaceId: workspace.id, + docId, + user: owner, + blob: createDocWithMarkdown(docId, markdown, docId), + }); + await app.get(IndexerService).indexDoc(workspace.id, docId); + } const result = await app.gql({ query: indexerAggregateQuery, @@ -88,72 +33,25 @@ e2e('should aggregate by docId', async t => { input: { table: SearchTable.block, query: { - // @ts-expect-error allow to use string as enum - type: 'boolean', - // @ts-expect-error allow to use string as enum - occur: 'must', - queries: [ - { - // @ts-expect-error allow to use string as enum - type: 'match', - field: 'content', - match: 'hello world', - }, - { - // @ts-expect-error allow to use string as enum - type: 'boolean', - // @ts-expect-error allow to use string as enum - occur: 'should', - queries: [ - { - // @ts-expect-error allow to use string as enum - type: 'match', - field: 'content', - match: 'hello world', - }, - { - // @ts-expect-error allow to use string as enum - type: 'boost', - boost: 1.5, - query: { - // @ts-expect-error allow to use string as enum - type: 'match', - field: 'flavour', - match: 'affine:page', - }, - }, - ], - }, - ], + type: SearchQueryType.match, + field: 'content', + match: 'hello', }, field: 'docId', options: { - pagination: { - limit: 50, - skip: 0, - }, + pagination: { limit: 50, skip: 0 }, hits: { - pagination: { - limit: 2, - skip: 0, - }, - fields: ['blockId', 'flavour'], - highlights: [ - { - field: 'content', - before: '', - end: '', - }, - ], + pagination: { limit: 2, skip: 0 }, + fields: ['docId', 'blockId', 'content'], }, }, }, }, }); - t.truthy(result.workspace.aggregate, 'failed to aggregate'); - t.is(result.workspace.aggregate.pagination.count, 5); - t.is(result.workspace.aggregate.pagination.hasMore, true); - t.truthy(result.workspace.aggregate.pagination.nextCursor); - t.snapshot(result.workspace.aggregate.buckets); + t.is(result.workspace.aggregate.pagination.count, 2); + t.deepEqual( + result.workspace.aggregate.buckets.map(bucket => bucket.key).sort(), + ['doc-0', 'doc-1'] + ); }); diff --git a/packages/backend/server/src/__tests__/e2e/indexer/search-docs.spec.ts b/packages/backend/server/src/__tests__/e2e/indexer/search-docs.spec.ts index 95fc5e1bbb..1ad098d79a 100644 --- a/packages/backend/server/src/__tests__/e2e/indexer/search-docs.spec.ts +++ b/packages/backend/server/src/__tests__/e2e/indexer/search-docs.spec.ts @@ -1,182 +1,57 @@ -import { indexerSearchDocsQuery, SearchTable } from '@affine/graphql'; -import { omit } from 'lodash-es'; +import { indexerSearchDocsQuery } from '@affine/graphql'; +import { ConfigFactory } from '../../../base'; +import { createDocWithMarkdown } from '../../../native'; +import { SearchProviderType } from '../../../plugins/indexer/config'; import { IndexerService } from '../../../plugins/indexer/service'; import { Mockers } from '../../mocks'; import { app, e2e } from '../test'; e2e('should search docs by keyword', async t => { const owner = await app.signup(); + const workspace = await app.create(Mockers.Workspace, { owner }); + for (const docId of ['doc-0', 'doc-1', 'doc-2']) { + await app.create(Mockers.DocMeta, { workspaceId: workspace.id, docId }); + await app.create(Mockers.DocSnapshot, { + workspaceId: workspace.id, + docId, + user: owner, + blob: createDocWithMarkdown(docId, `${docId} hello`, docId), + }); + await app.get(IndexerService).indexDoc(workspace.id, docId); + } - const workspace = await app.create(Mockers.Workspace, { - owner, - }); - - const indexerService = app.get(IndexerService); - - await indexerService.write( - SearchTable.block, - [ - { - docId: 'doc-0', - workspaceId: workspace.id, - content: 'test1 hello', - flavour: 'markdown', - blockId: 'block-0', - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2025-04-22T00:00:00.000Z'), - updatedAt: new Date('2025-04-22T00:00:00.000Z'), - }, - { - docId: 'doc-1', - workspaceId: workspace.id, - content: 'test2 hello', - flavour: 'markdown', - blockId: 'block-1', - refDocId: ['doc-0'], - ref: ['{"foo": "bar1"}'], - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2021-04-22T00:00:00.000Z'), - updatedAt: new Date('2021-04-22T00:00:00.000Z'), - }, - { - docId: 'doc-2', - workspaceId: workspace.id, - content: 'test3 hello', - flavour: 'markdown', - blockId: 'block-2', - refDocId: ['doc-0', 'doc-2'], - ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'], - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2025-03-22T00:00:00.000Z'), - updatedAt: new Date('2025-03-22T03:00:01.000Z'), - }, - ], - { - refresh: true, - } - ); - - const result = await app.gql({ + const search = app.gql({ query: indexerSearchDocsQuery, - variables: { - id: workspace.id, - input: { - keyword: 'hello', - }, - }, + variables: { id: workspace.id, input: { keyword: 'hello', limit: 2 } }, }); + if ( + app.get(ConfigFactory).config.indexer.provider.type === + SearchProviderType.Manticoresearch + ) { + await t.throwsAsync(search, { + message: /Invalid indexer input: unsupported_query/, + }); + return; + } - t.is(result.workspace.searchDocs.length, 3); - t.snapshot( - result.workspace.searchDocs.map(doc => - omit(doc, 'createdByUser', 'updatedByUser') - ) - ); -}); - -e2e('should search docs by keyword with limit 1', async t => { - const owner = await app.signup(); - - const workspace = await app.create(Mockers.Workspace, { - owner, - }); - - const indexerService = app.get(IndexerService); - - await indexerService.write( - SearchTable.block, - [ - { - docId: 'doc-0', - workspaceId: workspace.id, - content: 'test1 hello', - flavour: 'markdown', - blockId: 'block-0', - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2025-04-22T00:00:00.000Z'), - updatedAt: new Date('2025-04-22T00:00:00.000Z'), - }, - { - docId: 'doc-1', - workspaceId: workspace.id, - content: 'test2 hello', - flavour: 'markdown', - blockId: 'block-1', - refDocId: ['doc-0'], - ref: ['{"foo": "bar1"}'], - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2021-04-22T00:00:00.000Z'), - updatedAt: new Date('2021-04-22T00:00:00.000Z'), - }, - { - docId: 'doc-2', - workspaceId: workspace.id, - content: 'test3 hello', - flavour: 'markdown', - blockId: 'block-2', - refDocId: ['doc-0', 'doc-2'], - ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'], - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2025-03-22T00:00:00.000Z'), - updatedAt: new Date('2025-03-22T03:00:01.000Z'), - }, - ], - { - refresh: true, - } - ); - - const result = await app.gql({ - query: indexerSearchDocsQuery, - variables: { - id: workspace.id, - input: { - keyword: 'hello', - limit: 1, - }, - }, - }); - - t.is(result.workspace.searchDocs.length, 1); - t.snapshot( - result.workspace.searchDocs.map(doc => - omit(doc, 'createdByUser', 'updatedByUser') - ) - ); + const result = await search; + t.is(result.workspace.searchDocs.length, 2); + t.true(result.workspace.searchDocs.every(doc => doc.highlight.length > 0)); }); e2e( 'should search docs by keyword failed when workspace is no permission', async t => { const owner = await app.signup(); - - const workspace = await app.create(Mockers.Workspace, { - owner, - }); - - // signup another user + const workspace = await app.create(Mockers.Workspace, { owner }); await app.signup(); - await t.throwsAsync( app.gql({ query: indexerSearchDocsQuery, - variables: { - id: workspace.id, - input: { - keyword: 'hello', - }, - }, + variables: { id: workspace.id, input: { keyword: 'hello' } }, }), - { - message: /You do not have permission to access Space/, - } + { message: /You do not have permission to access Space/ } ); } ); diff --git a/packages/backend/server/src/__tests__/e2e/indexer/search.spec.ts b/packages/backend/server/src/__tests__/e2e/indexer/search.spec.ts index 5bce9f5240..7a81e15a9c 100644 --- a/packages/backend/server/src/__tests__/e2e/indexer/search.spec.ts +++ b/packages/backend/server/src/__tests__/e2e/indexer/search.spec.ts @@ -1,68 +1,40 @@ import { indexerSearchQuery, - SearchQueryOccur, SearchQueryType, SearchTable, } from '@affine/graphql'; import { DocRole } from '../../../models'; +import { createDocWithMarkdown } from '../../../native'; import { IndexerService } from '../../../plugins/indexer/service'; import { Mockers } from '../../mocks'; import { app, e2e } from '../test'; +async function indexDoc( + workspaceId: string, + user: { id: string }, + docId: string, + markdown: string, + defaultRole = DocRole.Manager +) { + await app.create(Mockers.DocMeta, { workspaceId, docId, defaultRole }); + await app.create(Mockers.DocSnapshot, { + workspaceId, + docId, + user, + blob: createDocWithMarkdown(docId, markdown, docId), + }); + await app.get(IndexerService).indexDoc(workspaceId, docId); +} + e2e('should search with query', async t => { const owner = await app.signup(); - - const workspace = await app.create(Mockers.Workspace, { - owner: { id: owner.id }, - }); - - const indexerService = app.get(IndexerService); - - await indexerService.write( - SearchTable.block, - [ - { - docId: 'doc-0', - workspaceId: workspace.id, - content: 'test1', - flavour: 'markdown', - blockId: 'block-0', - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2025-04-22T00:00:00.000Z'), - updatedAt: new Date('2025-04-22T00:00:00.000Z'), - }, - { - docId: 'doc-1', - workspaceId: workspace.id, - content: 'test2', - flavour: 'markdown', - blockId: 'block-1', - refDocId: ['doc-0'], - ref: ['{"foo": "bar1"}'], - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2021-04-22T00:00:00.000Z'), - updatedAt: new Date('2021-04-22T00:00:00.000Z'), - }, - { - docId: 'doc-2', - workspaceId: workspace.id, - content: 'test3', - flavour: 'markdown', - blockId: 'block-2', - refDocId: ['doc-0', 'doc-2'], - ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'], - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2025-03-22T00:00:00.000Z'), - updatedAt: new Date('2025-03-22T00:00:00.000Z'), - }, - ], - { - refresh: true, - } + const workspace = await app.create(Mockers.Workspace, { owner }); + await indexDoc( + workspace.id, + owner, + 'doc-0', + 'searchable first\n\nsearchable second' ); const result = await app.gql({ @@ -72,158 +44,95 @@ e2e('should search with query', async t => { input: { table: SearchTable.block, query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.should, - queries: ['doc-0', 'doc-1', 'doc-2'].map(id => ({ - type: SearchQueryType.match, - field: 'docId', - match: id, - })), - }, - { - type: SearchQueryType.exists, - field: 'refDocId', - }, - ], + type: SearchQueryType.match, + field: 'content', + match: 'searchable', }, options: { - fields: ['refDocId', 'ref'], - pagination: { - limit: 100, - }, + fields: ['docId', 'blockId', 'content'], + highlights: [{ field: 'content', before: '', end: '' }], + pagination: { limit: 100 }, }, }, }, }); - t.truthy(result.workspace.search, 'failed to search'); - t.is(result.workspace.search.pagination.count, 2); - t.is(result.workspace.search.pagination.hasMore, true); - t.truthy(result.workspace.search.pagination.nextCursor); - t.is(result.workspace.search.nodes.length, 2); - t.snapshot(result.workspace.search.nodes); + t.true(result.workspace.search.pagination.count > 0); + t.true( + result.workspace.search.nodes.every(node => + node.fields.docId.includes('doc-0') + ) + ); + t.true( + result.workspace.search.nodes.some(node => + node.highlights?.content?.some((value: string) => value.includes('')) + ) + ); + + const firstPage = await app.gql({ + query: indexerSearchQuery, + variables: { + id: workspace.id, + input: { + table: SearchTable.block, + query: { + type: SearchQueryType.match, + field: 'content', + match: 'searchable', + }, + options: { + fields: ['docId', 'blockId'], + pagination: { limit: 1 }, + }, + }, + }, + }); + const secondPage = await app.gql({ + query: indexerSearchQuery, + variables: { + id: workspace.id, + input: { + table: SearchTable.block, + query: { + type: SearchQueryType.match, + field: 'content', + match: 'searchable', + }, + options: { + fields: ['docId', 'blockId'], + pagination: { + limit: 1, + cursor: firstPage.workspace.search.pagination.nextCursor, + }, + }, + }, + }, + }); + t.not( + firstPage.workspace.search.nodes[0].fields.blockId[0], + secondPage.workspace.search.nodes[0].fields.blockId[0] + ); }); e2e('should filter no read permission docs on team workspace', async t => { const owner = await app.signup(); - const workspace = await app.create(Mockers.Workspace, { + const workspace = await app.create(Mockers.Workspace, { owner }); + await app.create(Mockers.TeamWorkspace, { id: workspace.id }); + await indexDoc( + workspace.id, owner, - }); - await app.create(Mockers.TeamWorkspace, { - id: workspace.id, - }); - - const indexerService = app.get(IndexerService); - await indexerService.write( - SearchTable.block, - [ - { - docId: 'doc-0', - workspaceId: workspace.id, - content: 'test1', - flavour: 'markdown', - blockId: 'block-0', - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2025-04-22T00:00:00.000Z'), - updatedAt: new Date('2025-04-22T00:00:00.000Z'), - }, - { - docId: 'doc-1', - workspaceId: workspace.id, - content: 'test2', - flavour: 'markdown', - blockId: 'block-1', - refDocId: ['doc-0'], - ref: ['{"foo": "bar1"}'], - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2021-04-22T00:00:00.000Z'), - updatedAt: new Date('2021-04-22T00:00:00.000Z'), - }, - { - docId: 'doc-2', - workspaceId: workspace.id, - content: 'test3', - flavour: 'markdown', - blockId: 'block-2', - refDocId: ['doc-0', 'doc-2'], - ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'], - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2025-03-22T00:00:00.000Z'), - updatedAt: new Date('2025-03-22T00:00:00.000Z'), - }, - ], - { - refresh: true, - } + 'private-doc', + 'team secret searchable', + DocRole.None ); - // set all docs to no access - await app.create(Mockers.DocMeta, { - workspaceId: workspace.id, - docId: 'doc-0', - defaultRole: DocRole.None, - }); - await app.create(Mockers.DocMeta, { - workspaceId: workspace.id, - docId: 'doc-1', - defaultRole: DocRole.None, - }); - await app.create(Mockers.DocMeta, { - workspaceId: workspace.id, - docId: 'doc-2', - defaultRole: DocRole.None, - }); - // owner can read all docs - const result = await app.gql({ - query: indexerSearchQuery, - variables: { - id: workspace.id, - input: { - table: SearchTable.block, - query: { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - options: { - fields: ['docId', 'blockId', 'refDocId', 'ref'], - pagination: { - limit: 100, - }, - }, - }, - }, - }); - - t.snapshot(result.workspace.search.nodes); - - // other user can only read docs that they have read permission - const other = await app.signup(); + const member = await app.signup(); await app.create(Mockers.WorkspaceUser, { workspaceId: workspace.id, - userId: other.id, + userId: member.id, }); - await app.create(Mockers.DocUser, { - workspaceId: workspace.id, - docId: 'doc-0', - userId: other.id, - type: DocRole.Reader, - }); - await app.create(Mockers.DocUser, { - workspaceId: workspace.id, - docId: 'doc-1', - userId: other.id, - type: DocRole.Manager, - }); - - const otherResult = await app.gql({ + await app.get(IndexerService).reconcileWorkspace(workspace.id); + const denied = await app.gql({ query: indexerSearchQuery, variables: { id: workspace.id, @@ -231,132 +140,74 @@ e2e('should filter no read permission docs on team workspace', async t => { table: SearchTable.block, query: { type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - options: { - fields: ['docId', 'blockId', 'refDocId', 'ref'], - pagination: { - limit: 100, - }, + field: 'content', + match: 'secret', }, + options: { fields: ['docId'], pagination: { limit: 10 } }, }, }, }); + t.is(denied.workspace.search.pagination.count, 0); - t.snapshot(otherResult.workspace.search.nodes); + await app.create(Mockers.DocUser, { + workspaceId: workspace.id, + docId: 'private-doc', + userId: member.id, + type: DocRole.Reader, + }); + await app.get(IndexerService).reconcileWorkspace(workspace.id); + const allowed = await app.gql({ + query: indexerSearchQuery, + variables: { + id: workspace.id, + input: { + table: SearchTable.block, + query: { + type: SearchQueryType.match, + field: 'content', + match: 'secret', + }, + options: { fields: ['docId'], pagination: { limit: 10 } }, + }, + }, + }); + t.true(allowed.workspace.search.pagination.count > 0); + + await app.models.docUser.delete(workspace.id, 'private-doc', member.id); + await app.get(IndexerService).reconcileWorkspace(workspace.id); + const revoked = await app.gql({ + query: indexerSearchQuery, + variables: { + id: workspace.id, + input: { + table: SearchTable.block, + query: { + type: SearchQueryType.match, + field: 'content', + match: 'secret', + }, + options: { fields: ['docId'], pagination: { limit: 10 } }, + }, + }, + }); + t.is(revoked.workspace.search.pagination.count, 0); }); e2e('should return empty results when search not match any docs', async t => { const owner = await app.signup(); - const workspace = await app.create(Mockers.Workspace, { - owner, - }); - - const result = await app.gql({ - query: indexerSearchQuery, - variables: { - id: workspace.id, - input: { - table: SearchTable.block, - query: { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - options: { - fields: ['docId', 'blockId', 'refDocId', 'ref'], - pagination: { - limit: 100, - }, - }, - }, - }, - }); - - t.snapshot(result); -}); - -e2e('should return empty nodes when docId not exists', async t => { - const owner = await app.signup(); - const workspace = await app.create(Mockers.Workspace, { - owner, - }); - + const workspace = await app.create(Mockers.Workspace, { owner }); + await app.get(IndexerService).reconcileWorkspace(workspace.id); const result = await app.gql({ query: indexerSearchQuery, variables: { id: workspace.id, input: { table: SearchTable.doc, - query: { - type: SearchQueryType.match, - field: 'docId', - match: 'not-exists-doc-id', - }, - options: { - fields: ['summary'], - pagination: { - limit: 1, - }, - }, + query: { type: SearchQueryType.match, field: 'title', match: 'absent' }, + options: { fields: ['docId'], pagination: { limit: 10 } }, }, }, }); - - t.snapshot(result); + t.is(result.workspace.search.pagination.count, 0); + t.deepEqual(result.workspace.search.nodes, []); }); - -e2e( - 'should empty doc summary string when doc exists but no summary', - async t => { - const owner = await app.signup(); - const workspace = await app.create(Mockers.Workspace, { - owner, - }); - - const indexerService = app.get(IndexerService); - - await indexerService.write( - SearchTable.doc, - [ - { - docId: 'doc-1-without-summary', - workspaceId: workspace.id, - title: 'test1', - summary: '', - createdByUserId: owner.id, - updatedByUserId: owner.id, - createdAt: new Date('2025-04-22T00:00:00.000Z'), - updatedAt: new Date('2025-04-22T00:00:00.000Z'), - }, - ], - { - refresh: true, - } - ); - - const result = await app.gql({ - query: indexerSearchQuery, - variables: { - id: workspace.id, - input: { - table: SearchTable.doc, - query: { - type: SearchQueryType.match, - field: 'docId', - match: 'doc-1-without-summary', - }, - options: { - fields: ['summary'], - pagination: { - limit: 1, - }, - }, - }, - }, - }); - - t.snapshot(result.workspace.search.nodes); - } -); diff --git a/packages/backend/server/src/__tests__/e2e/notification/resolver.spec.ts b/packages/backend/server/src/__tests__/e2e/notification/resolver.spec.ts index 3046640593..9b0f356d20 100644 --- a/packages/backend/server/src/__tests__/e2e/notification/resolver.spec.ts +++ b/packages/backend/server/src/__tests__/e2e/notification/resolver.spec.ts @@ -121,6 +121,51 @@ e2e('should mention user in a doc', async t => { t.falsy(body2.workspace!.avatarUrl); }); +e2e( + 'notification totalCount selection does not load the notification list', + async t => { + const { member, owner, workspace } = await init(); + + await app.login(owner); + await app.gql({ + query: mentionUserMutation, + variables: { + input: { + userId: member.id, + workspaceId: workspace.id, + doc: { + id: 'count-only-doc', + title: 'count-only-doc', + mode: DocMode.page, + }, + }, + }, + }); + + await app.login(member); + const result = (await app.gql({ + query: { + ...listNotificationsQuery, + op: 'CountOnlyNotifications', + query: ` + query CountOnlyNotifications($pagination: PaginationInput!) { + currentUser { + notifications(pagination: $pagination) { + totalCount + } + } + } + `, + }, + variables: { pagination: { first: 10, offset: 0 } }, + })) as unknown as { + currentUser: { notifications: { totalCount: number } }; + }; + + t.is(result.currentUser.notifications.totalCount, 1); + } +); + e2e('should mention doc mode support string value', async t => { const { member, owner, workspace } = await init(); diff --git a/packages/backend/server/src/__tests__/env.spec.ts b/packages/backend/server/src/__tests__/env.spec.ts index 97cecdb7b4..a5a8482c29 100644 --- a/packages/backend/server/src/__tests__/env.spec.ts +++ b/packages/backend/server/src/__tests__/env.spec.ts @@ -68,14 +68,20 @@ test('should read DEPLOYMENT_TYPE', t => { test('should read FLAVOR', t => { t.deepEqual( - ['allinone', 'graphql', 'sync', 'renderer', 'front', 'doc', 'script'].map( - envVal => { - process.env.SERVER_FLAVOR = envVal; - const env = new Env(); - return env.FLAVOR; - } - ), - ['allinone', 'graphql', 'sync', 'renderer', 'front', 'doc', 'script'] + [ + 'allinone', + 'graphql', + 'sync', + 'renderer', + 'front', + 'worker', + 'script', + ].map(envVal => { + process.env.SERVER_FLAVOR = envVal; + const env = new Env(); + return env.FLAVOR; + }), + ['allinone', 'graphql', 'sync', 'renderer', 'front', 'worker', 'script'] ); t.throws( @@ -85,7 +91,7 @@ test('should read FLAVOR', t => { }, { message: - 'Invalid value "unknown" for environment variable SERVER_FLAVOR, expected one of ["allinone","graphql","sync","renderer","front","doc","script"]', + 'Invalid value "unknown" for environment variable SERVER_FLAVOR, expected one of ["allinone","graphql","sync","renderer","front","worker","script"]', } ); }); @@ -113,7 +119,7 @@ test('should tell flavors correctly', t => { sync: true, renderer: true, front: false, - doc: true, + worker: true, script: false, }); @@ -123,7 +129,7 @@ test('should tell flavors correctly', t => { sync: false, renderer: false, front: false, - doc: false, + worker: false, script: false, }); @@ -133,7 +139,7 @@ test('should tell flavors correctly', t => { sync: false, renderer: false, front: true, - doc: false, + worker: false, script: false, }); @@ -143,7 +149,7 @@ test('should tell flavors correctly', t => { sync: false, renderer: false, front: false, - doc: false, + worker: false, script: true, }); }); diff --git a/packages/backend/server/src/__tests__/event/cluster.spec.ts b/packages/backend/server/src/__tests__/event/cluster.spec.ts index c51e919313..1fb65e00cb 100644 --- a/packages/backend/server/src/__tests__/event/cluster.spec.ts +++ b/packages/backend/server/src/__tests__/event/cluster.spec.ts @@ -63,6 +63,21 @@ test('should broadcast event to cluster instances', async t => { off(); }); +test('should preserve encoded binary updates across cluster instances', async t => { + const { app1, app2 } = t.context; + const eventbus1 = app1.get(EventBus); + const eventbus2 = app2.get(EventBus); + const listener = Sinon.spy(app1.get(Listeners), 'onEncodedBinaryEvent'); + const payload = { + updates: [Buffer.from(new Uint8Array([1, 2, 3])).toString('base64')], + }; + + eventbus2.broadcast('__test__.encodedBinary', payload); + await eventbus1.waitFor('__test__.encodedBinary'); + + t.true(listener.calledOnceWith(payload)); +}); + test('should continuously use the same request id', async t => { const { app1, app2 } = t.context; diff --git a/packages/backend/server/src/__tests__/event/provider.ts b/packages/backend/server/src/__tests__/event/provider.ts index 1d707bdf5b..5a305d8881 100644 --- a/packages/backend/server/src/__tests__/event/provider.ts +++ b/packages/backend/server/src/__tests__/event/provider.ts @@ -7,6 +7,7 @@ declare global { interface Events { '__test__.event': { count: number }; '__test__.event2': { count: number }; + '__test__.encodedBinary': { updates: string[] }; '__test__.throw': { count: number }; '__test__.suppressThrow': {}; '__test__.requestId': {}; @@ -28,6 +29,11 @@ export class Listeners { return payload; } + @OnEvent('__test__.encodedBinary') + onEncodedBinaryEvent(payload: Events['__test__.encodedBinary']) { + return payload; + } + @OnEvent('__test__.throw') onThrow() { throw new Error('Error in event handler'); diff --git a/packages/backend/server/src/__tests__/mocks/eventbus.mock.ts b/packages/backend/server/src/__tests__/mocks/eventbus.mock.ts index 9a09b59fc5..94332699df 100644 --- a/packages/backend/server/src/__tests__/mocks/eventbus.mock.ts +++ b/packages/backend/server/src/__tests__/mocks/eventbus.mock.ts @@ -6,10 +6,10 @@ import { EventName } from '../../base/event/def'; export class MockEventBus { private readonly stub = Sinon.createStubInstance(EventBus); - emit = this.stub.emitAsync; - emitAsync = this.stub.emitAsync; - emitDetached = this.stub.emitAsync; - broadcast = this.stub.broadcast; + emit: Sinon.SinonStub = this.stub.emitAsync; + emitAsync: Sinon.SinonStub = this.stub.emitAsync; + emitDetached: Sinon.SinonStub = this.stub.emitAsync; + broadcast: Sinon.SinonStub = this.stub.broadcast; last( name: Event @@ -22,7 +22,6 @@ export class MockEventBus { throw new Error(`Event ${name} never called`); } - // @ts-expect-error allow return { name, payload: call.args[1], diff --git a/packages/backend/server/src/__tests__/models/copilot-job.spec.ts b/packages/backend/server/src/__tests__/models/copilot-job.spec.ts index 9752580f60..e1aea4ed7d 100644 --- a/packages/backend/server/src/__tests__/models/copilot-job.spec.ts +++ b/packages/backend/server/src/__tests__/models/copilot-job.spec.ts @@ -136,127 +136,3 @@ test('should claim job', async t => { 'should update job status to claimed' ); }); - -test('should fence transcript dispatch generations atomically', async t => { - const task = await t.context.transcriptTask.create({ - userId: user.id, - workspaceId: workspace.id, - blobId: 'transcript-blob', - recipeId: 'transcript.audio', - recipeVersion: 'v1', - inputSnapshot: { normalizedTranscript: 'source' }, - }); - const adoptions = await Promise.all([ - t.context.transcriptTask.adoptLegacyDispatch( - task.id, - null, - 'legacy-generation-a' - ), - t.context.transcriptTask.adoptLegacyDispatch( - task.id, - null, - 'legacy-generation-b' - ), - ]); - t.is(adoptions.filter(Boolean).length, 1); - const adopted = await t.context.transcriptTask.get(task.id); - const adoptedGeneration = adopted?.dispatchGeneration; - if (!adoptedGeneration) { - t.fail('legacy dispatch should have a generation'); - return; - } - t.true( - await t.context.transcriptTask.claimDispatch( - task.id, - adoptedGeneration, - null - ) - ); - t.true( - await t.context.transcriptTask.completeDispatch( - task.id, - adoptedGeneration, - null, - { - status: 'failed', - protectedResult: { normalizedTranscript: 'source' }, - errorCode: 'provider_failed', - } - ) - ); - - const claims = await Promise.all([ - t.context.transcriptTask.claimRetry( - task.id, - user.id, - workspace.id, - null, - 'generation-a' - ), - t.context.transcriptTask.claimRetry( - task.id, - user.id, - workspace.id, - null, - 'generation-b' - ), - ]); - t.is(claims.filter(Boolean).length, 1); - - const claimed = await t.context.transcriptTask.get(task.id); - const generation = claimed?.dispatchGeneration; - if (!generation) { - t.fail('retry should have a dispatch generation'); - return; - } - t.false( - await t.context.transcriptTask.claimDispatch( - task.id, - generation === 'generation-a' ? 'generation-b' : 'generation-a', - null - ) - ); - t.true( - await t.context.transcriptTask.claimDispatch(task.id, generation, null) - ); - t.true( - await t.context.transcriptTask.attachActionRun( - task.id, - generation, - null, - 'run-next' - ) - ); - t.false( - await t.context.transcriptTask.attachActionRun( - task.id, - generation, - null, - 'run-duplicate' - ) - ); - t.false( - await t.context.transcriptTask.completeDispatch( - task.id, - generation, - 'run-duplicate', - { status: 'ready' } - ) - ); - t.true( - await t.context.transcriptTask.completeDispatch( - task.id, - generation, - 'run-next', - { - status: 'ready', - protectedResult: { normalizedTranscript: 'result' }, - } - ) - ); - t.like(await t.context.transcriptTask.get(task.id), { - status: 'ready', - dispatchGeneration: null, - actionRunId: 'run-next', - }); -}); diff --git a/packages/backend/server/src/__tests__/models/session.spec.ts b/packages/backend/server/src/__tests__/models/session.spec.ts index 4c17b1d33d..7fd548d563 100644 --- a/packages/backend/server/src/__tests__/models/session.spec.ts +++ b/packages/backend/server/src/__tests__/models/session.spec.ts @@ -4,7 +4,7 @@ import ava, { TestFn } from 'ava'; import { Config } from '../../base/config'; import { SessionModel } from '../../models/session'; import { UserModel } from '../../models/user'; -import { createTestingModule, type TestingModule } from '../utils'; +import { createTestingModule, sleep, type TestingModule } from '../utils'; interface Context { config: Config; @@ -109,6 +109,7 @@ test('should refresh exists userSession', async t => { t.is(userSession.userId, user.id); t.not(userSession.expiresAt, null); + await sleep(1); const existsUserSession = await t.context.session.createOrRefreshUserSession( user.id, session.id diff --git a/packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts b/packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts index bbe598e1f6..57cef071e4 100644 --- a/packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts +++ b/packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts @@ -4,7 +4,11 @@ import ava, { TestFn } from 'ava'; import Sinon from 'sinon'; import { OneDay } from '../../base'; -import { StorageModule, WorkspaceBlobStorage } from '../../core/storage'; +import { + StorageModule, + StorageWorkerModule, + WorkspaceBlobStorage, +} from '../../core/storage'; import { BlobUploadCleanupJob } from '../../core/storage/job'; import { StorageRuntimeProvider } from '../../core/storage-runtime'; import { MockUser, MockWorkspace } from '../mocks'; @@ -25,7 +29,7 @@ test.before(async t => { cleanupExpiredPendingBlobs: Sinon.stub(), }; t.context.module = await createTestingModule({ - imports: [ScheduleModule.forRoot(), StorageModule], + imports: [ScheduleModule.forRoot(), StorageModule, StorageWorkerModule], tapModule: builder => { builder .overrideProvider(StorageRuntimeProvider) diff --git a/packages/backend/server/src/__tests__/sync/gateway.spec.ts b/packages/backend/server/src/__tests__/sync/gateway.spec.ts index dca851ae7d..e1ef95e6b8 100644 --- a/packages/backend/server/src/__tests__/sync/gateway.spec.ts +++ b/packages/backend/server/src/__tests__/sync/gateway.spec.ts @@ -3,7 +3,7 @@ import test, { type ExecutionContext } from 'ava'; import { io, type Socket as SocketIOClient } from 'socket.io-client'; import { Doc, encodeStateAsUpdate } from 'yjs'; -import { CANARY_CLIENT_VERSION_MAX_AGE_DAYS } from '../../base'; +import { CANARY_CLIENT_VERSION_MAX_AGE_DAYS, EventBus } from '../../base'; import { DocRole, Models, @@ -312,71 +312,7 @@ test('should reject websocket jwt auth after session deletion', async t => { } }); -test('clientVersion=0.25.0 should only receive space:broadcast-doc-update', async t => { - const { user, cookieHeader } = await login(app); - const spaceId = user.id; - const update = createYjsUpdateBase64(); - - const sender = createClient(url, cookieHeader); - const receiver = createClient(url, cookieHeader); - - try { - await Promise.all([waitForConnect(sender), waitForConnect(receiver)]); - - const receiverJoin = unwrapResponse( - t, - await emitWithAck<{ clientId: string; success: boolean }>( - receiver, - 'space:join', - { spaceType: 'userspace', spaceId, clientVersion: '0.25.0' } - ) - ); - t.true(receiverJoin.success); - - const senderJoin = unwrapResponse( - t, - await emitWithAck<{ clientId: string; success: boolean }>( - sender, - 'space:join', - { spaceType: 'userspace', spaceId, clientVersion: '0.26.0' } - ) - ); - t.true(senderJoin.success); - - const onUpdate = waitForEvent<{ - spaceType: string; - spaceId: string; - docId: string; - update: string; - }>(receiver, 'space:broadcast-doc-update'); - const noUpdates = expectNoEvent(receiver, 'space:broadcast-doc-updates'); - - const pushRes = await emitWithAck<{ accepted: true; timestamp?: number }>( - sender, - 'space:push-doc-update', - { - spaceType: 'userspace', - spaceId, - docId: 'doc-1', - update, - } - ); - unwrapResponse(t, pushRes); - - const message = await onUpdate; - t.is(message.spaceType, 'userspace'); - t.is(message.spaceId, spaceId); - t.is(message.docId, 'doc-1'); - t.is(message.update, update); - - await noUpdates; - } finally { - sender.disconnect(); - receiver.disconnect(); - } -}); - -test('clientVersion>=0.26.0 should only receive space:broadcast-doc-updates', async t => { +test('clientVersion>=0.26.0 should receive legacy space:broadcast-doc-updates', async t => { const { user, cookieHeader } = await loginWithCookie(app); const spaceId = user.id; const update = createYjsUpdateBase64(); @@ -402,7 +338,7 @@ test('clientVersion>=0.26.0 should only receive space:broadcast-doc-updates', as await emitWithAck<{ clientId: string; success: boolean }>( sender, 'space:join', - { spaceType: 'userspace', spaceId, clientVersion: '0.25.0' } + { spaceType: 'userspace', spaceId, clientVersion: '0.26.0' } ) ); t.true(senderJoin.success); @@ -413,7 +349,6 @@ test('clientVersion>=0.26.0 should only receive space:broadcast-doc-updates', as docId: string; updates: string[]; }>(receiver, 'space:broadcast-doc-updates'); - const noUpdate = expectNoEvent(receiver, 'space:broadcast-doc-update'); const pushRes = await emitWithAck<{ accepted: true; timestamp?: number }>( sender, @@ -432,15 +367,13 @@ test('clientVersion>=0.26.0 should only receive space:broadcast-doc-updates', as t.is(message.spaceId, spaceId); t.is(message.docId, 'doc-2'); t.deepEqual(message.updates, [update]); - - await noUpdate; } finally { sender.disconnect(); receiver.disconnect(); } }); -test('canary date clientVersion should use sync-026 in canary namespace', async t => { +test('canary date clientVersion should use sync-027 in canary namespace', async t => { const prevNamespace = env.NAMESPACE; // @ts-expect-error test env.NAMESPACE = 'dev'; @@ -456,15 +389,18 @@ test('canary date clientVersion should use sync-026 in canary namespace', async try { await Promise.all([waitForConnect(sender), waitForConnect(receiver)]); + const canaryVersion = makeCanaryDateVersion(new Date(), '015'); const receiverJoin = unwrapResponse( t, await emitWithAck<{ clientId: string; success: boolean }>( receiver, - 'space:join', + 'space:join-batch', { - spaceType: 'userspace', - spaceId, - clientVersion: makeCanaryDateVersion(new Date(), '015'), + spaces: [ + { spaceType: 'userspace', spaceId }, + { spaceType: 'userspace', spaceId, docId: 'doc-canary' }, + ], + clientVersion: canaryVersion, } ) ); @@ -474,8 +410,14 @@ test('canary date clientVersion should use sync-026 in canary namespace', async t, await emitWithAck<{ clientId: string; success: boolean }>( sender, - 'space:join', - { spaceType: 'userspace', spaceId, clientVersion: '0.25.0' } + 'space:join-batch', + { + spaces: [ + { spaceType: 'userspace', spaceId }, + { spaceType: 'userspace', spaceId, docId: 'doc-canary' }, + ], + clientVersion: canaryVersion, + } ) ); t.true(senderJoin.success); @@ -486,7 +428,6 @@ test('canary date clientVersion should use sync-026 in canary namespace', async docId: string; updates: string[]; }>(receiver, 'space:broadcast-doc-updates'); - const noUpdate = expectNoEvent(receiver, 'space:broadcast-doc-update'); const pushRes = await emitWithAck<{ accepted: true; timestamp?: number }>( sender, @@ -505,8 +446,6 @@ test('canary date clientVersion should use sync-026 in canary namespace', async t.is(message.spaceId, spaceId); t.is(message.docId, 'doc-canary'); t.deepEqual(message.updates, [update]); - - await noUpdate; } finally { sender.disconnect(); receiver.disconnect(); @@ -517,7 +456,7 @@ test('canary date clientVersion should use sync-026 in canary namespace', async } }); -test('clientVersion<0.25.0 should be rejected and disconnected', async t => { +test('clientVersion<0.26.0 should be rejected and disconnected', async t => { const { user, cookieHeader } = await login(app); const spaceId = user.id; @@ -530,7 +469,7 @@ test('clientVersion<0.25.0 should be rejected and disconnected', async t => { await emitWithAck<{ clientId: string; success: boolean }>( socket, 'space:join', - { spaceType: 'userspace', spaceId, clientVersion: '0.24.4' } + { spaceType: 'userspace', spaceId, clientVersion: '0.25.0' } ) ); t.false(res.success); @@ -620,7 +559,7 @@ test('canary date clientVersion should be rejected outside canary namespace', as } }); -test('space:join-awareness should reject clientVersion<0.25.0', async t => { +test('space:join-awareness should reject clientVersion<0.26.0', async t => { const { user, cookieHeader } = await login(app); const spaceId = user.id; @@ -637,7 +576,7 @@ test('space:join-awareness should reject clientVersion<0.25.0', async t => { spaceType: 'userspace', spaceId, docId: 'doc-awareness', - clientVersion: '0.24.4', + clientVersion: '0.25.0', } ) ); @@ -649,6 +588,611 @@ test('space:join-awareness should reject clientVersion<0.25.0', async t => { } }); +test('new clients must use batch join endpoints on new servers', async t => { + const { user, cookieHeader } = await login(app); + const requests = [ + { + event: 'space:join', + payload: { + spaceType: 'userspace', + spaceId: user.id, + clientVersion: '0.27.5', + }, + }, + { + event: 'space:join-awareness', + payload: { + spaceType: 'userspace', + spaceId: user.id, + docId: 'doc-awareness', + clientVersion: '0.27.5', + }, + }, + ] as const; + + for (const request of requests) { + const socket = createClient(url, cookieHeader); + try { + await waitForConnect(socket); + const result = unwrapResponse( + t, + await emitWithAck<{ clientId: string; success: boolean }>( + socket, + request.event, + request.payload + ) + ); + t.false(result.success); + await waitForDisconnect(socket); + } finally { + socket.disconnect(); + } + } +}); + +test('space:join-batch should validate entries before joining', async t => { + const { user, cookieHeader } = await login(app); + const socket = createClient(url, cookieHeader); + const spaceId = user.id; + + try { + await waitForConnect(socket); + + const invalidBatches = [ + { + label: 'empty', + payload: { spaces: [], clientVersion: '0.27.5' }, + }, + { + label: 'missing client version', + payload: { spaces: [{ spaceType: 'userspace', spaceId }] }, + }, + { + label: 'cross workspace', + payload: { + spaces: [ + { spaceType: 'userspace', spaceId }, + { spaceType: 'userspace', spaceId: `${spaceId}-other` }, + ], + clientVersion: '0.27.5', + }, + }, + { + label: 'duplicate', + payload: { + spaces: [ + { spaceType: 'userspace', spaceId, docId: 'doc-1' }, + { spaceType: 'userspace', spaceId, docId: 'doc-1' }, + ], + clientVersion: '0.27.5', + }, + }, + { + label: 'invalid entry', + payload: { + spaces: [{ spaceType: 'invalid', spaceId }], + clientVersion: '0.27.5', + }, + }, + { + label: 'over limit', + payload: { + spaces: Array.from({ length: 101 }, (_, index) => ({ + spaceType: 'userspace', + spaceId, + docId: `doc-${index}`, + })), + clientVersion: '0.27.5', + }, + }, + ]; + + for (const { label, payload } of invalidBatches) { + const error = getErrorResponse( + t, + await emitWithAck(socket, 'space:join-batch', payload) + ); + t.is(error.name, 'BAD_REQUEST', label); + } + } finally { + socket.disconnect(); + } +}); + +test('space:join-batch should reject clients before 0.27.5', async t => { + const { user, cookieHeader } = await login(app); + const socket = createClient(url, cookieHeader); + + try { + await waitForConnect(socket); + const result = unwrapResponse( + t, + await emitWithAck<{ clientId: string; success: boolean }>( + socket, + 'space:join-batch', + { + spaces: [{ spaceType: 'userspace', spaceId: user.id }], + clientVersion: '0.27.4', + } + ) + ); + t.false(result.success); + await waitForDisconnect(socket); + } finally { + socket.disconnect(); + } +}); + +test('space:join-batch should authorize once and join all requested rooms', async t => { + const models = app.get(Models); + const { user: owner, cookieHeader: ownerCookieHeader } = await login(app); + const { cookieHeader: deniedCookieHeader } = await login(app); + const workspace = await models.workspace.create(owner.id); + + const ownerSocket = createClient(url, ownerCookieHeader); + const receiverSocket = createClient(url, ownerCookieHeader); + const deniedSocket = createClient(url, deniedCookieHeader); + + try { + await Promise.all([ + waitForConnect(ownerSocket), + waitForConnect(receiverSocket), + waitForConnect(deniedSocket), + ]); + + const batch = { + spaces: [ + { spaceType: 'workspace', spaceId: workspace.id }, + { spaceType: 'workspace', spaceId: workspace.id, docId: 'doc-a' }, + { spaceType: 'workspace', spaceId: workspace.id, docId: 'doc-b' }, + ], + clientVersion: '0.27.5', + }; + + for (const socket of [ownerSocket, receiverSocket]) { + const result = unwrapResponse( + t, + await emitWithAck<{ clientId: string; success: boolean }>( + socket, + 'space:join-batch', + batch + ) + ); + t.true(result.success); + } + + const awarenessOnlyResult = unwrapResponse( + t, + await emitWithAck<{ clientId: string; success: boolean }>( + ownerSocket, + 'space:join-batch', + { + spaces: [ + { + spaceType: 'workspace', + spaceId: workspace.id, + docId: 'doc-c', + }, + ], + clientVersion: '0.27.5', + } + ) + ); + t.true(awarenessOnlyResult.success); + + const timestamps = unwrapResponse( + t, + await emitWithAck>( + ownerSocket, + 'space:load-doc-timestamps', + { + spaceType: 'workspace', + spaceId: workspace.id, + } + ) + ); + t.deepEqual(timestamps, {}); + + const deniedError = getErrorResponse( + t, + await emitWithAck(deniedSocket, 'space:join-batch', batch) + ); + t.is(deniedError.name, 'SPACE_ACCESS_DENIED'); + + const deniedSyncRoomError = getErrorResponse( + t, + await emitWithAck(deniedSocket, 'space:load-doc-timestamps', { + spaceType: 'workspace', + spaceId: workspace.id, + }) + ); + t.is(deniedSyncRoomError.name, 'NOT_IN_SPACE'); + + const deniedAwarenessRoomError = getErrorResponse( + t, + await emitWithAck(deniedSocket, 'space:load-awarenesses', { + spaceType: 'workspace', + spaceId: workspace.id, + docId: 'doc-a', + }) + ); + t.is(deniedAwarenessRoomError.name, 'NOT_IN_SPACE'); + + const receivedA = waitForEvent<{ + spaceType: string; + spaceId: string; + docId: string; + awarenessUpdate: string; + }>(receiverSocket, 'space:broadcast-awareness-update'); + const noDeniedEvent = expectNoEvent( + deniedSocket, + 'space:broadcast-awareness-update' + ); + + ownerSocket.emit('space:update-awareness', { + spaceType: 'workspace', + spaceId: workspace.id, + docId: 'doc-a', + awarenessUpdate: 'AQID', + }); + const messageA = await receivedA; + + const receivedB = waitForEvent<{ + spaceType: string; + spaceId: string; + docId: string; + awarenessUpdate: string; + }>(receiverSocket, 'space:broadcast-awareness-update'); + ownerSocket.emit('space:update-awareness', { + spaceType: 'workspace', + spaceId: workspace.id, + docId: 'doc-b', + awarenessUpdate: 'BAUG', + }); + const messageB = await receivedB; + + t.deepEqual( + new Set([messageA.docId, messageB.docId]), + new Set(['doc-a', 'doc-b']) + ); + await noDeniedEvent; + } finally { + ownerSocket.disconnect(); + receiverSocket.disconnect(); + deniedSocket.disconnect(); + } +}); + +test('batch doc entries require Doc.Read atomically', async t => { + const db = app.get(PrismaClient); + const models = app.get(Models); + const { user: owner } = await login(app); + const { user: collaborator, cookieHeader } = await login(app); + const workspace = await models.workspace.create(owner.id); + const docId = 'batch-private-doc'; + + await models.workspaceUser.set( + workspace.id, + collaborator.id, + WorkspaceRole.Collaborator, + { status: WorkspaceMemberStatus.Accepted } + ); + await models.doc.setDefaultRole(workspace.id, docId, DocRole.None); + await createSnapshot(db, { + workspaceId: workspace.id, + docId, + userId: owner.id, + }); + + const socket = createClient(url, cookieHeader); + try { + await waitForConnect(socket); + + const error = getErrorResponse( + t, + await emitWithAck(socket, 'space:join-batch', { + spaces: [ + { spaceType: 'workspace', spaceId: workspace.id }, + { spaceType: 'workspace', spaceId: workspace.id, docId }, + ], + clientVersion: '0.27.5', + }) + ); + t.true(error.message.includes('Doc.Read')); + + const timestampsError = getErrorResponse( + t, + await emitWithAck(socket, 'space:load-doc-timestamps', { + spaceType: 'workspace', + spaceId: workspace.id, + }) + ); + t.is(timestampsError.name, 'NOT_IN_SPACE'); + } finally { + socket.disconnect(); + } +}); + +test('batch sync routes active updates and only broadcasts invalidation to control room', async t => { + const { user, cookieHeader } = await login(app); + const spaceId = user.id; + const sender = createClient(url, cookieHeader); + const receiver = createClient(url, cookieHeader); + const passive = createClient(url, cookieHeader); + + try { + await Promise.all([ + waitForConnect(sender), + waitForConnect(receiver), + waitForConnect(passive), + ]); + + const activeBatch = { + spaces: [ + { spaceType: 'userspace', spaceId }, + { spaceType: 'userspace', spaceId, docId: 'some-doc' }, + ], + clientVersion: '0.27.5', + }; + for (const socket of [sender, receiver]) { + const result = unwrapResponse( + t, + await emitWithAck<{ clientId: string; success: boolean }>( + socket, + 'space:join-batch', + activeBatch + ) + ); + t.true(result.success); + } + const passiveJoin = unwrapResponse( + t, + await emitWithAck<{ clientId: string; success: boolean }>( + passive, + 'space:join-batch', + { + spaces: [{ spaceType: 'userspace', spaceId }], + clientVersion: '0.27.5', + } + ) + ); + t.true(passiveJoin.success); + + const receivedUpdate = waitForEvent<{ + docId: string; + updates: string[]; + }>(receiver, 'space:broadcast-doc-updates'); + const receivedInvalidation = waitForEvent<{ + spaceType: string; + spaceId: string; + timestamp: number; + docId?: string; + updates?: string[]; + }>(receiver, 'space:broadcast-doc-invalidation'); + const noPassiveUpdate = expectNoEvent( + passive, + 'space:broadcast-doc-updates' + ); + + unwrapResponse( + t, + await emitWithAck(sender, 'space:push-doc-update', { + spaceType: 'userspace', + spaceId, + docId: 'some-doc', + update: createYjsUpdateBase64(), + }) + ); + + const [update, invalidation] = await Promise.all([ + receivedUpdate, + receivedInvalidation, + ]); + t.is(update.docId, 'some-doc'); + t.deepEqual(Object.keys(invalidation).sort(), [ + 'spaceId', + 'spaceType', + 'timestamp', + ]); + await noPassiveUpdate; + + const leave = unwrapResponse( + t, + await emitWithAck<{ clientId: string; success: boolean }>( + receiver, + 'space:leave-batch', + { + spaceType: 'userspace', + spaceId, + docIds: ['some-doc'], + } + ) + ); + t.true(leave.success); + + const noLeftUpdate = expectNoEvent(receiver, 'space:broadcast-doc-updates'); + const receivedAfterLeave = waitForEvent( + receiver, + 'space:broadcast-doc-invalidation' + ); + unwrapResponse( + t, + await emitWithAck(sender, 'space:push-doc-update', { + spaceType: 'userspace', + spaceId, + docId: 'some-doc', + update: createYjsUpdateBase64(), + }) + ); + await Promise.all([noLeftUpdate, receivedAfterLeave]); + } finally { + sender.disconnect(); + receiver.disconnect(); + passive.disconnect(); + } +}); + +test('permission revocation removes a active document subscription', async t => { + const db = app.get(PrismaClient); + const models = app.get(Models); + const { user: owner, cookieHeader: ownerCookie } = await login(app); + const { user: collaborator, cookieHeader: collaboratorCookie } = + await login(app); + const workspace = await models.workspace.create(owner.id); + const docId = 'revoked-doc'; + + await models.workspaceUser.set( + workspace.id, + collaborator.id, + WorkspaceRole.Collaborator, + { status: WorkspaceMemberStatus.Accepted } + ); + await models.doc.setDefaultRole(workspace.id, docId, DocRole.None); + await models.docUser.set( + workspace.id, + docId, + collaborator.id, + DocRole.Reader + ); + await createSnapshot(db, { + workspaceId: workspace.id, + docId, + userId: owner.id, + }); + + const ownerSocket = createClient(url, ownerCookie); + const collaboratorSocket = createClient(url, collaboratorCookie); + try { + await Promise.all([ + waitForConnect(ownerSocket), + waitForConnect(collaboratorSocket), + ]); + + for (const socket of [ownerSocket, collaboratorSocket]) { + const response = unwrapResponse( + t, + await emitWithAck<{ clientId: string; success: boolean }>( + socket, + 'space:join-batch', + { + spaces: [ + { spaceType: 'workspace', spaceId: workspace.id }, + { spaceType: 'workspace', spaceId: workspace.id, docId }, + ], + clientVersion: '0.27.5', + } + ) + ); + t.true(response.success); + } + + await models.docUser.delete(workspace.id, docId, collaborator.id); + await app.get(EventBus).emitAsync('doc.grants.changed', { + workspaceId: workspace.id, + docId, + }); + + const noRevokedUpdate = expectNoEvent( + collaboratorSocket, + 'space:broadcast-doc-updates' + ); + unwrapResponse( + t, + await emitWithAck(ownerSocket, 'space:push-doc-update', { + spaceType: 'workspace', + spaceId: workspace.id, + docId, + update: createYjsUpdateBase64(), + }) + ); + await noRevokedUpdate; + } finally { + ownerSocket.disconnect(); + collaboratorSocket.disconnect(); + } +}); + +test('awareness requires Doc.Read but not Doc.Update', async t => { + const db = app.get(PrismaClient); + const models = app.get(Models); + const { user: owner, cookieHeader: ownerCookie } = await login(app); + const { user: reader, cookieHeader: readerCookie } = await login(app); + const workspace = await models.workspace.create(owner.id); + const docId = 'awareness-reader-doc'; + + await models.workspaceUser.set( + workspace.id, + reader.id, + WorkspaceRole.Collaborator, + { status: WorkspaceMemberStatus.Accepted } + ); + await models.doc.setDefaultRole(workspace.id, docId, DocRole.None); + await models.docUser.set(workspace.id, docId, reader.id, DocRole.Reader); + await createSnapshot(db, { + workspaceId: workspace.id, + docId, + userId: owner.id, + }); + + const ownerSocket = createClient(url, ownerCookie); + const readerSocket = createClient(url, readerCookie); + try { + await Promise.all([ + waitForConnect(ownerSocket), + waitForConnect(readerSocket), + ]); + + for (const socket of [ownerSocket, readerSocket]) { + const response = unwrapResponse( + t, + await emitWithAck<{ clientId: string; success: boolean }>( + socket, + 'space:join-batch', + { + spaces: [ + { spaceType: 'workspace', spaceId: workspace.id }, + { spaceType: 'workspace', spaceId: workspace.id, docId }, + ], + clientVersion: '0.27.5', + } + ) + ); + t.true(response.success); + } + + const receivedAwareness = waitForEvent<{ + docId: string; + awarenessUpdate: string; + }>(readerSocket, 'space:broadcast-awareness-update'); + ownerSocket.emit('space:update-awareness', { + spaceType: 'workspace', + spaceId: workspace.id, + docId, + awarenessUpdate: 'AQID', + }); + t.deepEqual(await receivedAwareness, { + spaceType: 'workspace', + spaceId: workspace.id, + docId, + awarenessUpdate: 'AQID', + }); + + const updateError = getErrorResponse( + t, + await emitWithAck(readerSocket, 'space:push-doc-update', { + spaceType: 'workspace', + spaceId: workspace.id, + docId, + update: createYjsUpdateBase64(), + }) + ); + t.is(updateError.name, 'DOC_ACTION_DENIED'); + } finally { + ownerSocket.disconnect(); + readerSocket.disconnect(); + } +}); + test('active users metric should dedupe multiple sockets for one user', async t => { const db = app.get(PrismaClient); await ensureSyncActiveUsersTable(db); diff --git a/packages/backend/server/src/__tests__/utils/runtime-config.ts b/packages/backend/server/src/__tests__/utils/runtime-config.ts index b86896c576..8876ad3570 100644 --- a/packages/backend/server/src/__tests__/utils/runtime-config.ts +++ b/packages/backend/server/src/__tests__/utils/runtime-config.ts @@ -8,7 +8,10 @@ const testPrivateKey = privateKey .export({ format: 'pem', type: 'pkcs8' }) .toString(); -export async function createTestRuntimeConfig(databaseUrl: string) { +export async function createTestRuntimeConfig( + databaseUrl: string, + indexer: AppConfig['indexer'] +) { const directory = await mkdtemp(join(tmpdir(), 'affine-server-test-')); const storagePath = join(directory, 'storage'); const storage = (bucket: string) => ({ @@ -30,6 +33,10 @@ export async function createTestRuntimeConfig(databaseUrl: string) { enabled: true, storage: storage('copilot'), }, + indexer: { + enabled: indexer.enabled, + provider: indexer.provider, + }, }) ); return { diff --git a/packages/backend/server/src/__tests__/utils/testing-module.ts b/packages/backend/server/src/__tests__/utils/testing-module.ts index 72dbe16452..385dd47aae 100644 --- a/packages/backend/server/src/__tests__/utils/testing-module.ts +++ b/packages/backend/server/src/__tests__/utils/testing-module.ts @@ -75,8 +75,10 @@ export async function createTestingModule( moduleDef: TestingModuleMetadata = {}, autoInitialize = true ): Promise { + const config = new ConfigFactory().config; const runtimeConfig = await createTestRuntimeConfig( - new ConfigFactory().config.db.datasourceUrl + config.db.datasourceUrl, + config.indexer ); // setting up let imports = moduleDef.imports ?? [buildAppModule(globalThis.env)]; diff --git a/packages/backend/server/src/__tests__/workspace/controller.spec.ts b/packages/backend/server/src/__tests__/workspace/controller.spec.ts index 7e687348d6..77346c988e 100644 --- a/packages/backend/server/src/__tests__/workspace/controller.spec.ts +++ b/packages/backend/server/src/__tests__/workspace/controller.spec.ts @@ -38,12 +38,17 @@ test.before(async t => { const db = app.get(PrismaClient); - t.context.u1 = await app.signupV1('u1@affine.pro'); t.context.db = db; t.context.app = app; t.context.storage = app.get(WorkspaceBlobStorage); t.context.workspace = app.get(PgWorkspaceDocStorageAdapter); t.context.models = app.get(Models); +}); + +test.beforeEach(async t => { + const { app, db } = t.context; + await app.initTestingDB(); + t.context.u1 = await app.signupV1('u1@affine.pro'); await db.workspaceDoc.create({ data: { diff --git a/packages/backend/server/src/app.module.ts b/packages/backend/server/src/app.module.ts index 32e09037d5..7bd8bf2d83 100644 --- a/packages/backend/server/src/app.module.ts +++ b/packages/backend/server/src/app.module.ts @@ -28,12 +28,16 @@ import { RedisModule } from './base/redis'; import { RateLimiterModule } from './base/throttler'; import { WebSocketModule } from './base/websocket'; import { AuthModule } from './core/auth'; -import { BackendRuntimeModule } from './core/backend-runtime'; +import { + BackendRuntimeModule, + BackendRuntimeProducerModule, + BackendRuntimeWorkerModule, +} from './core/backend-runtime'; import { CommentModule } from './core/comment'; import { ServerConfigModule, ServerConfigResolverModule } from './core/config'; import { DocStorageModule } from './core/doc'; +import { DocJobsModule } from './core/doc-jobs'; import { DocRendererModule } from './core/doc-renderer'; -import { DocServiceModule } from './core/doc-service'; import { FeatureModule } from './core/features'; import { MailModule } from './core/mail'; import { MonitorModule } from './core/monitor'; @@ -44,20 +48,20 @@ import { QuotaModule } from './core/quota'; import { RealtimeModule } from './core/realtime'; import { SelfhostModule } from './core/selfhost'; import { StaticFileModule } from './core/static-files'; -import { StorageModule } from './core/storage'; +import { StorageApiModule, StorageWorkerModule } from './core/storage'; import { StorageRuntimeModule } from './core/storage-runtime'; import { SyncModule } from './core/sync'; import { TelemetryModule } from './core/telemetry'; import { UserModule } from './core/user'; import { VersionModule } from './core/version'; import { WorkspaceModule } from './core/workspaces'; -import { Env } from './env'; +import { Env, ServerRole } from './env'; import { ModelsModule } from './models'; import { CalendarModule } from './plugins/calendar'; import { CaptchaModule } from './plugins/captcha'; import { CopilotModule } from './plugins/copilot'; import { GCloudModule } from './plugins/gcloud'; -import { IndexerModule } from './plugins/indexer'; +import { IndexerModule, IndexerWorkerModule } from './plugins/indexer'; import { LicenseModule } from './plugins/license'; import { OAuthModule } from './plugins/oauth'; import { PaymentModule } from './plugins/payment'; @@ -120,6 +124,7 @@ export const FunctionalityModules = [ RealtimeModule, ModelsModule, BackendRuntimeModule, + BackendRuntimeProducerModule, StorageRuntimeModule, ScheduleModule.forRoot(), MonitorModule, @@ -157,29 +162,27 @@ export class AppModuleBuilder { export function buildAppModule(env: Env) { const factor = new AppModuleBuilder(); + const workerOnly = env.role === ServerRole.Worker; factor // basic .use(...FunctionalityModules) - // enable indexer module on graphql, doc and front service - .useIf( - () => env.flavors.graphql || env.flavors.doc || env.flavors.front, - IndexerModule - ) + // online roles publish indexer events; only the worker registers consumers + .useIf(() => env.isApi || env.isFrontend, IndexerModule) + .useIf(() => env.isWorker, IndexerWorkerModule) - // auth - .use(UserModule, AuthModule, PermissionModule) + // the worker owns doc consumers and schedulers + .useIf(() => env.isWorker, DocJobsModule) + .useIf(() => env.isWorker, BackendRuntimeWorkerModule) + + // auth and business APIs are not part of the queue worker application + .useIf(() => !workerOnly, UserModule, AuthModule, PermissionModule) // business modules - .use( - ServerConfigModule, - FeatureModule, - QuotaModule, - DocStorageModule, - NotificationModule, - MailModule - ) + .use(ServerConfigModule, QuotaModule, DocStorageModule) + .useIf(() => env.isWorker, StorageWorkerModule) + .useIf(() => !workerOnly, FeatureModule, NotificationModule, MailModule) // renderer server and front server .useIf(() => env.flavors.renderer || env.flavors.front, DocRendererModule) // sync server and front server @@ -197,7 +200,7 @@ export function buildAppModule(env: Env) { () => env.flavors.graphql, GqlModule, VersionModule, - StorageModule, + StorageApiModule, ServerConfigResolverModule, WorkspaceModule, LicenseModule, @@ -210,10 +213,12 @@ export function buildAppModule(env: Env) { CommentModule, QueueDashboardModule ) - // doc service and front service - .useIf(() => env.flavors.doc || env.flavors.front, DocServiceModule) // worker for and self-hosted API only for self-host and local development only - .useIf(() => env.dev || env.selfhosted, WorkerModule, SelfhostModule) + .useIf( + () => !workerOnly && (env.dev || env.selfhosted), + WorkerModule, + SelfhostModule + ) // static frontend routes for front flavor .useIf(() => env.flavors.front, StaticFileModule) diff --git a/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts b/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts index ea99b5210b..cb16e6b694 100644 --- a/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts +++ b/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts @@ -102,24 +102,6 @@ test('should be able to safe compare', t => { t.false(t.context.crypto.compare('abc', 'def')); }); -test('should sign and parse internal access token', t => { - const token = t.context.crypto.signInternalAccessToken({ - method: 'GET', - path: '/rpc/workspaces/123/docs/456', - now: 1700000000000, - nonce: 'nonce-123', - }); - - const payload = t.context.crypto.parseInternalAccessToken(token); - t.deepEqual(payload, { - v: 1, - ts: 1700000000000, - nonce: 'nonce-123', - m: 'GET', - p: '/rpc/workspaces/123/docs/456', - }); -}); - test('should be able to hash and verify password', async t => { const password = 'mySecurePassword'; const hash = await t.context.crypto.encryptPassword(password); diff --git a/packages/backend/server/src/base/helpers/crypto.ts b/packages/backend/server/src/base/helpers/crypto.ts index 59b3359716..e1ba70883d 100644 --- a/packages/backend/server/src/base/helpers/crypto.ts +++ b/packages/backend/server/src/base/helpers/crypto.ts @@ -173,67 +173,6 @@ export class CryptoHelper implements OnModuleInit { }); } - signInternalAccessToken(input: { - method: string; - path: string; - now?: number; - nonce?: string; - }) { - const payload = { - v: 1 as const, - ts: input.now ?? Date.now(), - nonce: input.nonce ?? this.randomBytes(16).toString('base64url'), - m: input.method.toUpperCase(), - p: input.path, - }; - const data = Buffer.from(JSON.stringify(payload), 'utf8').toString( - 'base64url' - ); - return this.sign(data); - } - - parseInternalAccessToken(signatureWithData: string): { - v: 1; - ts: number; - nonce: string; - m: string; - p: string; - } | null { - const [data, signature] = signatureWithData.split(','); - if (!signature) { - return null; - } - if (!this.verify(signatureWithData)) { - return null; - } - try { - const json = Buffer.from(data, 'base64url').toString('utf8'); - const payload = JSON.parse(json) as unknown; - if (!payload || typeof payload !== 'object') { - return null; - } - const val = payload as { - v?: unknown; - ts?: unknown; - nonce?: unknown; - m?: unknown; - p?: unknown; - }; - if ( - val.v !== 1 || - typeof val.ts !== 'number' || - typeof val.nonce !== 'string' || - typeof val.m !== 'string' || - typeof val.p !== 'string' - ) { - return null; - } - return { v: 1, ts: val.ts, nonce: val.nonce, m: val.m, p: val.p }; - } catch { - return null; - } - } - encrypt(data: string) { const iv = this.randomBytes(); const cipher = createCipheriv( diff --git a/packages/backend/server/src/base/job/queue/executor.ts b/packages/backend/server/src/base/job/queue/executor.ts index 3b5b31da5a..ba76616c79 100644 --- a/packages/backend/server/src/base/job/queue/executor.ts +++ b/packages/backend/server/src/base/job/queue/executor.ts @@ -2,7 +2,7 @@ import { getQueueToken, getSharedConfigToken } from '@nestjs/bullmq'; import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; import { ModuleRef } from '@nestjs/core'; import { Job, Queue as Bullmq, Worker, WorkerOptions } from 'bullmq'; -import { difference, merge } from 'lodash-es'; +import { merge } from 'lodash-es'; import { CLS_ID, ClsServiceManager } from 'nestjs-cls'; import { Config } from '../../config'; @@ -10,7 +10,8 @@ import { OnEvent } from '../../event'; import { metrics, wrapCallMetric } from '../../metrics'; import { QueueRedis } from '../../redis'; import { genRequestId } from '../../utils'; -import { JOB_SIGNAL, namespace, Queue, QUEUES } from './def'; +import { JOB_SIGNAL, namespace, Queue } from './def'; +import { queuesForRole } from './owner'; import { JobHandlerScanner } from './scanner'; @Injectable() @@ -27,18 +28,7 @@ export class JobExecutor implements OnModuleDestroy { @OnEvent('config.init') async onConfigInit() { - const queues = env.flavors.graphql - ? difference(QUEUES, [Queue.DOC, Queue.INDEXER]) - : []; - - // Enable doc/indexer queues in both doc and front service. - if (env.flavors.doc || env.flavors.front) { - queues.push(Queue.DOC); - // NOTE(@fengmk2): Once the index task cannot be processed in time, it needs to be separated from the doc service and deployed independently. - queues.push(Queue.INDEXER); - } - - await this.startWorkers(queues); + await this.startWorkers(queuesForRole(env.role)); } @OnEvent('config.changed') diff --git a/packages/backend/server/src/base/job/queue/index.ts b/packages/backend/server/src/base/job/queue/index.ts index fab30cbd3d..1f295a2669 100644 --- a/packages/backend/server/src/base/job/queue/index.ts +++ b/packages/backend/server/src/base/job/queue/index.ts @@ -55,3 +55,4 @@ export class JobModule { export { JobQueue }; export { JOB_SIGNAL, OnJob } from './def'; +export { queuesForRole, WORKER_QUEUES } from './owner'; diff --git a/packages/backend/server/src/base/job/queue/owner.ts b/packages/backend/server/src/base/job/queue/owner.ts new file mode 100644 index 0000000000..0eec5237cc --- /dev/null +++ b/packages/backend/server/src/base/job/queue/owner.ts @@ -0,0 +1,24 @@ +import { ServerRole } from '../../../env'; +import { Queue, QUEUES } from './def'; + +export const WORKER_QUEUES = [ + Queue.DOC, + Queue.INDEXER, + Queue.BACKENDRUNTIME, +] as const; + +export function queuesForRole(role: ServerRole | undefined): Queue[] { + switch (role) { + case ServerRole.AllInOne: + return [...QUEUES]; + case ServerRole.Api: + return QUEUES.filter( + queue => !(WORKER_QUEUES as readonly Queue[]).includes(queue) + ); + case ServerRole.Worker: + return [...WORKER_QUEUES]; + case ServerRole.Frontend: + case undefined: + return []; + } +} diff --git a/packages/backend/server/src/base/throttler/index.ts b/packages/backend/server/src/base/throttler/index.ts index 67e51e3a56..b8adff5bc4 100644 --- a/packages/backend/server/src/base/throttler/index.ts +++ b/packages/backend/server/src/base/throttler/index.ts @@ -15,13 +15,98 @@ import { import type { Request, Response } from 'express'; import { Config } from '../config'; +import { CacheRedis } from '../redis'; import { getRequestResponseFromContext } from '../utils/request'; import { getRequestTrackerId } from '../utils/request-tracker'; import type { ThrottlerType } from './config'; import { THROTTLER_PROTECTED, Throttlers } from './decorators'; +const REDIS_THROTTLE_SCRIPT = ` +local now = redis.call("TIME") +local nowMs = now[1] * 1000 + math.floor(now[2] / 1000) +local blockedUntil = tonumber(redis.call("HGET", KEYS[1], "blockedUntil")) or 0 + +if blockedUntil > nowMs then + return { + tonumber(redis.call("HGET", KEYS[1], "hits")) or 0, + redis.call("PTTL", KEYS[1]), + blockedUntil - nowMs + } +end + +if blockedUntil > 0 then + redis.call("HDEL", KEYS[1], "blockedUntil") + redis.call("HSET", KEYS[1], "hits", 0) +end + +local hits = redis.call("HINCRBY", KEYS[1], "hits", 1) +if hits == 1 then + redis.call("PEXPIRE", KEYS[1], ARGV[1]) +end + +local blockTtl = 0 +if hits > tonumber(ARGV[2]) then + blockedUntil = nowMs + tonumber(ARGV[3]) + redis.call("HSET", KEYS[1], "blockedUntil", blockedUntil) + if redis.call("PTTL", KEYS[1]) < tonumber(ARGV[3]) then + redis.call("PEXPIRE", KEYS[1], ARGV[3]) + end + blockTtl = tonumber(ARGV[3]) +end + +return { hits, redis.call("PTTL", KEYS[1]), blockTtl } +`; + @Injectable() -export class ThrottlerStorage extends ThrottlerStorageService {} +export class ThrottlerStorage extends ThrottlerStorageService { + constructor(private readonly redis: CacheRedis) { + super(); + } + + override async increment( + key: string, + ttl: number, + limit: number, + blockDuration: number, + throttlerName: string + ) { + if (env.testing) { + return super.increment(key, ttl, limit, blockDuration, throttlerName); + } + + try { + const result = await this.redis.eval( + REDIS_THROTTLE_SCRIPT, + 1, + key, + ttl, + limit, + Math.max(blockDuration, 1) + ); + if (!Array.isArray(result) || result.length !== 3) { + throw new Error('Unexpected Redis throttler response'); + } + + const totalHits = Number(result[0]); + const timeToExpire = Math.max(0, Math.ceil(Number(result[1]) / 1000)); + const timeToBlockExpire = Math.max( + 0, + Math.ceil(Number(result[2]) / 1000) + ); + + return { + totalHits, + timeToExpire, + isBlocked: timeToBlockExpire > 0, + timeToBlockExpire, + }; + } catch { + // Preserve availability if Redis is unavailable. The inherited local + // storage still protects each process while the shared limiter recovers. + return super.increment(key, ttl, limit, blockDuration, throttlerName); + } + } +} @Injectable() class CustomOptionsFactory implements ThrottlerOptionsFactory { diff --git a/packages/backend/server/src/cli.ts b/packages/backend/server/src/cli.ts index 723bb39588..2cdb6211d6 100644 --- a/packages/backend/server/src/cli.ts +++ b/packages/backend/server/src/cli.ts @@ -55,6 +55,17 @@ function buildProgram(logger: Logger) { }); }); + program + .command('admit-legacy-context-blobs') + .description( + 'Admit legacy context blobs before the cleanup schema migration' + ) + .action(async () => { + await withCliApp(logger, async app => { + await app.get(RunCommand).admitLegacyContextBlobs(); + }); + }); + program .command('revert [name]') .description('Revert one data migration with given name') diff --git a/packages/backend/server/src/core/auth/config.ts b/packages/backend/server/src/core/auth/config.ts index be4e67661e..8bc07c6d00 100644 --- a/packages/backend/server/src/core/auth/config.ts +++ b/packages/backend/server/src/core/auth/config.ts @@ -20,6 +20,11 @@ export interface AuthConfig { requireEmailVerification: boolean; newAccountShareActionDelay: number; trustedCloudflareHeaders: boolean; + signInRateLimit: ConfigItem<{ + ttl: number; + ipLimit: number; + emailLimit: number; + }>; inviteQuotaShadowMode: boolean; inviteQuotaFailOpenOnRuntimeError: boolean; passwordRequirements: ConfigItem<{ @@ -61,6 +66,21 @@ defineModuleConfig('auth', { default: false, shape: z.boolean(), }, + signInRateLimit: { + desc: 'Limits for sign-in attempts shared through Redis by source IP and email. ttl is measured in milliseconds.', + default: { + ttl: 60_000, + ipLimit: 20, + emailLimit: 5, + }, + shape: z + .object({ + ttl: z.number().int().positive(), + ipLimit: z.number().int().positive(), + emailLimit: z.number().int().positive(), + }) + .strict(), + }, inviteQuotaShadowMode: { desc: 'Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions.', default: false, diff --git a/packages/backend/server/src/core/auth/controller.ts b/packages/backend/server/src/core/auth/controller.ts index 4c9aac4ce6..bdc976637d 100644 --- a/packages/backend/server/src/core/auth/controller.ts +++ b/packages/backend/server/src/core/auth/controller.ts @@ -118,7 +118,7 @@ export class AuthController { ) { const credential = SignInBodySchema.parse(body); validators.assertValidEmail(credential.email); - const canSignIn = await this.auth.canSignIn(credential.email); + const canSignIn = await this.auth.canSignIn(credential.email, req); if (!canSignIn) { throw new ActionForbidden(); } diff --git a/packages/backend/server/src/core/auth/guard.ts b/packages/backend/server/src/core/auth/guard.ts index 47095c53f7..af80287956 100644 --- a/packages/backend/server/src/core/auth/guard.ts +++ b/packages/backend/server/src/core/auth/guard.ts @@ -11,12 +11,9 @@ import semver from 'semver'; import { Socket } from 'socket.io'; import { - AccessDenied, AuthenticationRequired, - Cache, checkCanaryDateClientVersion, Config, - CryptoHelper, getClientVersionFromRequest, getRequestResponseFromContext, parseCookies, @@ -32,9 +29,6 @@ import { AuthSessionHttpError } from './session-exchange'; import { isLikelyJwt } from './token'; const PUBLIC_ENTRYPOINT_SYMBOL = Symbol('public'); -const INTERNAL_ENTRYPOINT_SYMBOL = Symbol('internal'); -const INTERNAL_ACCESS_TOKEN_TTL_MS = 5 * 60 * 1000; -const INTERNAL_ACCESS_TOKEN_CLOCK_SKEW_MS = 30 * 1000; type AuthenticatedRequestSession = | { type: 'jwt'; session: Session } @@ -50,8 +44,6 @@ export class AuthGuard implements CanActivate, OnModuleInit { private static readonly CANARY_REQUIRED_VERSION = 'canary (within 2 months)'; constructor( - private readonly crypto: CryptoHelper, - private readonly cache: Cache, private readonly config: Config, private readonly ref: ModuleRef, private readonly reflector: Reflector @@ -67,38 +59,6 @@ export class AuthGuard implements CanActivate, OnModuleInit { const { req, res } = getRequestResponseFromContext(context); const clazz = context.getClass(); const handler = context.getHandler(); - // rpc request is internal - const isInternal = this.reflector.getAllAndOverride( - INTERNAL_ENTRYPOINT_SYMBOL, - [clazz, handler] - ); - if (isInternal) { - const accessToken = req.get('x-access-token'); - if (accessToken) { - const payload = this.crypto.parseInternalAccessToken(accessToken); - if (payload) { - const now = Date.now(); - const method = req.method.toUpperCase(); - const path = req.path; - - const timestampInRange = - payload.ts <= now + INTERNAL_ACCESS_TOKEN_CLOCK_SKEW_MS && - now - payload.ts <= INTERNAL_ACCESS_TOKEN_TTL_MS; - - if (timestampInRange && payload.m === method && payload.p === path) { - const nonceKey = `rpc:nonce:${payload.nonce}`; - const ok = await this.cache.setnx(nonceKey, 1, { - ttl: INTERNAL_ACCESS_TOKEN_TTL_MS, - }); - if (ok) { - return true; - } - } - } - } - throw new AccessDenied('Invalid internal request'); - } - // api is public const isPublic = this.reflector.getAllAndOverride( PUBLIC_ENTRYPOINT_SYMBOL, @@ -327,11 +287,6 @@ export class AuthGuard implements CanActivate, OnModuleInit { */ export const Public = () => SetMetadata(PUBLIC_ENTRYPOINT_SYMBOL, true); -/** - * Mark rpc api to be internal accessible - */ -export const Internal = () => SetMetadata(INTERNAL_ENTRYPOINT_SYMBOL, true); - export const AuthWebsocketOptionsProvider: FactoryProvider = { provide: WEBSOCKET_OPTIONS, useFactory: (config: Config, guard: AuthGuard) => { diff --git a/packages/backend/server/src/core/auth/service.ts b/packages/backend/server/src/core/auth/service.ts index 3ce92e2b1e..cd5bd37e62 100644 --- a/packages/backend/server/src/core/auth/service.ts +++ b/packages/backend/server/src/core/auth/service.ts @@ -1,11 +1,18 @@ -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { Injectable, OnApplicationBootstrap } from '@nestjs/common'; import { Transactional } from '@nestjs-cls/transactional'; import type { CookieOptions, Request, Response } from 'express'; import { assign, pick } from 'lodash-es'; -import { Config, OnEvent, SignUpForbidden } from '../../base'; +import { + Cache, + Config, + getRequestClientIp, + OnEvent, + SignUpForbidden, + TooManyRequest, +} from '../../base'; import { Models, type User, type UserSession } from '../../models'; import { EntitlementService } from '../entitlement'; import { Mailer } from '../mail/mailer'; @@ -46,7 +53,8 @@ export class AuthService implements OnApplicationBootstrap { private readonly models: Models, private readonly mailer: Mailer, private readonly authSessions: AuthSessionService, - private readonly entitlement: EntitlementService + private readonly entitlement: EntitlementService, + private readonly cache: Cache ) { this.cookieOptions = { sameSite: 'lax', @@ -69,11 +77,38 @@ export class AuthService implements OnApplicationBootstrap { } } - async canSignIn(_email: string) { + async canSignIn(email: string, req: Request) { + if (!env.testing) { + const { ttl, ipLimit, emailLimit } = this.config.auth.signInRateLimit; + const normalizedEmail = email.toLowerCase(); + const ip = getRequestClientIp(req); + + const emailAttempts = this.cache.increaseWithTtl( + this.signInRateLimitKey('email', normalizedEmail), + ttl + ); + const ipAttempts = ip + ? this.cache.increaseWithTtl(this.signInRateLimitKey('ip', ip), ttl) + : Promise.resolve(0); + const [emailCount, ipCount] = await Promise.all([ + emailAttempts, + ipAttempts, + ]); + + if (emailCount > emailLimit || ipCount > ipLimit) { + throw new TooManyRequest(); + } + } + // may add more sign-in check later return true; } + private signInRateLimitKey(scope: 'email' | 'ip', value: string) { + const digest = createHash('sha256').update(value).digest('hex'); + return `auth:sign-in-rate:${scope}:${digest}`; + } + /** * @deprecated * diff --git a/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts b/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts index 60d3a0996f..1a8f694cc9 100644 --- a/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts +++ b/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts @@ -16,15 +16,24 @@ import { CopilotSelectedSourcesUnavailable, } from '../../../base'; import { Models } from '../../../models'; -import { BackendRuntimeModule, BackendRuntimeProvider } from '../index'; +import { + BackendRuntimeModule, + BackendRuntimeProducerModule, + BackendRuntimeProvider, + BackendRuntimeWorkerModule, +} from '../index'; import { BackendRuntimeEmbeddingJob, + BackendRuntimeEmbeddingProducer, + BackendRuntimeEmbeddingService, BackendRuntimeHousekeepingJob, } from '../job'; interface Context { module: TestingModule; embeddingJob: BackendRuntimeEmbeddingJob; + embeddingProducer: BackendRuntimeEmbeddingProducer; + embeddingService: BackendRuntimeEmbeddingService; job: BackendRuntimeHousekeepingJob; getSnapshot: Sinon.SinonStub; allowEmbedding: Sinon.SinonStub; @@ -55,7 +64,12 @@ test.before(async t => { syncEmbeddingState: Sinon.stub(), }; t.context.module = await createTestingModule({ - imports: [ScheduleModule.forRoot(), BackendRuntimeModule], + imports: [ + ScheduleModule.forRoot(), + BackendRuntimeModule, + BackendRuntimeProducerModule, + BackendRuntimeWorkerModule, + ], tapModule: builder => { builder .overrideProvider(BackendRuntimeProvider) @@ -81,6 +95,12 @@ test.before(async t => { 'allowEmbedding' ).resolves(true); t.context.embeddingJob = t.context.module.get(BackendRuntimeEmbeddingJob); + t.context.embeddingProducer = t.context.module.get( + BackendRuntimeEmbeddingProducer + ); + t.context.embeddingService = t.context.module.get( + BackendRuntimeEmbeddingService + ); t.context.job = t.context.module.get(BackendRuntimeHousekeepingJob); }); @@ -102,7 +122,7 @@ test.after.always(async t => { }); test('backend-runtime jobs ingest documents and clean runtime state', async t => { - await t.context.embeddingJob.onDocSnapshotUpdated({ + await t.context.embeddingProducer.onDocSnapshotUpdated({ workspaceId: 'workspace-1', docId: 'doc-1', blob: Buffer.alloc(0), @@ -130,7 +150,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t => const documentJobCount = t.context.module.queue.count( 'backendRuntime.syncDocumentEmbedding' ); - await t.context.embeddingJob.onDocSnapshotUpdated({ + await t.context.embeddingProducer.onDocSnapshotUpdated({ workspaceId: 'workspace-1', docId: 'db$docProperties', blob: Buffer.alloc(0), @@ -140,7 +160,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t => documentJobCount ); - await t.context.embeddingJob.onDocSnapshotUpdated({ + await t.context.embeddingProducer.onDocSnapshotUpdated({ workspaceId: 'workspace-1', docId: 'workspace-1', blob: Buffer.alloc(0), @@ -155,7 +175,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t => reconcileDocuments: true, }); - await t.context.embeddingJob.prepareSelectedDocuments('workspace-1', [ + await t.context.embeddingService.prepareSelectedDocuments('workspace-1', [ 'doc-1', 'doc-1', ]); @@ -181,7 +201,9 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t => ] as const) { t.context.runtime.syncEmbeddingState.rejects(new Error(nativeError)); const error = await t.throwsAsync(() => - t.context.embeddingJob.prepareSelectedDocuments('workspace-1', ['doc-1']) + t.context.embeddingService.prepareSelectedDocuments('workspace-1', [ + 'doc-1', + ]) ); t.true(error instanceof expectedError); } @@ -189,7 +211,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t => await t.throwsAsync( () => - t.context.embeddingJob.prepareSelectedDocuments( + t.context.embeddingService.prepareSelectedDocuments( 'workspace-1', Array.from({ length: 65 }, (_, index) => `doc-${index}`) ), @@ -198,7 +220,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t => t.context.getSnapshot.resolves(null); await t.throwsAsync( () => - t.context.embeddingJob.prepareSelectedDocuments('workspace-1', [ + t.context.embeddingService.prepareSelectedDocuments('workspace-1', [ 'missing-doc', ]), { instanceOf: CopilotSelectedSourcesUnavailable } diff --git a/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts b/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts index 983427f048..e93852b8eb 100644 --- a/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts +++ b/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts @@ -11,7 +11,7 @@ const privateKey = generateKeyPairSync('ec', { }).privateKey.export({ format: 'pem', type: 'pkcs8' }) as string; const config = { crypto: { privateKey } } as Config; -test('backend-runtime provider starts once, runs migrations once, and reports health', async t => { +test('backend-runtime provider starts without migrations and exposes explicit migration', async t => { const provider = new BackendRuntimeProvider(config); const runtime = { start: Sinon.stub().resolves(), @@ -27,6 +27,7 @@ test('backend-runtime provider starts once, runs migrations once, and reports he await provider.start(); await provider.start(); + await provider.runMigrations(); await provider.onConfigChanged({ updates: { mailer: {} } }); await provider.onConfigChanged({ updates: { copilot: {} } }); await provider.onConfigChanged({ updates: { storages: {} } }); @@ -66,6 +67,106 @@ test('backend-runtime provider measures explicit typed methods', async t => { t.true(runtime.assertCopilotRoute.calledOnceWithExactly(routeInput)); }); +test('backend-runtime provider encodes recursive search contracts at the native boundary', async t => { + const provider = new BackendRuntimeProvider(config); + const runtime = { + searchAuthorized: Sinon.stub().resolves({ + ok: true, + value: { total: 0, nodes: [] }, + }), + aggregateAuthorized: Sinon.stub().resolves({ + ok: true, + value: { total: 0, buckets: [] }, + }), + }; + (provider as unknown as { runtime: typeof runtime }).runtime = runtime; + const query = { + type: 'boolean', + occur: 'must', + queries: [ + { type: 'exists', field: 'refDocId' }, + { + type: 'boost', + boost: 1.5, + query: { type: 'match', field: 'content', match: 'hello' }, + }, + ], + }; + + await provider.searchAuthorized('actor', 'workspace', { + table: 'block', + query, + options: { + fields: ['docId'], + highlights: [{ field: 'content', before: '', end: '' }], + pagination: { limit: 10, cursor: 'cursor' }, + }, + }); + await provider.aggregateAuthorized('actor', 'workspace', { + table: 'block', + query, + field: 'docId', + options: { + hits: { fields: ['content'] }, + pagination: { limit: 5, skip: 2 }, + }, + }); + + const search = runtime.searchAuthorized.firstCall.args[2]; + t.is(search.rootQuery, 0); + t.deepEqual(search.queries, [ + { + queryType: 'boolean', + field: undefined, + matchValue: undefined, + query: undefined, + queries: [1, 2], + occur: 'must', + boost: undefined, + }, + { + queryType: 'exists', + field: 'refDocId', + matchValue: undefined, + query: undefined, + queries: undefined, + occur: undefined, + boost: undefined, + }, + { + queryType: 'boost', + field: undefined, + matchValue: undefined, + query: 3, + queries: undefined, + occur: undefined, + boost: 1.5, + }, + { + queryType: 'match', + field: 'content', + matchValue: 'hello', + query: undefined, + queries: undefined, + occur: undefined, + boost: undefined, + }, + ]); + t.deepEqual(search.options, { + fields: ['docId'], + highlights: [{ field: 'content', before: '', end: '' }], + pagination: { limit: 10, cursor: 'cursor' }, + }); + t.deepEqual(runtime.aggregateAuthorized.firstCall.args[2].options, { + hits: { fields: ['content'], highlights: [], pagination: {} }, + pagination: { limit: 5, skip: 2 }, + }); + t.true(runtime.searchAuthorized.calledOnce); + t.true(runtime.searchAuthorized.calledWithMatch('actor', 'workspace')); + t.true(runtime.aggregateAuthorized.calledOnce); + t.true(runtime.aggregateAuthorized.calledWithMatch('actor', 'workspace')); +}); + test('backend-runtime provider aborts a stream handle that resolves after iterator cancellation', async t => { const provider = new BackendRuntimeProvider(config); const abort = Sinon.stub(); diff --git a/packages/backend/server/src/core/backend-runtime/index.ts b/packages/backend/server/src/core/backend-runtime/index.ts index f9a6ccbeb7..ab192f013d 100644 --- a/packages/backend/server/src/core/backend-runtime/index.ts +++ b/packages/backend/server/src/core/backend-runtime/index.ts @@ -2,6 +2,8 @@ import { Global, Module } from '@nestjs/common'; import { BackendRuntimeEmbeddingJob, + BackendRuntimeEmbeddingProducer, + BackendRuntimeEmbeddingService, BackendRuntimeHousekeepingJob, } from './job'; import { @@ -17,14 +19,30 @@ import { useValue: undefined, }, BackendRuntimeProvider, - BackendRuntimeEmbeddingJob, - BackendRuntimeHousekeepingJob, + BackendRuntimeEmbeddingService, ], - exports: [BackendRuntimeProvider, BackendRuntimeEmbeddingJob], + exports: [BackendRuntimeProvider, BackendRuntimeEmbeddingService], }) export class BackendRuntimeModule {} -export { BackendRuntimeEmbeddingJob } from './job'; +@Module({ + imports: [BackendRuntimeModule], + providers: [BackendRuntimeEmbeddingProducer], +}) +export class BackendRuntimeProducerModule {} + +@Module({ + imports: [BackendRuntimeModule], + providers: [BackendRuntimeEmbeddingJob, BackendRuntimeHousekeepingJob], +}) +export class BackendRuntimeWorkerModule {} + +export { + BackendRuntimeEmbeddingJob, + BackendRuntimeEmbeddingProducer, + BackendRuntimeEmbeddingService, + BackendRuntimeHousekeepingJob, +} from './job'; export { BACKEND_RUNTIME_CONFIG_PATHS, BackendRuntimeProvider, diff --git a/packages/backend/server/src/core/backend-runtime/job.ts b/packages/backend/server/src/core/backend-runtime/job.ts index 96408e7ccd..10a32d1571 100644 --- a/packages/backend/server/src/core/backend-runtime/job.ts +++ b/packages/backend/server/src/core/backend-runtime/job.ts @@ -22,7 +22,7 @@ const SELECTED_DOCUMENT_WAIT_MS = 90_000; declare global { interface Jobs { - 'nightly.cleanExpiredBackendRuntimeHousekeeping': {}; + 'backendRuntime.cleanExpiredHousekeeping': {}; 'backendRuntime.syncDocumentEmbedding': { workspaceId: string; docId: string; @@ -34,54 +34,13 @@ declare global { } @Injectable() -export class BackendRuntimeEmbeddingJob { +export class BackendRuntimeEmbeddingService { constructor( private readonly rt: BackendRuntimeProvider, - private readonly queue: JobQueue, private readonly models: Models ) {} - @OnEvent('doc.updated') - async onDocUpdated({ workspaceId, docId }: Events['doc.updated']) { - await this.queueDocument(workspaceId, docId); - } - - @OnEvent('doc.snapshot.updated') - async onDocSnapshotUpdated({ - workspaceId, - docId, - }: Events['doc.snapshot.updated']) { - if (workspaceId === docId) { - await this.queue.add( - 'backendRuntime.reconcileDocumentEmbeddings', - { workspaceId }, - { jobId: `reconcileDocumentEmbeddings/${workspaceId}` } - ); - return; - } - await this.queueDocument(workspaceId, docId); - } - - private async queueDocument(workspaceId: string, docId: string) { - if ( - workspaceId === docId || - docId.startsWith('db$') || - docId.startsWith('userdata$') - ) { - return; - } - await this.queue.add( - 'backendRuntime.syncDocumentEmbedding', - { workspaceId, docId }, - { jobId: `syncDocumentEmbedding/${workspaceId}/${docId}` } - ); - } - - @OnJob('backendRuntime.syncDocumentEmbedding') - async syncDocument({ - workspaceId, - docId, - }: Jobs['backendRuntime.syncDocumentEmbedding']) { + async syncDocument(workspaceId: string, docId: string) { await this.syncDocuments(workspaceId, [docId], true); } @@ -174,10 +133,7 @@ export class BackendRuntimeEmbeddingJob { }); } - @OnJob('backendRuntime.reconcileDocumentEmbeddings') - async reconcileDocuments({ - workspaceId, - }: Jobs['backendRuntime.reconcileDocumentEmbeddings']) { + async reconcileDocuments(workspaceId: string) { if (!(await this.rt.embeddingHealth()).enabled) return; await this.rt.syncEmbeddingState({ workspaceId, @@ -187,6 +143,67 @@ export class BackendRuntimeEmbeddingJob { } } +@Injectable() +export class BackendRuntimeEmbeddingProducer { + constructor(private readonly queue: JobQueue) {} + + @OnEvent('doc.updated') + async onDocUpdated({ workspaceId, docId }: Events['doc.updated']) { + await this.queueDocument(workspaceId, docId); + } + + @OnEvent('doc.snapshot.updated') + async onDocSnapshotUpdated({ + workspaceId, + docId, + }: Events['doc.snapshot.updated']) { + if (workspaceId === docId) { + await this.queue.add( + 'backendRuntime.reconcileDocumentEmbeddings', + { workspaceId }, + { jobId: `reconcileDocumentEmbeddings/${workspaceId}` } + ); + return; + } + await this.queueDocument(workspaceId, docId); + } + + private async queueDocument(workspaceId: string, docId: string) { + if ( + workspaceId === docId || + docId.startsWith('db$') || + docId.startsWith('userdata$') + ) { + return; + } + await this.queue.add( + 'backendRuntime.syncDocumentEmbedding', + { workspaceId, docId }, + { jobId: `syncDocumentEmbedding/${workspaceId}/${docId}` } + ); + } +} + +@Injectable() +export class BackendRuntimeEmbeddingJob { + constructor(private readonly service: BackendRuntimeEmbeddingService) {} + + @OnJob('backendRuntime.syncDocumentEmbedding') + async syncDocument({ + workspaceId, + docId, + }: Jobs['backendRuntime.syncDocumentEmbedding']) { + await this.service.syncDocument(workspaceId, docId); + } + + @OnJob('backendRuntime.reconcileDocumentEmbeddings') + async reconcileDocuments({ + workspaceId, + }: Jobs['backendRuntime.reconcileDocumentEmbeddings']) { + await this.service.reconcileDocuments(workspaceId); + } +} + @Injectable() export class BackendRuntimeHousekeepingJob { private readonly logger = new Logger(BackendRuntimeHousekeepingJob.name); @@ -199,7 +216,7 @@ export class BackendRuntimeHousekeepingJob { @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) async nightlyJob() { await this.queue.add( - 'nightly.cleanExpiredBackendRuntimeHousekeeping', + 'backendRuntime.cleanExpiredHousekeeping', {}, { jobId: 'nightly-backend-runtime-housekeeping', @@ -207,7 +224,7 @@ export class BackendRuntimeHousekeepingJob { ); } - @OnJob('nightly.cleanExpiredBackendRuntimeHousekeeping') + @OnJob('backendRuntime.cleanExpiredHousekeeping') async cleanExpiredRuntimeHousekeeping() { const states = await this.cleanBatches(() => this.rt.cleanupExpiredRuntimeStates(1000) diff --git a/packages/backend/server/src/core/backend-runtime/provider.ts b/packages/backend/server/src/core/backend-runtime/provider.ts index 88aa715079..42e4431d2f 100644 --- a/packages/backend/server/src/core/backend-runtime/provider.ts +++ b/packages/backend/server/src/core/backend-runtime/provider.ts @@ -35,6 +35,12 @@ import { type RuntimeWorkspaceArtifact, type SyncEmbeddingStateInput, } from '../../native'; +import { + type AggregateRequestInput, + encodeAggregateRequest, + encodeSearchRequest, + type SearchRequestInput, +} from './search'; type RuntimeInstance = InstanceType; @@ -299,11 +305,18 @@ export class BackendRuntimeProvider async start() { await this.runtime.start(); - await this.runMigrationsOnce(); const health = await this.runtime.health(); this.logger.log(`backend runtime started: db=${health.databaseConnected}`); } + /** + * Schema changes belong to the explicit predeploy path. Runtime startup only + * connects services and must not mutate the database schema. + */ + async runMigrations() { + await this.runMigrationsOnce(); + } + async stop() { await this.runtime.stop(); this.logger.log('backend runtime stopped'); @@ -315,6 +328,7 @@ export class BackendRuntimeProvider !updates.copilot && !updates.crypto && !updates.db && + !updates.indexer && !updates.storages ) { return; @@ -332,6 +346,74 @@ export class BackendRuntimeProvider ); } + async searchAuthorized( + actorUserId: string, + workspaceId: string, + request: SearchRequestInput + ) { + return await this.measured('searchAuthorized', runtime => + runtime.searchAuthorized( + actorUserId, + workspaceId, + encodeSearchRequest(request) + ) + ); + } + + async aggregateAuthorized( + actorUserId: string, + workspaceId: string, + request: AggregateRequestInput + ) { + return await this.measured('aggregateAuthorized', runtime => + runtime.aggregateAuthorized( + actorUserId, + workspaceId, + encodeAggregateRequest(request) + ) + ); + } + + async indexSearchDocument(workspaceId: string, docId: string) { + await this.measured('indexSearchDocument', runtime => + runtime.indexSearchDocument(workspaceId, docId) + ); + } + + async deleteSearchDocument(workspaceId: string, docId: string) { + await this.measured('deleteSearchDocument', runtime => + runtime.deleteSearchDocument(workspaceId, docId) + ); + } + + async reconcileSearchWorkspace(workspaceId: string) { + await this.measured('reconcileSearchWorkspace', runtime => + runtime.reconcileSearchWorkspace(workspaceId) + ); + } + + async deleteSearchWorkspace(workspaceId: string) { + await this.measured('deleteSearchWorkspace', runtime => + runtime.deleteSearchWorkspace(workspaceId) + ); + } + + async filterReadableDocs( + actorUserId: string, + workspaceId: string, + docIds: string[] + ) { + return await this.measured('filterReadableDocs', runtime => + runtime.filterReadableDocs(actorUserId, workspaceId, docIds) + ); + } + + async searchStatus() { + return await this.measured('searchStatus', runtime => + runtime.searchStatus() + ); + } + async embeddingQueueCounts() { return await this.measured('embeddingQueueCounts', runtime => runtime.embeddingQueueCounts() diff --git a/packages/backend/server/src/core/backend-runtime/search.ts b/packages/backend/server/src/core/backend-runtime/search.ts new file mode 100644 index 0000000000..d1d2a5aedb --- /dev/null +++ b/packages/backend/server/src/core/backend-runtime/search.ts @@ -0,0 +1,104 @@ +import type { + RuntimeAggregateRequest, + RuntimeSearchQuery, + RuntimeSearchRequest, +} from '../../native'; + +type SearchQueryInput = { + type: string; + field?: string; + match?: string; + query?: SearchQueryInput; + queries?: SearchQueryInput[]; + occur?: string; + boost?: number; +}; + +type SearchPaginationInput = { + limit?: number; + skip?: number; + cursor?: string; +}; + +type SearchHighlightInput = { + field: string; + before: string; + end: string; +}; + +type SearchOptionsInput = { + fields: string[]; + highlights?: SearchHighlightInput[]; + pagination?: SearchPaginationInput; +}; + +export type SearchRequestInput = { + table: 'doc' | 'block'; + query: SearchQueryInput; + options: SearchOptionsInput; +}; + +export type AggregateRequestInput = { + table: 'doc' | 'block'; + query: SearchQueryInput; + field: string; + options: { + hits: SearchOptionsInput; + pagination?: SearchPaginationInput; + }; +}; + +export function encodeSearchRequest( + request: SearchRequestInput +): RuntimeSearchRequest { + const { queries, rootQuery } = encodeQuery(request.query); + return { + table: request.table, + queries, + rootQuery, + options: encodeOptions(request.options), + }; +} + +export function encodeAggregateRequest( + request: AggregateRequestInput +): RuntimeAggregateRequest { + const { queries, rootQuery } = encodeQuery(request.query); + return { + table: request.table, + queries, + rootQuery, + field: request.field, + options: { + hits: encodeOptions(request.options.hits), + pagination: request.options.pagination ?? {}, + }, + }; +} + +function encodeOptions(options: SearchOptionsInput) { + return { + fields: options.fields, + highlights: options.highlights ?? [], + pagination: options.pagination ?? {}, + }; +} + +function encodeQuery(root: SearchQueryInput) { + const nodes: RuntimeSearchQuery[] = []; + const visit = (query: SearchQueryInput): number => { + const index = nodes.length; + nodes.push({ queryType: query.type }); + nodes[index] = { + queryType: query.type, + field: query.field, + matchValue: query.match, + query: query.query ? visit(query.query) : undefined, + queries: query.queries?.map(visit), + occur: query.occur, + boost: query.boost, + }; + return index; + }; + return { queries: nodes, rootQuery: visit(root) }; +} diff --git a/packages/backend/server/src/core/config/service.ts b/packages/backend/server/src/core/config/service.ts index 92ec7a9ad1..331f73589a 100644 --- a/packages/backend/server/src/core/config/service.ts +++ b/packages/backend/server/src/core/config/service.ts @@ -73,6 +73,25 @@ export class ServerService implements OnApplicationBootstrap { user: string, updates: Array<{ module: string; key: string; value: any }> ): Promise> { + const providerType = updates.find( + update => update.module === 'indexer' && update.key === 'provider.type' + ); + if (providerType?.value === 'embedded') { + updates = updates.filter(update => update !== providerType); + updates = [ + ...updates.filter( + update => !(update.module === 'indexer' && update.key === 'enabled') + ), + { module: 'indexer', key: 'enabled', value: false }, + ]; + } else if (providerType) { + updates = [ + ...updates.filter( + update => !(update.module === 'indexer' && update.key === 'enabled') + ), + { module: 'indexer', key: 'enabled', value: true }, + ]; + } const errors = this.validateConfig(updates); if (errors?.length) { diff --git a/packages/backend/server/src/core/doc-jobs/index.ts b/packages/backend/server/src/core/doc-jobs/index.ts new file mode 100644 index 0000000000..2398e15e47 --- /dev/null +++ b/packages/backend/server/src/core/doc-jobs/index.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; + +import { DocStorageModule, DocStorageWorkerModule } from '../doc'; +import { DocJobConsumer, DocJobScheduler } from './job'; + +@Module({ + imports: [DocStorageModule, DocStorageWorkerModule], + providers: [DocJobConsumer, DocJobScheduler], +}) +export class DocJobsModule {} diff --git a/packages/backend/server/src/core/doc-service/job.ts b/packages/backend/server/src/core/doc-jobs/job.ts similarity index 83% rename from packages/backend/server/src/core/doc-service/job.ts rename to packages/backend/server/src/core/doc-jobs/job.ts index 8326628eb4..5f170d3ddf 100644 --- a/packages/backend/server/src/core/doc-service/job.ts +++ b/packages/backend/server/src/core/doc-jobs/job.ts @@ -24,8 +24,8 @@ declare global { } @Injectable() -export class DocServiceCronJob { - private readonly logger = new Logger(DocServiceCronJob.name); +export class DocJobConsumer { + private readonly logger = new Logger(DocJobConsumer.name); constructor( private readonly workspace: PgWorkspaceDocStorageAdapter, @@ -40,7 +40,27 @@ export class DocServiceCronJob { workspaceId, docId, }: Jobs['doc.mergePendingDocUpdates']) { - await this.workspace.getDoc(workspaceId, docId); + const doc = await this.workspace.getDoc(workspaceId, docId); + if (doc) { + const snapshot = await this.models.doc.getSnapshot(workspaceId, docId, { + select: { updatedAt: true }, + }); + if (!snapshot) { + return JOB_SIGNAL.Done; + } + await this.job.add( + 'backendRuntime.projectWorkspaceDocBlobRefs', + { + workspaceId, + docId, + sourceRevision: snapshot.updatedAt.getTime(), + }, + { + jobId: `doc:blob-ref-projection:${workspaceId}:${docId}:${snapshot.updatedAt.getTime()}`, + priority: 100, + } + ); + } const updatesLeft = await this.models.doc.getUpdateCount( workspaceId, docId @@ -49,67 +69,12 @@ export class DocServiceCronJob { return updatesLeft > 100 ? JOB_SIGNAL.Repeat : JOB_SIGNAL.Done; } - @Cron(CronExpression.EVERY_30_SECONDS) - async schedule() { - const group = await this.models.doc.groupedUpdatesCount(); - - for (const update of group) { - const jobId = `doc:merge-pending-updates:${update.workspaceId}:${update.id}`; - - const job = await this.job.get(jobId, 'doc.mergePendingDocUpdates'); - - if (job && job.opts.priority !== 0 && update._count > 100) { - // reschedule long pending doc with highest priority, 0 is the highest priority - await this.job.remove(jobId, 'doc.mergePendingDocUpdates'); - } - - await this.job.add( - 'doc.mergePendingDocUpdates', - { - workspaceId: update.workspaceId, - docId: update.id, - }, - { - jobId: `doc:merge-pending-updates:${update.workspaceId}:${update.id}`, - priority: update._count > 100 ? 0 : 100, - delay: 0, - } - ); - } - } - @OnJob('doc.recordPendingDocUpdatesCount') async recordPendingDocUpdatesCount() { const count = await this.prisma.update.count(); metrics.doc.gauge('pending_updates').record(count); } - @Cron(CronExpression.EVERY_30_SECONDS) - async scheduleRecordPendingDocUpdatesCount() { - await this.job.add( - 'doc.recordPendingDocUpdatesCount', - {}, - { - // make sure only one job is running at a time - delay: 30 * 1000, - jobId: 'doc:record-pending-updates-count', - } - ); - } - - @Cron(CronExpression.EVERY_30_SECONDS) - async scheduleFindEmptySummaryDocs() { - await this.job.add( - 'doc.findEmptySummaryDocs', - {}, - { - // make sure only one job is running at a time - delay: 30 * 1000, - jobId: 'findEmptySummaryDocs', - } - ); - } - @OnJob('doc.findEmptySummaryDocs') async findEmptySummaryDocs(payload: Jobs['doc.findEmptySummaryDocs']) { const startSid = payload.lastFixedWorkspaceSid ?? 0; @@ -167,3 +132,62 @@ export class DocServiceCronJob { return; } } + +@Injectable() +export class DocJobScheduler { + constructor( + private readonly job: JobQueue, + private readonly models: Models + ) {} + + @Cron(CronExpression.EVERY_30_SECONDS) + async schedule() { + const group = await this.models.doc.groupedUpdatesCount(); + + for (const update of group) { + const jobId = `doc:merge-pending-updates:${update.workspaceId}:${update.id}`; + const job = await this.job.get(jobId, 'doc.mergePendingDocUpdates'); + + if (job && job.opts.priority !== 0 && update._count > 100) { + await this.job.remove(jobId, 'doc.mergePendingDocUpdates'); + } + + await this.job.add( + 'doc.mergePendingDocUpdates', + { + workspaceId: update.workspaceId, + docId: update.id, + }, + { + jobId, + priority: update._count > 100 ? 0 : 100, + delay: 0, + } + ); + } + } + + @Cron(CronExpression.EVERY_30_SECONDS) + async scheduleRecordPendingDocUpdatesCount() { + await this.job.add( + 'doc.recordPendingDocUpdatesCount', + {}, + { + delay: 30 * 1000, + jobId: 'doc:record-pending-updates-count', + } + ); + } + + @Cron(CronExpression.EVERY_30_SECONDS) + async scheduleFindEmptySummaryDocs() { + await this.job.add( + 'doc.findEmptySummaryDocs', + {}, + { + delay: 30 * 1000, + jobId: 'findEmptySummaryDocs', + } + ); + } +} diff --git a/packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts b/packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts index df4473f1f5..9e59e77de7 100644 --- a/packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts +++ b/packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts @@ -7,8 +7,9 @@ import { Doc as YDoc } from 'yjs'; import { MockEventBus } from '../../../__tests__/mocks'; import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; -import { ConfigFactory, EventBus } from '../../../base'; -import { Flavor } from '../../../env'; +import { buildAppModule } from '../../../app.module'; +import { EventBus } from '../../../base'; +import { Env, Flavor } from '../../../env'; import { Models } from '../../../models'; import { DocReader, PgWorkspaceDocStorageAdapter } from '../../doc'; @@ -23,9 +24,10 @@ interface Context { const test = ava as TestFn; test.before(async t => { - // @ts-expect-error testing - env.FLAVOR = Flavor.Renderer; + const rendererEnv = new Env(); + rendererEnv.FLAVOR = Flavor.Renderer; const app = await createTestingApp({ + imports: [buildAppModule(rendererEnv)], tapModule: m => m.overrideProvider(EventBus).useClass(MockEventBus), }); @@ -39,11 +41,6 @@ let user: User; let workspace: Workspace; test.beforeEach(async t => { - t.context.app.get(ConfigFactory).override({ - docService: { - endpoint: t.context.app.url(), - }, - }); await t.context.app.initTestingDB(); user = await t.context.models.user.create({ email: 'test@affine.pro', @@ -59,9 +56,7 @@ test.afterEach.always(t => { t.context.recordDocView?.restore(); }); -test.after.always(async t => { - await t.context.app.close(); -}); +test.after.always(async t => t.context.app.close()); async function createDoc( adapter: PgWorkspaceDocStorageAdapter, diff --git a/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts b/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts deleted file mode 100644 index c8163384d8..0000000000 --- a/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts +++ /dev/null @@ -1,450 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { mock } from 'node:test'; - -import { User, Workspace } from '@prisma/client'; -import ava, { TestFn } from 'ava'; - -import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; -import { CryptoHelper } from '../../../base'; -import { Models } from '../../../models'; -import { DatabaseDocReader } from '../../doc'; - -const test = ava as TestFn<{ - models: Models; - app: TestingApp; - crypto: CryptoHelper; - databaseDocReader: DatabaseDocReader; -}>; - -test.before(async t => { - const app = await createTestingApp(); - - t.context.models = app.get(Models); - t.context.crypto = app.get(CryptoHelper); - t.context.app = app; - t.context.databaseDocReader = app.get(DatabaseDocReader); -}); - -let user: User; -let workspace: Workspace; - -test.beforeEach(async t => { - await t.context.app.initTestingDB(); - user = await t.context.models.user.create({ - email: 'test@affine.pro', - }); - workspace = await t.context.models.workspace.create(user.id); -}); - -test.afterEach.always(async () => { - mock.reset(); -}); - -test.after.always(async t => { - await t.context.app.close(); -}); - -test('should forbid access to rpc api without access token', async t => { - const { app } = t.context; - - await app - .GET('/rpc/workspaces/123/docs/123') - .expect({ - status: 403, - code: 'Forbidden', - type: 'NO_PERMISSION', - name: 'ACCESS_DENIED', - message: 'Invalid internal request', - }) - .expect(403); - t.pass(); -}); - -test('should forbid access to rpc api with invalid access token', async t => { - const { app } = t.context; - - await app - .GET('/rpc/workspaces/123/docs/123') - .set('x-access-token', 'invalid,wrong-signature') - .expect({ - status: 403, - code: 'Forbidden', - type: 'NO_PERMISSION', - name: 'ACCESS_DENIED', - message: 'Invalid internal request', - }) - .expect(403); - t.pass(); -}); - -test('should forbid replayed internal access token', async t => { - const { app } = t.context; - - const workspaceId = '123'; - const docId = '123'; - const path = `/rpc/workspaces/${workspaceId}/docs/${docId}`; - const token = t.context.crypto.signInternalAccessToken({ - method: 'GET', - path, - nonce: `nonce-${randomUUID()}`, - }); - - await app.GET(path).set('x-access-token', token).expect(404); - - await app - .GET(path) - .set('x-access-token', token) - .expect({ - status: 403, - code: 'Forbidden', - type: 'NO_PERMISSION', - name: 'ACCESS_DENIED', - message: 'Invalid internal request', - }) - .expect(403); - t.pass(); -}); - -test('should forbid internal access token when method mismatched', async t => { - const { app } = t.context; - - const workspaceId = '123'; - const docId = '123'; - const path = `/rpc/workspaces/${workspaceId}/docs/${docId}/diff`; - await app - .POST(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect({ - status: 403, - code: 'Forbidden', - type: 'NO_PERMISSION', - name: 'ACCESS_DENIED', - message: 'Invalid internal request', - }) - .expect(403); - t.pass(); -}); - -test('should forbid internal access token when path mismatched', async t => { - const { app } = t.context; - - const workspaceId = '123'; - const docId = '123'; - const wrongPath = `/rpc/workspaces/${workspaceId}/docs/${docId}`; - const path = `/rpc/workspaces/${workspaceId}/docs/${docId}/content`; - await app - .GET(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ - method: 'GET', - path: wrongPath, - }) - ) - .expect({ - status: 403, - code: 'Forbidden', - type: 'NO_PERMISSION', - name: 'ACCESS_DENIED', - message: 'Invalid internal request', - }) - .expect(403); - t.pass(); -}); - -test('should forbid internal access token when expired', async t => { - const { app } = t.context; - - const workspaceId = '123'; - const docId = '123'; - const path = `/rpc/workspaces/${workspaceId}/docs/${docId}`; - await app - .GET(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ - method: 'GET', - path, - now: Date.now() - 10 * 60 * 1000, - nonce: `nonce-${randomUUID()}`, - }) - ) - .expect({ - status: 403, - code: 'Forbidden', - type: 'NO_PERMISSION', - name: 'ACCESS_DENIED', - message: 'Invalid internal request', - }) - .expect(403); - t.pass(); -}); - -test('should 404 when doc not found', async t => { - const { app } = t.context; - - const workspaceId = '123'; - const docId = '123'; - const path = `/rpc/workspaces/${workspaceId}/docs/${docId}`; - await app - .GET(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect({ - status: 404, - code: 'Not Found', - type: 'RESOURCE_NOT_FOUND', - name: 'NOT_FOUND', - message: 'Doc not found', - }) - .expect(404); - t.pass(); -}); - -test('should return doc when found', async t => { - const { app } = t.context; - - const docId = randomUUID(); - const timestamp = Date.now(); - await t.context.models.doc.createUpdates([ - { - spaceId: workspace.id, - docId, - blob: Buffer.from('blob1 data'), - timestamp, - editorId: user.id, - }, - ]); - - const path = `/rpc/workspaces/${workspace.id}/docs/${docId}`; - const res = await app - .GET(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .set('x-cloud-trace-context', 'test-trace-id/span-id') - .expect(200) - .expect('x-request-id', 'test-trace-id') - .expect('Content-Type', 'application/octet-stream'); - const bin = res.body as Buffer; - t.is(bin.toString(), 'blob1 data'); - t.is(res.headers['x-doc-timestamp'], timestamp.toString()); - t.is(res.headers['x-doc-editor-id'], user.id); -}); - -test('should 404 when doc diff not found', async t => { - const { app } = t.context; - - const workspaceId = '123'; - const docId = '123'; - const path = `/rpc/workspaces/${workspaceId}/docs/${docId}/diff`; - await app - .POST(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'POST', path }) - ) - .expect({ - status: 404, - code: 'Not Found', - type: 'RESOURCE_NOT_FOUND', - name: 'NOT_FOUND', - message: 'Doc not found', - }) - .expect(404); - t.pass(); -}); - -test('should 404 when doc content not found', async t => { - const { app } = t.context; - - const workspaceId = '123'; - const docId = '123'; - const path = `/rpc/workspaces/${workspaceId}/docs/${docId}/content`; - await app - .GET(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect({ - status: 404, - code: 'Not Found', - type: 'RESOURCE_NOT_FOUND', - name: 'NOT_FOUND', - message: 'Doc not found', - }) - .expect(404); - t.pass(); -}); - -test('should get doc content in json format', async t => { - const { app } = t.context; - mock.method(t.context.databaseDocReader, 'getDocContent', async () => { - return { - title: 'test title', - summary: 'test summary', - }; - }); - - const docId = randomUUID(); - const path = `/rpc/workspaces/${workspace.id}/docs/${docId}/content`; - await app - .GET(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect('Content-Type', 'application/json; charset=utf-8') - .expect({ - title: 'test title', - summary: 'test summary', - }) - .expect(200); - - await app - .GET(`${path}?full=false`) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect('Content-Type', 'application/json; charset=utf-8') - .expect({ - title: 'test title', - summary: 'test summary', - }) - .expect(200); - t.pass(); -}); - -test('should get full doc content in json format', async t => { - const { app } = t.context; - mock.method(t.context.databaseDocReader, 'getFullDocContent', async () => { - return { - title: 'test title', - summary: 'test summary full', - }; - }); - - const docId = randomUUID(); - const path = `/rpc/workspaces/${workspace.id}/docs/${docId}/content`; - await app - .GET(`${path}?full=true`) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect('Content-Type', 'application/json; charset=utf-8') - .expect({ - title: 'test title', - summary: 'test summary full', - }) - .expect(200); - t.pass(); -}); - -test('should 404 when workspace content not found', async t => { - const { app } = t.context; - - const workspaceId = '123'; - const path = `/rpc/workspaces/${workspaceId}/content`; - await app - .GET(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect({ - status: 404, - code: 'Not Found', - type: 'RESOURCE_NOT_FOUND', - name: 'NOT_FOUND', - message: 'Workspace not found', - }) - .expect(404); - t.pass(); -}); - -test('should get workspace content in json format', async t => { - const { app } = t.context; - mock.method(t.context.databaseDocReader, 'getWorkspaceContent', async () => { - return { - name: 'test name', - avatarKey: 'avatar key', - }; - }); - - const workspaceId = randomUUID(); - const path = `/rpc/workspaces/${workspaceId}/content`; - await app - .GET(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect(200) - .expect({ - name: 'test name', - avatarKey: 'avatar key', - }); - t.pass(); -}); - -test('should get doc markdown in json format', async t => { - const { app } = t.context; - mock.method(t.context.databaseDocReader, 'getDocMarkdown', async () => { - return { - title: 'test title', - markdown: 'test markdown', - knownUnsupportedBlocks: [], - unknownBlocks: [], - }; - }); - - const docId = randomUUID(); - const path = `/rpc/workspaces/${workspace.id}/docs/${docId}/markdown`; - await app - .GET(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect('Content-Type', 'application/json; charset=utf-8') - .expect(200) - .expect({ - title: 'test title', - markdown: 'test markdown', - knownUnsupportedBlocks: [], - unknownBlocks: [], - }); - t.pass(); -}); - -test('should 404 when doc markdown not found', async t => { - const { app } = t.context; - - const workspaceId = '123'; - const docId = '123'; - const path = `/rpc/workspaces/${workspaceId}/docs/${docId}/markdown`; - await app - .GET(path) - .set( - 'x-access-token', - t.context.crypto.signInternalAccessToken({ method: 'GET', path }) - ) - .expect({ - status: 404, - code: 'Not Found', - type: 'RESOURCE_NOT_FOUND', - name: 'NOT_FOUND', - message: 'Doc not found', - }) - .expect(404); - t.pass(); -}); diff --git a/packages/backend/server/src/core/doc-service/config.ts b/packages/backend/server/src/core/doc-service/config.ts deleted file mode 100644 index cbf3f733d0..0000000000 --- a/packages/backend/server/src/core/doc-service/config.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { defineModuleConfig } from '../../base'; - -declare global { - interface AppConfigSchema { - docService: { - endpoint: string; - }; - } -} - -defineModuleConfig('docService', { - endpoint: { - desc: 'The endpoint of the doc service.', - default: '', - env: 'DOC_SERVICE_ENDPOINT', - }, -}); diff --git a/packages/backend/server/src/core/doc-service/controller.ts b/packages/backend/server/src/core/doc-service/controller.ts deleted file mode 100644 index c1adc317fb..0000000000 --- a/packages/backend/server/src/core/doc-service/controller.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { - Controller, - Get, - Logger, - Param, - Post, - Query, - RawBody, - Res, -} from '@nestjs/common'; -import type { Response } from 'express'; - -import { NotFound, SkipThrottle } from '../../base'; -import { Internal } from '../auth'; -import { DatabaseDocReader } from '../doc'; - -@Controller('/rpc') -export class DocRpcController { - private readonly logger = new Logger(DocRpcController.name); - - constructor(private readonly docReader: DatabaseDocReader) {} - - @SkipThrottle() - @Internal() - @Get('/workspaces/:workspaceId/docs/:docId') - async getDoc( - @Param('workspaceId') workspaceId: string, - @Param('docId') docId: string, - @Res() res: Response - ) { - const doc = await this.docReader.getDoc(workspaceId, docId); - if (!doc) { - throw new NotFound('Doc not found'); - } - this.logger.debug( - `get doc ${docId} from workspace ${workspaceId}, size: ${doc.bin.length}` - ); - res.setHeader('x-doc-timestamp', doc.timestamp.toString()); - if (doc.editor) { - res.setHeader('x-doc-editor-id', doc.editor); - } - res.send(doc.bin); - } - - @SkipThrottle() - @Internal() - @Get('/workspaces/:workspaceId/docs/:docId/markdown') - async getDocMarkdown( - @Param('workspaceId') workspaceId: string, - @Param('docId') docId: string, - @Query('aiEditable') aiEditable?: string - ) { - const result = await this.docReader.getDocMarkdown( - workspaceId, - docId, - aiEditable === 'true' - ); - if (!result) { - throw new NotFound('Doc not found'); - } - return result; - } - - @SkipThrottle() - @Internal() - @Post('/workspaces/:workspaceId/docs/:docId/diff') - async getDocDiff( - @Param('workspaceId') workspaceId: string, - @Param('docId') docId: string, - @RawBody() stateVector: Buffer | undefined, - @Res() res: Response - ) { - const diff = await this.docReader.getDocDiff( - workspaceId, - docId, - stateVector - ); - if (!diff) { - throw new NotFound('Doc not found'); - } - this.logger.debug( - `get doc diff ${docId} from workspace ${workspaceId}, missing size: ${diff.missing.length}, old state size: ${stateVector?.length}, new state size: ${diff.state.length}` - ); - res.setHeader('x-doc-timestamp', diff.timestamp.toString()); - res.setHeader('x-doc-missing-offset', `0,${diff.missing.length}`); - const stateOffset = diff.missing.length; - res.setHeader( - 'x-doc-state-offset', - `${stateOffset},${stateOffset + diff.state.length}` - ); - res.send(Buffer.concat([diff.missing, diff.state])); - } - - @SkipThrottle() - @Internal() - @Get('/workspaces/:workspaceId/docs/:docId/canvas') - async getDocCanvas( - @Param('workspaceId') workspaceId: string, - @Param('docId') docId: string - ) { - const projection = await this.docReader.getDocCanvas(workspaceId, docId); - if (!projection) { - throw new NotFound('Doc not found'); - } - return projection; - } - - @SkipThrottle() - @Internal() - @Get('/workspaces/:workspaceId/docs/:docId/content') - async getDocContent( - @Param('workspaceId') workspaceId: string, - @Param('docId') docId: string, - @Query('full') fullContent?: string - ) { - const content = - fullContent === 'true' - ? await this.docReader.getFullDocContent(workspaceId, docId) - : await this.docReader.getDocContent(workspaceId, docId); - if (!content) { - throw new NotFound('Doc not found'); - } - this.logger.debug(`get doc content ${docId} from workspace ${workspaceId}`); - return content; - } - - @SkipThrottle() - @Internal() - @Get('/workspaces/:workspaceId/content') - async getWorkspaceContent(@Param('workspaceId') workspaceId: string) { - const content = await this.docReader.getWorkspaceContent(workspaceId); - if (!content) { - throw new NotFound('Workspace not found'); - } - this.logger.debug(`get workspace content ${workspaceId}`); - return content; - } -} diff --git a/packages/backend/server/src/core/doc-service/index.ts b/packages/backend/server/src/core/doc-service/index.ts deleted file mode 100644 index 65b95e8604..0000000000 --- a/packages/backend/server/src/core/doc-service/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -import './config'; - -import { Module } from '@nestjs/common'; - -import { DocStorageModule } from '../doc'; -import { DocRpcController } from './controller'; -import { DocServiceCronJob } from './job'; - -@Module({ - imports: [DocStorageModule], - providers: [DocServiceCronJob], - controllers: [DocRpcController], -}) -export class DocServiceModule {} diff --git a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md deleted file mode 100644 index 898ac18e77..0000000000 --- a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md +++ /dev/null @@ -1,86 +0,0 @@ -# Snapshot report for `src/core/doc/__tests__/reader-from-rpc.spec.ts` - -The actual snapshot is saved in `reader-from-rpc.spec.ts.snap`. - -Generated by [AVA](https://avajs.dev). - -## should return doc markdown success - -> Snapshot 1 - - { - knownUnsupportedBlocks: [ - 'RX4CG2zsBk:affine:note', - 'S1mkc8zUoU:affine:note', - 'yGlBdshAqN:affine:note', - '6lDiuDqZGL:affine:note', - 'cauvaHOQmh:affine:note', - '2jwCeO8Yot:affine:note', - 'c9MF_JiRgx:affine:note', - '6x7ALjUDjj:affine:surface', - ], - markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊ - ␊ - ␊ - ␊ - # You own your data, with no compromises␊ - ␊ - ## Local-first & Real-time collaborative␊ - ␊ - We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊ - ␊ - AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊ - ␊ - ␊ - ␊ - ### Blocks that assemble your next docs, tasks kanban or whiteboard␊ - ␊ - There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊ - ␊ - We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊ - ␊ - If you want to learn more about the product design of AFFiNE, here goes the concepts:␊ - ␊ - To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊ - ␊ - ## A true canvas for blocks in any form␊ - ␊ - [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊ - ␊ - ␊ - ␊ - "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊ - ␊ - * Quip & Notion with their great concept of "everything is a block"␊ - * Trello with their Kanban␊ - * Airtable & Miro with their no-code programable datasheets␊ - * Miro & Whimiscal with their edgeless visual whiteboard␊ - * Remnote & Capacities with their object-based tag system␊ - For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊ - ␊ - ## Self Host␊ - ␊ - Self host AFFiNE␊ - ␊ - ␊ - ### Learning From␊ - ||Title|Tag|␊ - |---|---|---|␊ - |Affine Development|Affine Development|AFFiNE|␊ - |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc|Developers|␊ - |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"|Reference|␊ - |Trello with their Kanban|Trello with their Kanban|Reference|␊ - |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets|Reference|␊ - |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|Reference|␊ - |Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊ - ␊ - ## Affine Development␊ - ␊ - For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊ - ␊ - ␊ - ␊ - `, - title: 'Write, Draw, Plan all at Once.', - unknownBlocks: [], - } diff --git a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap deleted file mode 100644 index dc0fdb3923..0000000000 Binary files a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap and /dev/null differ diff --git a/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts b/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts deleted file mode 100644 index 2dc79a81f8..0000000000 --- a/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts +++ /dev/null @@ -1,432 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { mock } from 'node:test'; - -import { User, Workspace } from '@prisma/client'; -import ava, { TestFn } from 'ava'; -import { applyUpdate, Doc as YDoc } from 'yjs'; - -import { createModule } from '../../../__tests__/create-module'; -import { Mockers } from '../../../__tests__/mocks'; -import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; -import { UserFriendlyError } from '../../../base'; -import { ConfigFactory } from '../../../base/config'; -import { Models } from '../../../models'; -import { - DatabaseDocReader, - DocReader, - DocStorageModule, - PgWorkspaceDocStorageAdapter, -} from '../index'; -import { RpcDocReader } from '../reader'; - -const module = await createModule({ - imports: [DocStorageModule], -}); - -const test = ava as TestFn<{ - models: Models; - app: TestingApp; - docApp: TestingApp; - docReader: DocReader; - databaseDocReader: DatabaseDocReader; - adapter: PgWorkspaceDocStorageAdapter; - config: ConfigFactory; -}>; - -test.before(async t => { - // test key - process.env.AFFINE_PRIVATE_KEY = `-----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgS3IAkshQuSmFWGpe -rGTg2vwaC3LdcvBQlYHHMBYJZMyhRANCAAQXdT/TAh4neNEpd4UqpDIEqWv0XvFo -BRJxGsC5I/fetqObdx1+KEjcm8zFU2xLaUTw9IZCu8OslloOjQv4ur0a ------END PRIVATE KEY-----`; - // @ts-expect-error testing - env.FLAVOR = 'renderer'; - const notDocApp = await createTestingApp(); - // @ts-expect-error testing - env.FLAVOR = 'doc'; - const docApp = await createTestingApp(); - - t.context.models = notDocApp.get(Models); - t.context.docReader = notDocApp.get(DocReader); - t.context.databaseDocReader = docApp.get(DatabaseDocReader); - t.context.adapter = docApp.get(PgWorkspaceDocStorageAdapter); - t.context.config = notDocApp.get(ConfigFactory); - t.context.app = notDocApp; - t.context.docApp = docApp; -}); - -let user: User; -let workspace: Workspace; - -test.beforeEach(async t => { - t.context.config.override({ - docService: { - endpoint: t.context.docApp.url(), - }, - }); - await t.context.app.initTestingDB(); - user = await t.context.models.user.create({ - email: 'test@affine.pro', - }); - workspace = await t.context.models.workspace.create(user.id); -}); - -test.afterEach.always(() => { - mock.reset(); -}); - -test.after.always(async t => { - await t.context.app.close(); - await t.context.docApp.close(); - await module.close(); -}); - -test('should be rpc reader', async t => { - const { docReader } = t.context; - t.true(docReader instanceof RpcDocReader); -}); - -test('should return null when doc not found', async t => { - const { docReader } = t.context; - const docId = randomUUID(); - const doc = await docReader.getDoc(workspace.id, docId); - t.is(doc, null); -}); - -test('should throw error when doc service internal error', async t => { - const { docReader, adapter } = t.context; - const docId = randomUUID(); - mock.method(adapter, 'getDoc', async () => { - throw new Error('mock doc service internal error'); - }); - mock.method(adapter, 'getDocBinNative', async () => { - throw new Error('mock doc service internal error'); - }); - let err = await t.throwsAsync(docReader.getDoc(workspace.id, docId), { - instanceOf: UserFriendlyError, - message: 'An internal error occurred.', - name: 'internal_server_error', - }); - t.is(err.type, 'internal_server_error'); - t.is(err.status, 500); - - err = await t.throwsAsync(docReader.getDocDiff(workspace.id, docId), { - instanceOf: UserFriendlyError, - message: 'An internal error occurred.', - name: 'internal_server_error', - }); - t.is(err.type, 'internal_server_error'); - t.is(err.status, 500); - - err = await t.throwsAsync(docReader.getDocContent(workspace.id, docId), { - instanceOf: UserFriendlyError, - message: 'An internal error occurred.', - name: 'internal_server_error', - }); - t.is(err.type, 'internal_server_error'); - t.is(err.status, 500); - - err = await t.throwsAsync(docReader.getWorkspaceContent(workspace.id), { - instanceOf: UserFriendlyError, - message: 'An internal error occurred.', - name: 'internal_server_error', - }); - t.is(err.type, 'internal_server_error'); - t.is(err.status, 500); -}); - -test('should fallback to database doc reader when endpoint network error', async t => { - const { docReader } = t.context; - t.context.config.override({ - docService: { - endpoint: 'http://localhost:13010', - }, - }); - const docId = randomUUID(); - const timestamp = Date.now(); - await t.context.models.doc.createUpdates([ - { - spaceId: workspace.id, - docId, - blob: Buffer.from('blob1 data'), - timestamp, - editorId: user.id, - }, - ]); - - const doc = await docReader.getDoc(workspace.id, docId); - t.truthy(doc); - t.is(Buffer.from(doc!.bin).toString('utf8'), 'blob1 data'); - t.is(doc!.timestamp, timestamp); - t.is(doc!.editor, user.id); -}); - -test('should return doc when found', async t => { - const { docReader } = t.context; - - const docId = randomUUID(); - const timestamp = Date.now(); - await t.context.models.doc.createUpdates([ - { - spaceId: workspace.id, - docId, - blob: Buffer.from('blob1 data'), - timestamp, - editorId: user.id, - }, - ]); - - const doc = await docReader.getDoc(workspace.id, docId); - t.truthy(doc); - t.is(doc!.bin.toString(), 'blob1 data'); - t.is(doc!.timestamp, timestamp); - t.is(doc!.editor, user.id); -}); - -test('should return doc diff', async t => { - const { docReader } = t.context; - const docId = randomUUID(); - const timestamp = Date.now(); - let updates: Buffer[] = []; - const doc1 = new YDoc(); - doc1.on('update', data => { - updates.push(Buffer.from(data)); - }); - - const text = doc1.getText('content'); - text.insert(0, 'hello'); - text.insert(5, 'world'); - text.insert(5, ' '); - text.insert(11, '!'); - - await t.context.models.doc.createUpdates( - updates.map((update, index) => ({ - spaceId: workspace.id, - docId, - blob: update, - timestamp: timestamp + index, - editorId: user.id, - })) - ); - // clear updates - updates.splice(0, updates.length); - - const doc2 = new YDoc(); - const diff = await docReader.getDocDiff(workspace.id, docId); - t.truthy(diff); - t.truthy(diff!.missing); - t.truthy(diff!.state); - applyUpdate(doc2, diff!.missing); - t.is(doc2.getText('content').toString(), 'hello world!'); - - // nothing changed - const diff2 = await docReader.getDocDiff(workspace.id, docId, diff!.state); - t.truthy(diff2); - t.truthy(diff2!.missing); - t.deepEqual(diff2!.missing, new Uint8Array([0, 0])); - t.truthy(diff2!.state); - applyUpdate(doc2, diff2!.missing); - t.is(doc2.getText('content').toString(), 'hello world!'); - - // add new content on doc1 - text.insert(12, '@'); - await t.context.models.doc.createUpdates( - updates.map((update, index) => ({ - spaceId: workspace.id, - docId, - blob: update, - timestamp: Date.now() + index + 1000, - editorId: user.id, - })) - ); - - const diff3 = await docReader.getDocDiff(workspace.id, docId, diff2!.state); - t.truthy(diff3); - t.truthy(diff3!.missing); - t.truthy(diff3!.state); - applyUpdate(doc2, diff3!.missing); - t.is(doc2.getText('content').toString(), 'hello world!@'); -}); - -test('should get doc diff fallback to database doc reader when endpoint network error', async t => { - const { docReader } = t.context; - t.context.config.override({ - docService: { - endpoint: 'http://localhost:13010', - }, - }); - const docId = randomUUID(); - const timestamp = Date.now(); - let updates: Buffer[] = []; - const doc1 = new YDoc(); - doc1.on('update', data => { - updates.push(Buffer.from(data)); - }); - - const text = doc1.getText('content'); - text.insert(0, 'hello'); - text.insert(5, 'world'); - text.insert(5, ' '); - text.insert(11, '!'); - - await t.context.models.doc.createUpdates( - updates.map((update, index) => ({ - spaceId: workspace.id, - docId, - blob: update, - timestamp: timestamp + index, - editorId: user.id, - })) - ); - // clear updates - updates.splice(0, updates.length); - - const doc2 = new YDoc(); - const diff = await docReader.getDocDiff(workspace.id, docId); - t.truthy(diff); - t.truthy(diff!.missing); - t.truthy(diff!.state); - applyUpdate(doc2, diff!.missing); - t.is(doc2.getText('content').toString(), 'hello world!'); -}); - -test('should get doc content', async t => { - const docId = randomUUID(); - const { docReader, databaseDocReader } = t.context; - mock.method(databaseDocReader, 'getDocContent', async () => { - return { - title: 'test title', - summary: 'test summary', - }; - }); - const docContent = await docReader.getDocContent(workspace.id, docId); - t.deepEqual(docContent, { - title: 'test title', - summary: 'test summary', - }); -}); - -test('should return null when doc content not exists', async t => { - const docId = randomUUID(); - const { docReader, adapter } = t.context; - - const doc = new YDoc(); - const text = doc.getText('content'); - const updates: Buffer[] = []; - - doc.on('update', update => { - updates.push(Buffer.from(update)); - }); - - text.insert(0, 'hello'); - text.insert(5, 'world'); - text.insert(5, ' '); - - await adapter.pushDocUpdates(workspace.id, docId, updates, user.id); - - const docContent = await docReader.getDocContent(workspace.id, docId); - t.is(docContent, null); - - const notExists = await docReader.getDocContent(workspace.id, randomUUID()); - t.is(notExists, null); -}); - -test('should get workspace content from doc service rpc', async t => { - const { docReader, databaseDocReader } = t.context; - const track = mock.method( - databaseDocReader, - 'getWorkspaceContent', - async () => { - return { - id: workspace.id, - name: 'test name', - avatarKey: '', - }; - } - ); - - const workspaceContent = await docReader.getWorkspaceContent(workspace.id); - t.is(track.mock.callCount(), 1); - t.deepEqual(workspaceContent, { - id: workspace.id, - name: 'test name', - avatarKey: '', - }); -}); - -test('should return null when workspace bin meta not exists', async t => { - const { docReader, adapter } = t.context; - const doc = new YDoc(); - const text = doc.getText('content'); - const updates: Buffer[] = []; - - doc.on('update', update => { - updates.push(Buffer.from(update)); - }); - - text.insert(0, 'hello'); - text.insert(5, 'world'); - text.insert(5, ' '); - - await adapter.pushDocUpdates(workspace.id, workspace.id, updates, user.id); - - const workspaceContent = await docReader.getWorkspaceContent(workspace.id); - t.is(workspaceContent, null); - - // workspace not exists - const notExists = await docReader.getWorkspaceContent(randomUUID()); - t.is(notExists, null); -}); - -test('should return doc markdown success', async t => { - const { docReader } = t.context; - - const workspace = await module.create(Mockers.Workspace, { - owner: user, - name: '', - }); - - const docSnapshot = await module.create(Mockers.DocSnapshot, { - workspaceId: workspace.id, - user, - }); - - const result = await docReader.getDocMarkdown( - workspace.id, - docSnapshot.id, - false - ); - if (result) { - const { revision, ...markdown } = result; - t.truthy(revision); - t.snapshot(markdown); - } - const canvas = await docReader.getDocCanvas(workspace.id, docSnapshot.id); - t.is(canvas?.version, 1); - t.is(canvas?.docId, docSnapshot.id); - t.truthy(canvas?.revision); - t.deepEqual(canvas?.counts, { - connector: 6, - group: 6, - shape: 7, - text: 7, - }); -}); - -test('should read markdown return null when doc not exists', async t => { - const { docReader } = t.context; - - const workspace = await module.create(Mockers.Workspace, { - owner: user, - name: '', - }); - - const result = await docReader.getDocMarkdown( - workspace.id, - randomUUID(), - false - ); - t.is(result, null); - t.is(await docReader.getDocCanvas(workspace.id, randomUUID()), null); -}); diff --git a/packages/backend/server/src/core/doc/index.ts b/packages/backend/server/src/core/doc/index.ts index 04dc620070..55dfd7948a 100644 --- a/packages/backend/server/src/core/doc/index.ts +++ b/packages/backend/server/src/core/doc/index.ts @@ -20,7 +20,6 @@ import { DocWriter } from './writer'; DocStorageOptions, PgWorkspaceDocStorageAdapter, PgUserspaceDocStorageAdapter, - DocStorageCronJob, DocReaderProvider, DatabaseDocReader, DocEventsListener, @@ -35,8 +34,14 @@ import { DocWriter } from './writer'; ], }) export class DocStorageModule {} + +@Module({ + imports: [DocStorageModule], + providers: [DocStorageCronJob], +}) +export class DocStorageWorkerModule {} + export { - // only for doc-service DatabaseDocReader, DocReader, DocWriter, diff --git a/packages/backend/server/src/core/doc/job.ts b/packages/backend/server/src/core/doc/job.ts index ef516286ca..6e8e5f8fb1 100644 --- a/packages/backend/server/src/core/doc/job.ts +++ b/packages/backend/server/src/core/doc/job.ts @@ -6,7 +6,7 @@ import { BackendRuntimeProvider } from '../backend-runtime'; declare global { interface Jobs { - 'nightly.cleanExpiredHistories': {}; + 'doc.cleanExpiredHistories': {}; } } @@ -20,7 +20,7 @@ export class DocStorageCronJob { @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) async nightlyJob() { await this.queue.add( - 'nightly.cleanExpiredHistories', + 'doc.cleanExpiredHistories', {}, { jobId: 'nightly-doc-clean-expired-histories', @@ -28,7 +28,7 @@ export class DocStorageCronJob { ); } - @OnJob('nightly.cleanExpiredHistories') + @OnJob('doc.cleanExpiredHistories') async cleanExpiredHistories() { for (;;) { const count = await this.rt.cleanupExpiredSnapshotHistories(1000); diff --git a/packages/backend/server/src/core/doc/reader.ts b/packages/backend/server/src/core/doc/reader.ts index 6f8a898099..9912d77da8 100644 --- a/packages/backend/server/src/core/doc/reader.ts +++ b/packages/backend/server/src/core/doc/reader.ts @@ -1,21 +1,12 @@ -import { FactoryProvider, Injectable, Logger } from '@nestjs/common'; -import { ModuleRef } from '@nestjs/core'; +import { Injectable, Logger } from '@nestjs/common'; import { diffUpdate, encodeStateVectorFromUpdate } from 'yjs'; -import { - Cache, - Config, - CryptoHelper, - getOrGenRequestId, - safeFetch, - UserFriendlyError, -} from '../../base'; +import { Cache } from '../../base'; import { Models } from '../../models'; import { WorkspaceBlobStorage } from '../storage'; import { type CanvasProjectionV1, type PageDocContent, - parseCanvasProjection, parseDocToMarkdownFromDocSnapshot, parsePageDoc, parseWorkspaceDoc, @@ -288,248 +279,7 @@ export class DatabaseDocReader extends DocReader { } } -@Injectable() -export class RpcDocReader extends DatabaseDocReader { - protected override readonly logger = new Logger(DocReader.name); - - constructor( - private readonly config: Config, - private readonly crypto: CryptoHelper, - protected override readonly cache: Cache, - protected override readonly models: Models, - protected override readonly blobStorage: WorkspaceBlobStorage, - protected override readonly workspace: PgWorkspaceDocStorageAdapter - ) { - super(cache, models, blobStorage, workspace); - } - - private async fetch(url: string, method: 'GET' | 'POST', body?: Uint8Array) { - const { pathname } = new URL(url); - const accessToken = this.crypto.signInternalAccessToken({ - method, - path: pathname, - }); - - const headers: Record = { - 'x-access-token': accessToken, - 'x-cloud-trace-context': getOrGenRequestId('rpc'), - }; - if (body) { - headers['content-type'] = 'application/octet-stream'; - } - const requestInit: RequestInit = { - method, - headers, - }; - if (body) { - requestInit.body = body; - } - const res = await safeFetch(url, requestInit, { - timeoutMs: 10_000, - maxRedirects: 0, - maxBytes: 50 * 1024 * 1024, - allowedHeaders: [ - 'content-type', - 'x-access-token', - 'x-cloud-trace-context', - ], - allowPrivateTargetOrigin: true, - }); - if (!res.ok) { - if (res.status === 404) { - return null; - } - const body = (await res.json()) as UserFriendlyError; - throw UserFriendlyError.fromUserFriendlyErrorJSON(body); - } - return res; - } - - override async getDoc( - workspaceId: string, - docId: string - ): Promise { - const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}`; - try { - const res = await this.fetch(url, 'GET'); - if (!res) { - return null; - } - const timestamp = res.headers.get('x-doc-timestamp') as string; - const editor = res.headers.get('x-doc-editor-id') ?? undefined; - const bin = await res.arrayBuffer(); - return { - spaceId: workspaceId, - docId, - bin: Buffer.from(bin), - timestamp: parseInt(timestamp), - editor, - }; - } catch (e) { - if (e instanceof UserFriendlyError) { - throw e; - } - const err = e as Error; - // other error - this.logger.error( - `Failed to fetch doc ${url}, fallback to database doc reader`, - err - ); - // fallback to database doc reader if the error is not user friendly, like network error - return await super.getDoc(workspaceId, docId); - } - } - - override async getDocMarkdown( - workspaceId: string, - docId: string, - aiEditable: boolean - ): Promise { - const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}/markdown?aiEditable=${aiEditable}`; - try { - const res = await this.fetch(url, 'GET'); - if (!res) { - return null; - } - return (await res.json()) as DocMarkdown; - } catch (e) { - if (e instanceof UserFriendlyError) { - throw e; - } - const err = e as Error; - // other error - this.logger.error( - `Failed to fetch doc markdown ${url}, fallback to database doc reader`, - err - ); - // fallback to database doc reader if the error is not user friendly, like network error - return await super.getDocMarkdown(workspaceId, docId, aiEditable); - } - } - - override async getDocCanvas( - workspaceId: string, - docId: string - ): Promise { - const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}/canvas`; - try { - const res = await this.fetch(url, 'GET'); - if (!res) { - return null; - } - return parseCanvasProjection(await res.json()); - } catch (e) { - if (e instanceof UserFriendlyError) { - throw e; - } - this.logger.error( - `Failed to fetch doc canvas ${url}, fallback to database doc reader`, - e as Error - ); - return await super.getDocCanvas(workspaceId, docId); - } - } - - override async getDocDiff( - workspaceId: string, - docId: string, - stateVector?: Uint8Array - ): Promise { - const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}/diff`; - try { - const res = await this.fetch(url, 'POST', stateVector); - if (!res) { - return null; - } - const timestamp = res.headers.get('x-doc-timestamp') as string; - // blob missing data offset [0, 123] - // x-doc-missing-offset: 0,123 - // blob stateVector data offset [124,789] - // x-doc-state-offset: 124,789 - const missingOffset = res.headers.get('x-doc-missing-offset') as string; - const [missingStart, missingEnd] = missingOffset.split(',').map(Number); - const stateOffset = res.headers.get('x-doc-state-offset') as string; - const [stateStart, stateEnd] = stateOffset.split(',').map(Number); - const bin = await res.arrayBuffer(); - return { - missing: new Uint8Array(bin, missingStart, missingEnd - missingStart), - state: new Uint8Array(bin, stateStart, stateEnd - stateStart), - timestamp: parseInt(timestamp), - }; - } catch (e) { - if (e instanceof UserFriendlyError) { - throw e; - } - const err = e as Error; - this.logger.error( - `Failed to fetch doc diff ${url}, fallback to database doc reader`, - err - ); - // fallback to database doc reader if the error is not user friendly, like network error - return await super.getDocDiff(workspaceId, docId, stateVector); - } - } - - protected override async getDocContentWithoutCache( - workspaceId: string, - docId: string, - fullContent = false - ): Promise { - const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}/content?full=${fullContent}`; - try { - const res = await this.fetch(url, 'GET'); - if (!res) { - return null; - } - return (await res.json()) as PageDocContent; - } catch (e) { - if (e instanceof UserFriendlyError) { - throw e; - } - const err = e as Error; - this.logger.error( - `Failed to fetch doc content ${url}, fallback to database doc reader`, - err - ); - return await super.getDocContentWithoutCache( - workspaceId, - docId, - fullContent - ); - } - } - - protected override async getWorkspaceContentWithoutCache( - workspaceId: string - ): Promise { - const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/content`; - try { - const res = await this.fetch(url, 'GET'); - if (!res) { - return null; - } - return (await res.json()) as WorkspaceDocInfo; - } catch (e) { - if (e instanceof UserFriendlyError) { - throw e; - } - const err = e as Error; - this.logger.error( - `Failed to fetch workspace content ${url}, fallback to database doc reader`, - err - ); - return await super.getWorkspaceContentWithoutCache(workspaceId); - } - } -} - -export const DocReaderProvider: FactoryProvider = { +export const DocReaderProvider = { provide: DocReader, - useFactory: (ref: ModuleRef) => { - if (env.flavors.doc || env.flavors.front) { - return ref.create(DatabaseDocReader); - } - return ref.create(RpcDocReader); - }, - inject: [ModuleRef], + useExisting: DatabaseDocReader, }; diff --git a/packages/backend/server/src/core/mail/__tests__/mailer.spec.ts b/packages/backend/server/src/core/mail/__tests__/mailer.spec.ts index 59a4966daf..78765b3a19 100644 --- a/packages/backend/server/src/core/mail/__tests__/mailer.spec.ts +++ b/packages/backend/server/src/core/mail/__tests__/mailer.spec.ts @@ -25,6 +25,7 @@ interface Context { commitMailDeliveryQuotaV1: Sinon.SinonStub; releaseMailDeliveryQuotaV1: Sinon.SinonStub; embeddingHealth: Sinon.SinonStub; + searchStatus: Sinon.SinonStub; }; } @@ -41,6 +42,7 @@ test.before(async t => { reason: 'test', workerRunning: false, }), + searchStatus: Sinon.stub().resolves({ ready: false }), }; t.context.module = await createTestingModule({ tapModule: builder => { diff --git a/packages/backend/server/src/core/notification/resolver.ts b/packages/backend/server/src/core/notification/resolver.ts index fb1ae7503f..10f943ea6e 100644 --- a/packages/backend/server/src/core/notification/resolver.ts +++ b/packages/backend/server/src/core/notification/resolver.ts @@ -1,4 +1,17 @@ -import { Args, ID, Mutation, ResolveField, Resolver } from '@nestjs/graphql'; +import { + Args, + ID, + Info, + Mutation, + ResolveField, + Resolver, +} from '@nestjs/graphql'; +import { + type FragmentDefinitionNode, + type GraphQLResolveInfo, + Kind, + type SelectionNode, +} from 'graphql'; import { MentionUserDocAccessDenied, @@ -17,6 +30,39 @@ import { UnionNotificationBodyType, } from './types'; +function hasSelectedField( + selections: readonly SelectionNode[], + fieldName: string, + fragments: Record +): boolean { + for (const selection of selections) { + if (selection.kind === Kind.FIELD) { + if (selection.name.value === fieldName) return true; + continue; + } + if (selection.kind === Kind.INLINE_FRAGMENT) { + if ( + hasSelectedField( + selection.selectionSet.selections, + fieldName, + fragments + ) + ) { + return true; + } + continue; + } + const fragment = fragments[selection.name.value]; + if ( + fragment && + hasSelectedField(fragment.selectionSet.selections, fieldName, fragments) + ) { + return true; + } + } + return false; +} + @Resolver(() => UserType) export class UserNotificationResolver { constructor( @@ -29,8 +75,21 @@ export class UserNotificationResolver { }) async notifications( @CurrentUser() me: UserType, - @Args('pagination', PaginationInput.decode) pagination: PaginationInput + @Args('pagination', PaginationInput.decode) pagination: PaginationInput, + @Info() info: GraphQLResolveInfo ): Promise { + const selections = info.fieldNodes.flatMap(node => + node.selectionSet ? [...node.selectionSet.selections] : [] + ); + const includesList = + hasSelectedField(selections, 'edges', info.fragments) || + hasSelectedField(selections, 'pageInfo', info.fragments); + + if (!includesList) { + const totalCount = await this.service.countByUserId(me.id); + return paginate([], 'createdAt', pagination, totalCount); + } + const [notifications, totalCount] = await Promise.all([ this.service.findManyByUserId(me.id, pagination), this.service.countByUserId(me.id), diff --git a/packages/backend/server/src/core/permission/__tests__/docs.spec.ts b/packages/backend/server/src/core/permission/__tests__/docs.spec.ts deleted file mode 100644 index 22b9b07e24..0000000000 --- a/packages/backend/server/src/core/permission/__tests__/docs.spec.ts +++ /dev/null @@ -1,426 +0,0 @@ -import { Prisma, PrismaClient } from '@prisma/client'; -import test from 'ava'; - -import { createModule } from '../../../__tests__/create-module'; -import { Mockers } from '../../../__tests__/mocks'; -import { Models } from '../../../models'; -import { AccessControllerBuilder } from '../builder'; -import { DocRole, PermissionModule, WorkspaceRole } from '../index'; -import { PermissionSqlPredicateBuilder } from '../sql-predicate'; -import type { DocAction } from '../types'; - -const module = await createModule({ - imports: [PermissionModule], -}); - -const builder = module.get(AccessControllerBuilder); -const models = module.get(Models); -const db = module.get(PrismaClient); -const sqlPredicate = module.get(PermissionSqlPredicateBuilder); - -test.after.always(async () => { - await module.close(); -}); - -async function sqlReadableDocIds(input: { - workspaceId: string; - userId?: string; - action?: DocAction; - docIds: string[]; -}) { - const values = Prisma.join( - input.docIds.map((docId, index) => Prisma.sql`(${docId}, ${index})`) - ); - const predicate = sqlPredicate.docReadableSql({ - workspaceId: input.workspaceId, - userId: input.userId, - action: input.action ?? 'Doc.Read', - docIdColumn: Prisma.raw('c.doc_id'), - }); - const rows = await db.$queryRaw<{ docId: string }[]>` - WITH candidates(doc_id, ord) AS (VALUES ${values}) - SELECT c.doc_id AS "docId" - FROM candidates c - WHERE ${predicate} - ORDER BY c.ord ASC - `; - return rows.map(row => row.docId); -} - -async function resetProjection(workspaceId: string) { - await db.$executeRaw`DELETE FROM doc_grants WHERE workspace_id = ${workspaceId}`; - await db.$executeRaw`DELETE FROM doc_access_policies WHERE workspace_id = ${workspaceId}`; - await db.$executeRaw`DELETE FROM workspace_members WHERE workspace_id = ${workspaceId}`; - await db.$executeRaw` - INSERT INTO workspace_access_policies ( - workspace_id, - visibility, - sharing_enabled, - url_preview_enabled, - member_default_doc_role, - updated_at - ) - VALUES (${workspaceId}, 'private', true, false, 'none', now()) - ON CONFLICT (workspace_id) - DO UPDATE SET - visibility = EXCLUDED.visibility, - sharing_enabled = EXCLUDED.sharing_enabled, - url_preview_enabled = EXCLUDED.url_preview_enabled, - member_default_doc_role = EXCLUDED.member_default_doc_role, - updated_at = now() - `; - await setWritableRuntime(workspaceId); -} - -async function setWritableRuntime(workspaceId: string) { - await db.effectiveWorkspaceQuotaState.upsert({ - where: { workspaceId }, - create: { - workspaceId, - plan: 'free', - usesOwnerQuota: false, - seatLimit: 0, - blobLimit: 0, - storageQuota: 0, - historyPeriodSeconds: 0, - readonly: false, - known: true, - }, - update: { - readonly: false, - readonlyReasons: [], - known: true, - stale: false, - }, - }); -} - -test('should filter docs by Doc.Read', async t => { - const owner = await module.create(Mockers.User); - const workspace = await module.create(Mockers.Workspace, { - owner, - }); - - const docs1 = await builder - .user(owner.id) - .workspace(workspace.id) - .docs( - [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }], - 'Doc.Read' - ); - - t.is(docs1.length, 3); - t.snapshot(docs1); - - // member should have access to the docs - const member = await module.create(Mockers.User); - await module.create(Mockers.WorkspaceUser, { - workspaceId: workspace.id, - userId: member.id, - type: WorkspaceRole.Collaborator, - }); - - await module.create(Mockers.DocUser, { - workspaceId: workspace.id, - docId: 'doc1', - userId: member.id, - type: DocRole.Reader, - }); - - await module.create(Mockers.DocUser, { - workspaceId: workspace.id, - docId: 'doc2', - userId: member.id, - type: DocRole.Manager, - }); - - const docs2 = await builder - .user(member.id) - .workspace(workspace.id) - .docs( - [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }], - 'Doc.Read' - ); - - t.is(docs2.length, 3); - t.snapshot(docs2); - - // other user should not have access to the docs - const other = await module.create(Mockers.User); - - const docs3 = await builder - .user(other.id) - .workspace(workspace.id) - .docs( - [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }], - 'Doc.Read' - ); - - t.is(docs3.length, 0); -}); - -test('SQL doc read predicate handles member default and public candidates', async t => { - const owner = await module.create(Mockers.User); - const member = await module.create(Mockers.User); - const workspace = await module.create(Mockers.Workspace, { - owner, - }); - await resetProjection(workspace.id); - await db.$executeRaw` - UPDATE workspace_access_policies - SET member_default_doc_role = 'reader' - WHERE workspace_id = ${workspace.id} - `; - await db.$executeRaw` - INSERT INTO workspace_members ( - workspace_id, - user_id, - role, - state, - source, - updated_at - ) - VALUES (${workspace.id}, ${member.id}, 'member', 'active', 'legacy', now()) - `; - await db.$executeRaw` - INSERT INTO doc_access_policies ( - workspace_id, - doc_id, - visibility, - public_role, - member_default_role, - updated_at - ) - VALUES - (${workspace.id}, 'member-default-none', 'private', NULL, 'none', now()), - (${workspace.id}, 'public-doc', 'public', 'external', NULL, now()) - `; - - const docIds = ['missing-policy', 'member-default-none', 'public-doc']; - const sqlReadable = await sqlReadableDocIds({ - workspaceId: workspace.id, - userId: member.id, - docIds, - }); - t.deepEqual(sqlReadable, ['missing-policy', 'public-doc']); -}); - -test('SQL doc read predicate handles non-member grant and sharing disabled', async t => { - const owner = await module.create(Mockers.User); - const nonMember = await module.create(Mockers.User); - const workspace = await module.create(Mockers.Workspace, { - owner, - }); - await resetProjection(workspace.id); - await db.$executeRaw` - INSERT INTO doc_access_policies ( - workspace_id, - doc_id, - visibility, - public_role, - member_default_role, - updated_at - ) - VALUES - (${workspace.id}, 'public-doc', 'public', 'external', NULL, now()), - (${workspace.id}, 'private-doc', 'private', NULL, NULL, now()), - (${workspace.id}, 'explicit-grant', 'private', NULL, NULL, now()), - (${workspace.id}, 'explicit-owner-grant', 'private', NULL, NULL, now()) - `; - await db.$executeRaw` - INSERT INTO doc_grants ( - workspace_id, - doc_id, - principal_type, - principal_id, - role, - updated_at - ) - VALUES - ( - ${workspace.id}, - 'explicit-grant', - 'user', - ${nonMember.id}, - 'reader', - now() - ), - ( - ${workspace.id}, - 'explicit-owner-grant', - 'user', - ${nonMember.id}, - 'owner', - now() - ) - `; - - const docIds = [ - 'public-doc', - 'private-doc', - 'explicit-grant', - 'explicit-owner-grant', - ]; - const sharingEnabledReadable = await sqlReadableDocIds({ - workspaceId: workspace.id, - userId: nonMember.id, - docIds, - }); - const sharingEnabledUpdate = await sqlReadableDocIds({ - workspaceId: workspace.id, - userId: nonMember.id, - action: 'Doc.Update', - docIds, - }); - - await db.$executeRaw` - UPDATE workspace_access_policies - SET sharing_enabled = false - WHERE workspace_id = ${workspace.id} - `; - const sharingDisabledReadable = await sqlReadableDocIds({ - workspaceId: workspace.id, - userId: nonMember.id, - docIds, - }); - - t.deepEqual(sharingEnabledReadable, [ - 'public-doc', - 'explicit-grant', - 'explicit-owner-grant', - ]); - t.deepEqual(sharingEnabledUpdate, ['explicit-owner-grant']); - t.deepEqual(sharingDisabledReadable, []); -}); - -test('SQL doc predicate suppresses member default when explicit grant exists', async t => { - const owner = await module.create(Mockers.User); - const member = await module.create(Mockers.User); - const workspace = await module.create(Mockers.Workspace, { - owner, - }); - await resetProjection(workspace.id); - await db.$executeRaw` - UPDATE workspace_access_policies - SET member_default_doc_role = 'manager' - WHERE workspace_id = ${workspace.id} - `; - await db.$executeRaw` - INSERT INTO workspace_members ( - workspace_id, - user_id, - role, - state, - source, - updated_at - ) - VALUES (${workspace.id}, ${member.id}, 'member', 'active', 'legacy', now()) - `; - await db.$executeRaw` - INSERT INTO doc_access_policies ( - workspace_id, - doc_id, - visibility, - public_role, - member_default_role, - updated_at - ) - VALUES - (${workspace.id}, 'default-manager', 'private', NULL, NULL, now()), - (${workspace.id}, 'explicit-reader', 'private', NULL, NULL, now()) - `; - await db.$executeRaw` - INSERT INTO doc_grants ( - workspace_id, - doc_id, - principal_type, - principal_id, - role, - updated_at - ) - VALUES ( - ${workspace.id}, - 'explicit-reader', - 'user', - ${member.id}, - 'reader', - now() - ) - `; - - const docIds = ['default-manager', 'explicit-reader']; - const sqlUpdateAllowed = await sqlReadableDocIds({ - workspaceId: workspace.id, - userId: member.id, - action: 'Doc.Update', - docIds, - }); - - t.deepEqual(sqlUpdateAllowed, ['default-manager']); -}); - -test('should filter docs by Doc.Publish', async t => { - const owner = await module.create(Mockers.User); - const workspace = await module.create(Mockers.Workspace, { - owner, - }); - await models.workspace.update(workspace.id, { enableSharing: true }); - await setWritableRuntime(workspace.id); - - const docs1 = await builder - .user(owner.id) - .workspace(workspace.id) - .docs( - [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }], - 'Doc.Publish' - ); - - t.is(docs1.length, 3); - t.snapshot(docs1); - - // member should have access to the docs - const member = await module.create(Mockers.User); - await module.create(Mockers.WorkspaceUser, { - workspaceId: workspace.id, - userId: member.id, - type: WorkspaceRole.Collaborator, - }); - - await module.create(Mockers.DocUser, { - workspaceId: workspace.id, - docId: 'doc1', - userId: member.id, - type: DocRole.Reader, - }); - - await module.create(Mockers.DocUser, { - workspaceId: workspace.id, - docId: 'doc2', - userId: member.id, - type: DocRole.Manager, - }); - - const docs2 = await builder - .user(member.id) - .workspace(workspace.id) - .docs( - [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }], - 'Doc.Publish' - ); - - t.is(docs2.length, 2); - t.snapshot(docs2); - - // other user should not have access to the docs - const other = await module.create(Mockers.User); - - const docs3 = await builder - .user(other.id) - .workspace(workspace.id) - .docs( - [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }], - 'Doc.Publish' - ); - - t.is(docs3.length, 0); -}); diff --git a/packages/backend/server/src/core/permission/__tests__/service.spec.ts b/packages/backend/server/src/core/permission/__tests__/service.spec.ts index 9229de5528..4af2b8435f 100644 --- a/packages/backend/server/src/core/permission/__tests__/service.spec.ts +++ b/packages/backend/server/src/core/permission/__tests__/service.spec.ts @@ -5,7 +5,6 @@ import { DocRole } from '../../../models'; import { docLegacyBoundary } from '../context'; import { PermissionContextLoader } from '../context-loader'; import { PermissionService } from '../service'; -import { PermissionSqlPredicateBuilder } from '../sql-predicate'; function createCls() { const store = new Map(); @@ -262,68 +261,3 @@ test('PermissionService maps native validation errors to internal errors', t => t.true(error instanceof InternalServerError); }); - -test('PermissionSqlPredicateBuilder rejects unsafe raw doc id columns', t => { - const builder = new PermissionSqlPredicateBuilder(); - - t.throws( - () => - builder.docReadable({ - workspaceId: 'w1', - userId: 'u1', - action: 'Doc.Read', - docIdColumn: 'docs.id; DROP TABLE docs' as never, - }), - { message: 'Unsupported doc id column: docs.id; DROP TABLE docs' } - ); -}); - -test('PermissionSqlPredicateBuilder caps non-member grants below manager', t => { - const builder = new PermissionSqlPredicateBuilder(); - const update = builder.docReadable({ - workspaceId: 'w1', - userId: 'u1', - action: 'Doc.Update', - }); - const transferOwner = builder.docReadable({ - workspaceId: 'w1', - userId: 'u1', - action: 'Doc.TransferOwner', - }); - - t.true((update.params[4] as string[]).includes('editor')); - t.true((update.params[4] as string[]).includes('manager')); - t.true((update.params[4] as string[]).includes('owner')); - t.deepEqual(transferOwner.params[3], ['owner']); - t.deepEqual(transferOwner.params[4], []); -}); - -test('PermissionSqlPredicateBuilder uses terminal permission tables', t => { - const predicate = new PermissionSqlPredicateBuilder().docReadable({ - workspaceId: 'w1', - userId: 'u1', - action: 'Doc.Read', - docIdColumn: 'docs.id', - }); - - t.true(predicate.sql.includes('FROM workspace_access_policies wap')); - t.true(predicate.sql.includes('LEFT JOIN doc_access_policies dap')); - t.true(predicate.sql.includes('workspace_members')); - t.true(predicate.sql.includes('doc_grants')); - t.false(predicate.sql.includes('workspace_user_permissions')); - t.false(predicate.sql.includes('workspace_page_user_permissions')); -}); - -test('PermissionService always uses the terminal SQL predicate', t => { - const predicate = new PermissionService( - createLoader().loader - ).docReadableSqlPredicate({ - workspaceId: 'w1', - userId: 'u1', - action: 'Doc.Read', - }); - const sql = (predicate as unknown as { sql: string }).sql; - - t.true(sql.includes('workspace_access_policies')); - t.false(sql.includes('workspace_user_permissions')); -}); diff --git a/packages/backend/server/src/core/permission/index.ts b/packages/backend/server/src/core/permission/index.ts index 07f839017b..c3a7fbded8 100644 --- a/packages/backend/server/src/core/permission/index.ts +++ b/packages/backend/server/src/core/permission/index.ts @@ -6,7 +6,6 @@ import { PermissionContextLoader } from './context-loader'; import { EventsListener } from './event'; import { WorkspacePolicyService } from './policy'; import { PermissionService } from './service'; -import { PermissionSqlPredicateBuilder } from './sql-predicate'; @Module({ imports: [QuotaServiceModule], @@ -14,16 +13,10 @@ import { PermissionSqlPredicateBuilder } from './sql-predicate'; AccessControllerBuilder, EventsListener, WorkspacePolicyService, - PermissionSqlPredicateBuilder, PermissionContextLoader, PermissionService, ], - exports: [ - AccessControllerBuilder, - WorkspacePolicyService, - PermissionSqlPredicateBuilder, - PermissionService, - ], + exports: [AccessControllerBuilder, WorkspacePolicyService, PermissionService], }) export class PermissionModule {} @@ -35,7 +28,6 @@ export { } from './permission-map'; export { WorkspacePolicyService } from './policy'; export { PermissionService } from './service'; -export { PermissionSqlPredicateBuilder } from './sql-predicate'; export { DOC_ACTIONS, type DocAction, diff --git a/packages/backend/server/src/core/permission/service.ts b/packages/backend/server/src/core/permission/service.ts index 3ebc95755f..ce4960d396 100644 --- a/packages/backend/server/src/core/permission/service.ts +++ b/packages/backend/server/src/core/permission/service.ts @@ -1,5 +1,4 @@ -import { Inject, Injectable, Optional } from '@nestjs/common'; -import { Prisma } from '@prisma/client'; +import { Injectable, Optional } from '@nestjs/common'; import { DocActionDenied, @@ -18,8 +17,6 @@ import { type PermissionWorkspaceAction, } from './context-loader'; import { WorkspacePolicyService } from './policy'; -import { PermissionSqlPredicateBuilder } from './sql-predicate'; -import type { DocAction } from './types'; const RUNTIME_RESTRICTED_WORKSPACE_ACTIONS = new Set( [ @@ -59,21 +56,9 @@ export class PermissionService { constructor( private readonly loader: PermissionContextLoader, @Optional() - @Inject(PermissionSqlPredicateBuilder) - private readonly sqlPredicate = new PermissionSqlPredicateBuilder(), - @Optional() private readonly workspacePolicy?: WorkspacePolicyService ) {} - docReadableSqlPredicate(input: { - userId: string; - workspaceId: string; - action: DocAction; - docIdColumn?: Prisma.Sql; - }) { - return this.sqlPredicate.docReadableSql(input); - } - evaluate(input: PermissionEvaluationInputV1) { try { return evaluatePermissionV1(input); diff --git a/packages/backend/server/src/core/permission/sql-predicate.ts b/packages/backend/server/src/core/permission/sql-predicate.ts deleted file mode 100644 index c8a9b80826..0000000000 --- a/packages/backend/server/src/core/permission/sql-predicate.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { Prisma } from '@prisma/client'; - -import { permissionActionRoleMatrixV1 } from '../../native'; -import type { DocAction } from './types'; - -export type PermissionSqlPredicate = { - sql: string; - params: unknown[]; -}; - -type RawDocIdColumn = 'doc_id' | 'docs.id'; - -@Injectable() -export class PermissionSqlPredicateBuilder { - private readonly matrix = permissionActionRoleMatrixV1() as { - doc?: { roles?: Record }; - workspace?: { roles?: Record }; - }; - - private docRolesForAction(action: DocAction) { - return Object.entries(this.matrix.doc?.roles ?? {}) - .filter(([, actions]) => actions.includes(action)) - .map(([role]) => role) - .filter(role => role !== 'none'); - } - - private inheritedWorkspaceRolesForDocAction(action: DocAction) { - const docRoles = new Set(this.docRolesForAction(action)); - return [ - docRoles.has('owner') ? 'owner' : null, - docRoles.has('manager') ? 'admin' : null, - ].filter((role): role is string => role !== null); - } - - private nonMemberDocGrantRolesForAction(action: DocAction) { - const roles = new Set(this.docRolesForAction(action)); - roles.delete('external'); - roles.delete('manager'); - roles.delete('owner'); - if (roles.has('editor')) { - roles.add('manager'); - roles.add('owner'); - } - return [...roles]; - } - - private rawDocIdColumn(column: RawDocIdColumn = 'doc_id') { - switch (column) { - case 'doc_id': - case 'docs.id': - return column; - default: - throw new Error(`Unsupported doc id column: ${column}`); - } - } - - docReadable(input: { - workspaceId: string; - userId?: string; - action: DocAction; - docIdColumn?: RawDocIdColumn; - }): PermissionSqlPredicate { - const docRoles = this.docRolesForAction(input.action); - const inheritedWorkspaceRoles = this.inheritedWorkspaceRolesForDocAction( - input.action - ); - const grantRoles = docRoles.filter(role => role !== 'external'); - const nonMemberGrantRoles = this.nonMemberDocGrantRolesForAction( - input.action - ); - const docIdColumn = this.rawDocIdColumn(input.docIdColumn); - - return { - sql: [ - `EXISTS (SELECT 1 FROM workspace_access_policies wap`, - `LEFT JOIN doc_access_policies dap ON dap.workspace_id = wap.workspace_id`, - `AND dap.doc_id = ${docIdColumn}`, - `LEFT JOIN workspace_members wm ON wm.workspace_id = wap.workspace_id`, - `AND wm.user_id = ? AND wm.state = 'active'`, - `LEFT JOIN doc_grants dg ON dg.workspace_id = wap.workspace_id`, - `AND dg.doc_id = ${docIdColumn} AND dg.principal_type = 'user' AND dg.principal_id = ?`, - `WHERE wap.workspace_id = ?`, - `AND (`, - `(wm.id IS NOT NULL AND dg.role = ANY(?::text[]))`, - `OR (wm.id IS NULL AND wap.sharing_enabled AND dg.role = ANY(?::text[]))`, - `OR wm.role = ANY(?::text[])`, - `OR (wm.id IS NOT NULL AND dg.principal_id IS NULL AND COALESCE(dap.member_default_role, wap.member_default_doc_role) = ANY(?::text[]))`, - `OR (wap.sharing_enabled AND dap.visibility = 'public' AND dap.public_role = ANY(?::text[]))`, - `))`, - ].join(' '), - params: [ - input.userId, - input.userId, - input.workspaceId, - grantRoles, - nonMemberGrantRoles, - inheritedWorkspaceRoles, - grantRoles, - docRoles, - ], - }; - } - - docReadableSql(input: { - workspaceId: string; - userId?: string; - action: DocAction; - docIdColumn?: Prisma.Sql; - }): Prisma.Sql { - const docRoles = this.docRolesForAction(input.action); - const grantRoles = docRoles.filter(role => role !== 'external'); - const nonMemberGrantRoles = this.nonMemberDocGrantRolesForAction( - input.action - ); - const inheritedWorkspaceRoles = this.inheritedWorkspaceRolesForDocAction( - input.action - ); - const docIdColumn = input.docIdColumn ?? Prisma.raw('doc_id'); - - return Prisma.sql` - EXISTS ( - SELECT 1 - FROM workspace_access_policies wap - LEFT JOIN doc_access_policies dap - ON dap.workspace_id = wap.workspace_id - AND dap.doc_id = ${docIdColumn} - LEFT JOIN workspace_members wm - ON wm.workspace_id = wap.workspace_id - AND wm.user_id = ${input.userId} - AND wm.state = 'active' - LEFT JOIN doc_grants dg - ON dg.workspace_id = wap.workspace_id - AND dg.doc_id = ${docIdColumn} - AND dg.principal_type = 'user' - AND dg.principal_id = ${input.userId} - WHERE wap.workspace_id = ${input.workspaceId} - AND ( - (wm.id IS NOT NULL AND dg.role = ANY(${Prisma.sql`${grantRoles}::text[]`})) - OR (wm.id IS NULL AND wap.sharing_enabled AND dg.role = ANY(${Prisma.sql`${nonMemberGrantRoles}::text[]`})) - OR wm.role = ANY(${Prisma.sql`${inheritedWorkspaceRoles}::text[]`}) - OR (wm.id IS NOT NULL AND dg.principal_id IS NULL AND COALESCE(dap.member_default_role, wap.member_default_doc_role) = ANY(${Prisma.sql`${grantRoles}::text[]`})) - OR (wap.sharing_enabled AND dap.visibility = 'public' AND dap.public_role = ANY(${Prisma.sql`${docRoles}::text[]`})) - ) - ) - `; - } -} diff --git a/packages/backend/server/src/core/storage-runtime/__tests__/provider.spec.ts b/packages/backend/server/src/core/storage-runtime/__tests__/provider.spec.ts index 16b455e0dc..8423fd4a98 100644 --- a/packages/backend/server/src/core/storage-runtime/__tests__/provider.spec.ts +++ b/packages/backend/server/src/core/storage-runtime/__tests__/provider.spec.ts @@ -51,18 +51,20 @@ test('storage-runtime provider restarts on storage config changes', async t => { const { provider, runtime } = createProvider(); await provider.start(); + await provider.runMigrations(); await provider.onConfigChanged({ updates: { storages: {} } }); t.is(runtime.stop.callCount, 1); t.is(runtime.configure.callCount, 2); t.is(runtime.start.callCount, 2); - t.is(runtime.runMigrations.callCount, 2); + t.is(runtime.runMigrations.callCount, 1); }); test('storage-runtime provider restarts on copilot storage config changes', async t => { const { provider, runtime } = createProvider(); await provider.start(); + await provider.runMigrations(); await provider.onConfigChanged({ updates: { copilot: { @@ -78,7 +80,7 @@ test('storage-runtime provider restarts on copilot storage config changes', asyn t.is(runtime.stop.callCount, 1); t.is(runtime.configure.callCount, 2); t.is(runtime.start.callCount, 2); - t.is(runtime.runMigrations.callCount, 2); + t.is(runtime.runMigrations.callCount, 1); }); test('storage-runtime provider ignores unrelated config changes', async t => { @@ -89,5 +91,5 @@ test('storage-runtime provider ignores unrelated config changes', async t => { t.is(runtime.stop.callCount, 0); t.is(runtime.start.callCount, 1); - t.is(runtime.runMigrations.callCount, 1); + t.is(runtime.runMigrations.callCount, 0); }); diff --git a/packages/backend/server/src/core/storage-runtime/provider.ts b/packages/backend/server/src/core/storage-runtime/provider.ts index 4ff84221f9..e149648fb2 100644 --- a/packages/backend/server/src/core/storage-runtime/provider.ts +++ b/packages/backend/server/src/core/storage-runtime/provider.ts @@ -49,7 +49,6 @@ export class StorageRuntimeProvider async start() { this.configureRuntime(); await this.runtime.start(); - await this.runMigrationsOnce(); const health = await this.runtime.health(); this.logger.log( `storage runtime started: db=${health.databaseConnected} provider=${health.provider ?? 'none'}` @@ -82,6 +81,10 @@ export class StorageRuntimeProvider return await this.runtime.health(); } + async runMigrations() { + await this.runMigrationsOnce(); + } + async providerCapabilities( scope: string ): Promise { @@ -258,6 +261,16 @@ export class StorageRuntimeProvider ); } + async rebuildDocBlobRefs( + workspaceId: string, + docId: string, + sourceRevision: number + ) { + return await this.measured('rebuildDocBlobRefs', rt => + rt.rebuildDocBlobRefs(workspaceId, docId, sourceRevision) + ); + } + async reconcileWorkspaceDocuments(workspaceId: string) { return await this.measured('reconcileWorkspaceDocuments', rt => rt.reconcileWorkspaceDocuments(workspaceId) diff --git a/packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts b/packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts index 6234fb7f03..903e121190 100644 --- a/packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts +++ b/packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts @@ -8,6 +8,7 @@ interface Context { health: Sinon.SinonStub; reconcileWorkspaceDocuments: Sinon.SinonStub; backfillMissingBlobMetadata: Sinon.SinonStub; + rebuildDocBlobRefs: Sinon.SinonStub; rebuildWorkspaceDocBlobRefs: Sinon.SinonStub; planUnreferencedWorkspaceBlobs: Sinon.SinonStub; executeBlobCleanupCandidates: Sinon.SinonStub; @@ -45,6 +46,7 @@ test.beforeEach(t => { recovered: 0, }), backfillMissingBlobMetadata: Sinon.stub(), + rebuildDocBlobRefs: Sinon.stub(), rebuildWorkspaceDocBlobRefs: Sinon.stub(), planUnreferencedWorkspaceBlobs: Sinon.stub(), executeBlobCleanupCandidates: Sinon.stub(), @@ -289,6 +291,31 @@ test('storage reconciliation still refreshes document retention without object s t.false(t.context.runtime.planUnreferencedWorkspaceBlobs.called); }); +test('document projection worker drains metadata incrementally after a document merge', async t => { + t.context.runtime.rebuildDocBlobRefs.resolves({ + scannedDocs: 1, + parsedDocs: 1, + refsWritten: 1, + refsDeleted: 0, + failedDocs: 0, + nextCursor: null, + }); + + await t.context.job.projectWorkspaceDocBlobRefs({ + workspaceId: 'workspace-1', + docId: 'doc-1', + sourceRevision: 123, + }); + + t.true( + t.context.runtime.rebuildDocBlobRefs.calledOnceWith( + 'workspace-1', + 'doc-1', + 123 + ) + ); +}); + test('document cleanup dispatches stable search effects', async t => { t.context.runtime.executeDocumentCleanupCandidates.resolves({ scannedCandidates: 1, diff --git a/packages/backend/server/src/core/storage/blob-job.ts b/packages/backend/server/src/core/storage/blob-job.ts index d1f083a97a..5462ecbc4c 100644 --- a/packages/backend/server/src/core/storage/blob-job.ts +++ b/packages/backend/server/src/core/storage/blob-job.ts @@ -5,8 +5,8 @@ import { PrismaClient } from '@prisma/client'; import { EventBus, JobQueue, metrics, OnJob } from '../../base'; import { StorageRuntimeProvider } from '../storage-runtime'; -// Queue keys are persisted API; keep the legacy backendRuntime.* names while -// StorageBlobJob and StorageRuntimeProvider own the implementation. +// Queue keys are persisted API; StorageBlobJob and StorageRuntimeProvider own +// the implementation. declare global { interface Jobs { 'backendRuntime.backfillMissingBlobMetadata': { @@ -23,6 +23,11 @@ declare global { workspaceLimit?: number; docLimit?: number; }; + 'backendRuntime.projectWorkspaceDocBlobRefs': { + workspaceId: string; + docId: string; + sourceRevision: number; + }; 'backendRuntime.executeDocumentCleanupCandidates': { workspaceId?: string; gracePeriodDays?: number; @@ -281,6 +286,23 @@ export class StorageBlobJob { } } + @OnJob('backendRuntime.projectWorkspaceDocBlobRefs') + async projectWorkspaceDocBlobRefs({ + workspaceId, + docId, + sourceRevision, + }: Jobs['backendRuntime.projectWorkspaceDocBlobRefs']) { + const result = await this.rt.rebuildDocBlobRefs( + workspaceId, + docId, + sourceRevision + ); + this.autoLog( + `projected doc blob refs workspace=${workspaceId} doc=${docId} sourceRevision=${sourceRevision} parsed=${result.parsedDocs} failed=${result.failedDocs}`, + Boolean(result.failedDocs) + ); + } + @OnJob('backendRuntime.executeDocumentCleanupCandidates') async executeDocumentCleanupCandidates({ workspaceId, diff --git a/packages/backend/server/src/core/storage/index.ts b/packages/backend/server/src/core/storage/index.ts index 0b70816de7..20aa1d9a12 100644 --- a/packages/backend/server/src/core/storage/index.ts +++ b/packages/backend/server/src/core/storage/index.ts @@ -14,22 +14,23 @@ import { @Module({ imports: [StorageRuntimeModule], - controllers: [R2UploadController], - providers: [ - WorkspaceBlobStorage, - AvatarStorage, - CommentAttachmentStorage, - StorageBlobJob, - BlobUploadCleanupJob, - ], - exports: [ - WorkspaceBlobStorage, - AvatarStorage, - CommentAttachmentStorage, - StorageBlobJob, - ], + providers: [WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage], + exports: [WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage], }) export class StorageModule {} +@Module({ + imports: [StorageModule], + controllers: [R2UploadController], +}) +export class StorageApiModule {} + +@Module({ + imports: [StorageModule], + providers: [StorageBlobJob, BlobUploadCleanupJob], + exports: [StorageBlobJob], +}) +export class StorageWorkerModule {} + export { StorageBlobJob } from './blob-job'; export { AvatarStorage, CommentAttachmentStorage, WorkspaceBlobStorage }; diff --git a/packages/backend/server/src/core/storage/job.ts b/packages/backend/server/src/core/storage/job.ts index bdd3dedabb..ecff43d03f 100644 --- a/packages/backend/server/src/core/storage/job.ts +++ b/packages/backend/server/src/core/storage/job.ts @@ -6,7 +6,7 @@ import { StorageRuntimeProvider } from '../storage-runtime'; declare global { interface Jobs { - 'nightly.cleanExpiredPendingBlobs': {}; + 'backendRuntime.cleanExpiredPendingBlobs': {}; } } @@ -23,7 +23,7 @@ export class BlobUploadCleanupJob { @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) async nightlyJob() { await this.queue.add( - 'nightly.cleanExpiredPendingBlobs', + 'backendRuntime.cleanExpiredPendingBlobs', {}, { jobId: 'nightly-blob-clean-expired-pending', @@ -31,7 +31,7 @@ export class BlobUploadCleanupJob { ); } - @OnJob('nightly.cleanExpiredPendingBlobs') + @OnJob('backendRuntime.cleanExpiredPendingBlobs') async cleanExpiredPendingBlobs() { const cutoff = Date.now() - OneDay; let scanned = 0; diff --git a/packages/backend/server/src/core/sync/gateway.ts b/packages/backend/server/src/core/sync/gateway.ts index a9d2a13928..0c36179732 100644 --- a/packages/backend/server/src/core/sync/gateway.ts +++ b/packages/backend/server/src/core/sync/gateway.ts @@ -20,10 +20,12 @@ import semver from 'semver'; import { type Server, Socket } from 'socket.io'; import { + BadRequest, CallMetric, checkCanaryDateClientVersion, DocNotFound, DocUpdateBlocked, + EventBus, GatewayErrorWrapper, metrics, NotInSpace, @@ -63,9 +65,9 @@ type EventResponse = Data extends never }; // sync: shared room for space membership checks and non-protocol broadcasts. -// sync-025: legacy 0.25 doc sync protocol (space:broadcast-doc-update). -// sync-026: current doc sync protocol (space:broadcast-doc-updates). -type RoomType = 'sync' | 'sync-025' | 'sync-026' | `${string}:awareness`; +// sync-026: legacy doc sync protocol (space:broadcast-doc-updates). +// sync-027: batch doc sync protocol (invalidation + active subscriptions). +type RoomType = 'sync' | 'sync-026' | 'sync-027' | `${string}:awareness`; function Room( spaceId: string, @@ -74,14 +76,14 @@ function Room( return `${spaceId}:${type}`; } -const MIN_WS_CLIENT_VERSION = new semver.Range('>=0.25.0', { +const MIN_WS_CLIENT_VERSION = new semver.Range('>=0.26.0', { includePrerelease: true, }); -const DOC_UPDATES_PROTOCOL_026 = new semver.Range('>=0.26.0-0', { +const MIN_BATCH_WS_CLIENT_VERSION = new semver.Range('>=0.27.5-0', { includePrerelease: true, }); +const MAX_SPACE_JOIN_BATCH_SIZE = 100; -type SyncProtocolRoomType = Extract; const SOCKET_PRESENCE_USER_ID_KEY = 'affinePresenceUserId'; function normalizeWsClientVersion(clientVersion: string): string | null { @@ -108,11 +110,9 @@ function isSupportedWsClientVersion(clientVersion: string): boolean { ); } -function getSyncProtocolRoomType(clientVersion: string): SyncProtocolRoomType { +function isBatchWsClientVersion(clientVersion: string): boolean { const normalized = normalizeWsClientVersion(clientVersion); - return DOC_UPDATES_PROTOCOL_026.test(normalized ?? clientVersion) - ? 'sync-026' - : 'sync-025'; + return Boolean(normalized && MIN_BATCH_WS_CLIENT_VERSION.test(normalized)); } enum SpaceType { @@ -133,11 +133,26 @@ interface JoinSpaceAwarenessMessage { clientVersion: string; } +interface JoinSpaceBatchEntry { + spaceType: SpaceType; + spaceId: string; + docId?: string; +} + +interface JoinSpaceBatchMessage { + spaces: [JoinSpaceBatchEntry, ...JoinSpaceBatchEntry[]]; + clientVersion: string; +} + interface LeaveSpaceMessage { spaceType: SpaceType; spaceId: string; } +interface LeaveSpaceBatchMessage extends LeaveSpaceMessage { + docIds: string[]; +} + interface LeaveSpaceAwarenessMessage { spaceType: SpaceType; spaceId: string; @@ -161,15 +176,6 @@ interface BroadcastDocUpdatesMessage { compressed?: boolean; } -interface BroadcastDocUpdateMessage { - spaceType: SpaceType; - spaceId: string; - docId: string; - update: string; - timestamp: number; - editor: string; -} - interface LoadDocMessage { spaceType: SpaceType; spaceId: string; @@ -201,6 +207,135 @@ interface UpdateAwarenessMessage { awarenessUpdate: string; } +interface SyncAwarenessEvent { + spaceType: SpaceType; + spaceId: string; + docId: string; + sourceSocketId?: string; +} + +interface SyncDocUpdatesPayload { + spaceType: SpaceType; + spaceId: string; + docId: string; + updates: Uint8Array[]; + timestamp: number; + editor?: string; +} + +declare global { + interface Events { + 'sync.doc.updates.pushed': { + spaceType: SpaceType; + spaceId: string; + docId: string; + updates: string[]; + timestamp: number; + editor?: string; + }; + 'sync.awareness.collect': SyncAwarenessEvent; + 'sync.awareness.updated': SyncAwarenessEvent & { + awarenessUpdate: string; + }; + 'sync.permissions.changed': { + spaceType: SpaceType; + spaceId: string; + docId?: string; + }; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function parseJoinSpaceBatchMessage(message: unknown): JoinSpaceBatchMessage { + if (!isRecord(message)) { + throw new BadRequest('Invalid space join batch payload.'); + } + + const { spaces, clientVersion } = message; + if (!Array.isArray(spaces) || spaces.length === 0) { + throw new BadRequest('Space join batch must not be empty.'); + } + if (spaces.length > MAX_SPACE_JOIN_BATCH_SIZE) { + throw new BadRequest( + `Space join batch exceeds limit (${MAX_SPACE_JOIN_BATCH_SIZE}).` + ); + } + if (typeof clientVersion !== 'string' || clientVersion.length === 0) { + throw new BadRequest('Space join batch requires a client version.'); + } + + const entries = spaces.map((space, index) => { + if (!isRecord(space)) { + throw new BadRequest(`Invalid space join batch entry at index ${index}.`); + } + + const { spaceType, spaceId, docId } = space; + if ( + (spaceType !== SpaceType.Userspace && + spaceType !== SpaceType.Workspace) || + typeof spaceId !== 'string' || + spaceId.trim().length === 0 || + (docId !== undefined && + (typeof docId !== 'string' || docId.trim().length === 0)) + ) { + throw new BadRequest(`Invalid space join batch entry at index ${index}.`); + } + + return { + spaceType, + spaceId, + ...(docId === undefined ? {} : { docId }), + } satisfies JoinSpaceBatchEntry; + }) as [JoinSpaceBatchEntry, ...JoinSpaceBatchEntry[]]; + + const first = entries[0]; + const duplicateKeys = new Set(); + for (const entry of entries) { + if ( + entry.spaceType !== first.spaceType || + entry.spaceId !== first.spaceId + ) { + throw new BadRequest( + 'Space join batch entries must belong to one space.' + ); + } + + const key = JSON.stringify([ + entry.spaceType, + entry.spaceId, + entry.docId ?? null, + ]); + if (duplicateKeys.has(key)) { + throw new BadRequest('Space join batch contains duplicate entries.'); + } + duplicateKeys.add(key); + } + + return { spaces: entries, clientVersion }; +} + +function parseLeaveSpaceBatchMessage(message: unknown): LeaveSpaceBatchMessage { + if (!isRecord(message)) { + throw new BadRequest('Invalid space leave batch payload.'); + } + + const { spaceType, spaceId, docIds } = message; + if ( + (spaceType !== SpaceType.Userspace && spaceType !== SpaceType.Workspace) || + typeof spaceId !== 'string' || + spaceId.trim().length === 0 || + !Array.isArray(docIds) || + docIds.some(docId => typeof docId !== 'string' || docId.trim().length === 0) + ) { + throw new BadRequest('Invalid space leave batch payload.'); + } + + return { spaceType, spaceId, docIds }; +} + @WebSocketGateway() @UseInterceptors(ClsInterceptor) export class SpaceSyncGateway @@ -223,13 +358,16 @@ export class SpaceSyncGateway private activeUsersFlushTimer?: NodeJS.Timeout; private activeUsersFlushInFlight = false; private activeUsersFlushQueued = false; + private readonly activeDocSockets = new Map>(); + private readonly activeSocketDocs = new Map>(); constructor( private readonly ac: PermissionAccess, private readonly workspace: PgWorkspaceDocStorageAdapter, private readonly userspace: PgUserspaceDocStorageAdapter, private readonly docReader: DocReader, - private readonly models: Models + private readonly models: Models, + private readonly event: EventBus ) {} onModuleInit() { @@ -345,6 +483,166 @@ export class SpaceSyncGateway } } + private activeDocKey(spaceType: SpaceType, spaceId: string, docId: string) { + return `${spaceType}:${spaceId}:${docId}`; + } + + private addActiveDocSubscription( + client: Socket, + spaceType: SpaceType, + spaceId: string, + docId: string + ) { + const key = this.activeDocKey(spaceType, spaceId, docId); + let sockets = this.activeDocSockets.get(key); + if (!sockets) { + sockets = new Set(); + this.activeDocSockets.set(key, sockets); + } + sockets.add(client); + + let docs = this.activeSocketDocs.get(client.id); + if (!docs) { + docs = new Set(); + this.activeSocketDocs.set(client.id, docs); + } + docs.add(key); + } + + private removeActiveDocSubscription( + client: Socket, + spaceType: SpaceType, + spaceId: string, + docId: string + ) { + const key = this.activeDocKey(spaceType, spaceId, docId); + const sockets = this.activeDocSockets.get(key); + sockets?.delete(client); + if (sockets && sockets.size === 0) { + this.activeDocSockets.delete(key); + } + + const docs = this.activeSocketDocs.get(client.id); + docs?.delete(key); + if (docs && docs.size === 0) { + this.activeSocketDocs.delete(client.id); + } + } + + private removeAllActiveDocSubscriptions(client: Socket) { + const docs = this.activeSocketDocs.get(client.id); + if (!docs) { + return; + } + + for (const key of docs) { + const sockets = this.activeDocSockets.get(key); + sockets?.delete(client); + if (sockets && sockets.size === 0) { + this.activeDocSockets.delete(key); + } + } + this.activeSocketDocs.delete(client.id); + } + + private removeActiveDocSubscriptionsInSpace( + client: Socket, + spaceType: SpaceType, + spaceId: string + ) { + const prefix = `${spaceType}:${spaceId}:`; + for (const key of Array.from(this.activeSocketDocs.get(client.id) ?? [])) { + if (!key.startsWith(prefix)) { + continue; + } + const [, keySpaceId, docId] = key.split(':'); + this.removeActiveDocSubscription(client, spaceType, keySpaceId, docId); + } + } + + private hasActiveDocSubscription( + client: Socket, + spaceType: SpaceType, + spaceId: string, + docId: string + ) { + return Boolean( + this.activeSocketDocs + .get(client.id) + ?.has(this.activeDocKey(spaceType, spaceId, docId)) + ); + } + + private emitActiveDocUpdate( + payload: SyncDocUpdatesPayload, + sourceSocketId?: string, + broadcastPayload?: BroadcastDocUpdatesMessage + ) { + const sockets = this.activeDocSockets.get( + this.activeDocKey(payload.spaceType, payload.spaceId, payload.docId) + ); + if (!sockets) { + return; + } + + const activeBroadcastPayload = + broadcastPayload ?? + this.buildBroadcastPayload( + payload.spaceType, + payload.spaceId, + payload.docId, + payload.updates, + payload.timestamp, + payload.editor + ); + for (const socket of sockets) { + if (socket.id !== sourceSocketId) { + socket.emit('space:broadcast-doc-updates', activeBroadcastPayload); + } + } + } + + private emitActiveAwarenessCollect(event: SyncAwarenessEvent) { + const sockets = this.activeDocSockets.get( + this.activeDocKey(event.spaceType, event.spaceId, event.docId) + ); + if (!sockets) { + return; + } + + for (const socket of sockets) { + if (socket.id !== event.sourceSocketId) { + socket.emit('space:collect-awareness', { + spaceType: event.spaceType, + spaceId: event.spaceId, + docId: event.docId, + }); + } + } + } + + private emitActiveAwarenessUpdate( + event: SyncAwarenessEvent & { awarenessUpdate: string } + ) { + const sockets = this.activeDocSockets.get( + this.activeDocKey(event.spaceType, event.spaceId, event.docId) + ); + if (!sockets) { + return; + } + + for (const socket of sockets) { + if (socket.id !== event.sourceSocketId) { + socket.emit('space:broadcast-awareness-update', { + spaceType: event.spaceType, + spaceId: event.spaceId, + docId: event.docId, + awarenessUpdate: event.awarenessUpdate, + }); + } + } + } + handleConnection(client: Socket) { this.connectionCount++; this.logger.debug(`New connection, total: ${this.connectionCount}`); @@ -355,6 +653,7 @@ export class SpaceSyncGateway } handleDisconnect(client: Socket) { + this.removeAllActiveDocSubscriptions(client); this.connectionCount = Math.max(0, this.connectionCount - 1); this.trackDisconnectedSocket(client.id); this.logger.debug( @@ -538,39 +837,176 @@ export class SpaceSyncGateway timestamp, editor, }: Events['doc.updates.pushed']) { - if (!this.server || updates.length === 0) { - return; - } - - const room025 = `${spaceType}:${Room(spaceId, 'sync-025')}`; - const encodedUpdates = this.encodeUpdates(updates); - for (const update of encodedUpdates) { - const payload: BroadcastDocUpdateMessage = { - spaceType: spaceType as SpaceType, - spaceId, - docId, - update, - timestamp, - editor: editor ?? '', - }; - this.server.to(room025).emit('space:broadcast-doc-update', payload); - } - - const room026 = `${spaceType}:${Room(spaceId, 'sync-026')}`; - const payload = this.buildBroadcastPayload( - spaceType as SpaceType, + this.publishDocUpdate({ + spaceType: spaceType as SpaceType, spaceId, docId, updates, timestamp, - editor + editor, + }); + } + + @OnEvent('sync.doc.updates.pushed') + onClusterDocUpdatesPushed(payload: Events['sync.doc.updates.pushed']) { + this.emitActiveDocUpdate({ + ...payload, + updates: payload.updates.map(update => + Uint8Array.from(Buffer.from(update, 'base64')) + ), + }); + } + + @OnEvent('sync.awareness.collect') + onClusterAwarenessCollect(event: Events['sync.awareness.collect']) { + this.emitActiveAwarenessCollect(event); + } + + @OnEvent('sync.awareness.updated') + onClusterAwarenessUpdated(event: Events['sync.awareness.updated']) { + this.emitActiveAwarenessUpdate(event); + } + + @OnEvent('doc.grants.changed') + @OnEvent('doc.owner.changed') + @OnEvent('doc.default_role.changed') + @OnEvent('doc.public_state.changed') + @OnEvent('workspace.members.updated') + @OnEvent('workspace.members.roleChanged') + @OnEvent('workspace.members.removed') + @OnEvent('workspace.members.leave') + @OnEvent('workspace.owner.changed') + async onPermissionChanged({ + workspaceId, + docId, + }: { + workspaceId: string; + docId?: string; + }) { + await this.publishPermissionChange({ + spaceType: SpaceType.Workspace, + spaceId: workspaceId, + docId, + }); + } + + @OnEvent('sync.permissions.changed') + async onClusterPermissionsChanged(event: Events['sync.permissions.changed']) { + await this.revalidateActiveDocSubscriptions(event); + } + + private async publishPermissionChange( + event: Events['sync.permissions.changed'] + ) { + await this.revalidateActiveDocSubscriptions(event); + this.event.broadcast('sync.permissions.changed', event); + } + + private async revalidateActiveDocSubscriptions( + event: Events['sync.permissions.changed'] + ) { + const spacePrefix = `${event.spaceType}:${event.spaceId}:`; + const exactKey = event.docId + ? this.activeDocKey(event.spaceType, event.spaceId, event.docId) + : undefined; + const candidates = [...this.activeDocSockets.entries()].filter( + ([key]) => key === exactKey || (!exactKey && key.startsWith(spacePrefix)) ); - this.server.to(room026).emit('space:broadcast-doc-updates', payload); + + for (const [key, sockets] of candidates) { + const [, spaceId, docId] = key.split(':'); + for (const socket of Array.from(sockets)) { + const userId = this.resolvePresenceUserId(socket); + if (!userId) { + this.removeActiveDocSubscription( + socket, + event.spaceType, + spaceId, + docId + ); + continue; + } + + try { + this.assertUserdataSubject(event.spaceType, userId, spaceId, docId); + await this.assertDocActionAllowed( + event.spaceType, + userId, + spaceId, + docId, + 'Doc.Read' + ); + } catch { + this.removeActiveDocSubscription( + socket, + event.spaceType, + spaceId, + docId + ); + } + } + } + } + + private publishDocUpdate( + payload: SyncDocUpdatesPayload, + sourceSocket?: Socket + ) { + if (!this.server || payload.updates.length === 0) { + return; + } + + const legacyRoom = `${payload.spaceType}:${Room( + payload.spaceId, + 'sync-026' + )}`; + const broadcastPayload = this.buildBroadcastPayload( + payload.spaceType, + payload.spaceId, + payload.docId, + payload.updates, + payload.timestamp, + payload.editor + ); + if (sourceSocket) { + sourceSocket + .to(legacyRoom) + .emit('space:broadcast-doc-updates', broadcastPayload); + } else { + this.server + .to(legacyRoom) + .emit('space:broadcast-doc-updates', broadcastPayload); + } + + const batchRoom = `${payload.spaceType}:${Room( + payload.spaceId, + 'sync-027' + )}`; + const invalidation = { + spaceType: payload.spaceType, + spaceId: payload.spaceId, + timestamp: payload.timestamp, + }; + if (sourceSocket) { + sourceSocket + .to(batchRoom) + .emit('space:broadcast-doc-invalidation', invalidation); + } else { + this.server + .to(batchRoom) + .emit('space:broadcast-doc-invalidation', invalidation); + } + + this.emitActiveDocUpdate(payload, sourceSocket?.id, broadcastPayload); metrics.socketio .counter('doc_updates_broadcast') - .add(payload.updates.length, { - mode: payload.compressed ? 'compressed' : 'batch', + .add(broadcastPayload.updates.length, { + mode: broadcastPayload.compressed ? 'compressed' : 'batch', }); + this.event.broadcast('sync.doc.updates.pushed', { + ...payload, + updates: this.encodeUpdates(payload.updates), + }); } selectAdapter(client: Socket, spaceType: SpaceType): SyncSocketAdapter { @@ -611,21 +1047,140 @@ export class SpaceSyncGateway this.rejectJoin(client); return { data: { clientId: client.id, success: false } }; } + if (isBatchWsClientVersion(clientVersion)) { + this.rejectJoin(client); + return { data: { clientId: client.id, success: false } }; + } const adapter = this.selectAdapter(client, spaceType); await adapter.join(user.id, spaceId); + this.removeActiveDocSubscriptionsInSpace(client, spaceType, spaceId); - const protocolRoomType = getSyncProtocolRoomType(clientVersion); - const protocolRoom = adapter.room(spaceId, protocolRoomType); - const otherProtocolRoom = adapter.room( - spaceId, - protocolRoomType === 'sync-025' ? 'sync-026' : 'sync-025' - ); - if (client.rooms.has(otherProtocolRoom)) { - await client.leave(otherProtocolRoom); + const legacyRoom = adapter.room(spaceId, 'sync-026'); + const batchRoom = adapter.room(spaceId, 'sync-027'); + if (client.rooms.has(batchRoom)) { + await client.leave(batchRoom); } - if (!client.rooms.has(protocolRoom)) { - await client.join(protocolRoom); + if (!client.rooms.has(legacyRoom)) { + await client.join(legacyRoom); + } + + return { data: { clientId: client.id, success: true } }; + } + + @SubscribeMessage('space:join-batch') + async onJoinSpaceBatch( + @CurrentUser() user: CurrentUser, + @ConnectedSocket() client: Socket, + @MessageBody() message: unknown + ): Promise> { + const { spaces, clientVersion } = parseJoinSpaceBatchMessage(message); + if ( + !isSupportedWsClientVersion(clientVersion) || + !isBatchWsClientVersion(clientVersion) + ) { + this.rejectJoin(client); + return { data: { clientId: client.id, success: false } }; + } + + const [first] = spaces; + const adapter = this.selectAdapter(client, first.spaceType); + + // Authorize the whole batch before mutating any Socket.IO room. This is + // intentionally separate from SyncSocketAdapter.join(), which is also + // used by the legacy single-room handlers. + await adapter.assertAccessible(first.spaceId, user.id, 'Workspace.Sync'); + + for (const space of spaces) { + if (space.docId === undefined) { + continue; + } + this.assertUserdataSubject( + space.spaceType, + user.id, + space.spaceId, + space.docId + ); + await this.assertDocActionAllowed( + space.spaceType, + user.id, + space.spaceId, + space.docId, + 'Doc.Read' + ); + } + + const rooms = new Set(); + rooms.add(adapter.room(first.spaceId)); + rooms.add(adapter.room(first.spaceId, 'sync-027')); + const legacyRoom = adapter.room(first.spaceId, 'sync-026'); + + const roomsToJoin = [...rooms].filter(room => !client.rooms.has(room)); + const subscriptionsToAdd = spaces.filter( + (space): space is JoinSpaceBatchEntry & { docId: string } => + space.docId !== undefined && + !this.hasActiveDocSubscription( + client, + space.spaceType, + space.spaceId, + space.docId + ) + ); + try { + if (roomsToJoin.length > 0) { + await client.join(roomsToJoin); + } + for (const space of subscriptionsToAdd) { + this.addActiveDocSubscription( + client, + space.spaceType, + space.spaceId, + space.docId + ); + } + if (client.rooms.has(legacyRoom)) { + await client.leave(legacyRoom); + } + } catch (error) { + for (const space of subscriptionsToAdd) { + this.removeActiveDocSubscription( + client, + space.spaceType, + space.spaceId, + space.docId + ); + } + await Promise.all( + roomsToJoin + .filter(room => client.rooms.has(room)) + .map(async room => { + await client.leave(room); + }) + ); + throw error; + } + + return { data: { clientId: client.id, success: true } }; + } + + @SubscribeMessage('space:leave-batch') + async onLeaveSpaceBatch( + @ConnectedSocket() client: Socket, + @MessageBody() message: unknown + ): Promise> { + const { spaceType, spaceId, docIds } = parseLeaveSpaceBatchMessage(message); + for (const docId of docIds) { + this.removeActiveDocSubscription(client, spaceType, spaceId, docId); + } + + const activeDocs = this.activeSocketDocs.get(client.id); + const hasActiveDocsInSpace = Array.from(activeDocs ?? []).some(key => + key.startsWith(`${spaceType}:${spaceId}:`) + ); + if (docIds.length === 0 && !hasActiveDocsInSpace) { + const adapter = this.selectAdapter(client, spaceType); + await adapter.leave(spaceId, 'sync-027'); + await adapter.leave(spaceId); } return { data: { clientId: client.id, success: true } }; @@ -636,7 +1191,11 @@ export class SpaceSyncGateway @ConnectedSocket() client: Socket, @MessageBody() { spaceType, spaceId }: LeaveSpaceMessage ): Promise> { - await this.selectAdapter(client, spaceType).leave(spaceId); + const adapter = this.selectAdapter(client, spaceType); + this.removeActiveDocSubscriptionsInSpace(client, spaceType, spaceId); + await adapter.leave(spaceId); + await adapter.leave(spaceId, 'sync-026'); + await adapter.leave(spaceId, 'sync-027'); return { data: { clientId: client.id, success: true } }; } @@ -729,33 +1288,17 @@ export class SpaceSyncGateway user.id ); - const payload = this.buildBroadcastPayload( - spaceType, - spaceId, - docId, - [Buffer.from(update, 'base64')], - timestamp, - user.id - ); - client - .to(adapter.room(spaceId, 'sync-026')) - .emit('space:broadcast-doc-updates', payload); - metrics.socketio - .counter('doc_updates_broadcast') - .add(payload.updates.length, { - mode: payload.compressed ? 'compressed' : 'batch', - }); - - client - .to(adapter.room(spaceId, 'sync-025')) - .emit('space:broadcast-doc-update', { + this.publishDocUpdate( + { spaceType, spaceId, docId, - update, + updates: [Buffer.from(update, 'base64')], timestamp, editor: user.id, - } satisfies BroadcastDocUpdateMessage); + }, + client + ); return { data: { @@ -813,6 +1356,10 @@ export class SpaceSyncGateway this.rejectJoin(client); return { data: { clientId: client.id, success: false } }; } + if (isBatchWsClientVersion(clientVersion)) { + this.rejectJoin(client); + return { data: { clientId: client.id, success: false } }; + } await this.selectAdapter(client, spaceType).join( user.id, @@ -845,11 +1392,18 @@ export class SpaceSyncGateway ) { const adapter = this.selectAdapter(client, spaceType); - const roomType = `${docId}:awareness` as const; - adapter.assertIn(spaceId, roomType); - client - .to(adapter.room(spaceId, roomType)) - .emit('space:collect-awareness', { spaceType, spaceId, docId }); + if (this.hasActiveDocSubscription(client, spaceType, spaceId, docId)) { + adapter.assertIn(spaceId); + const event = { spaceType, spaceId, docId, sourceSocketId: client.id }; + this.emitActiveAwarenessCollect(event); + this.event.broadcast('sync.awareness.collect', event); + } else { + const roomType = `${docId}:awareness` as const; + adapter.assertIn(spaceId, roomType); + client + .to(adapter.room(spaceId, roomType)) + .emit('space:collect-awareness', { spaceType, spaceId, docId }); + } return { data: { clientId: client.id } }; } @@ -862,11 +1416,18 @@ export class SpaceSyncGateway const { spaceType, spaceId, docId } = message; const adapter = this.selectAdapter(client, spaceType); - const roomType = `${docId}:awareness` as const; - adapter.assertIn(spaceId, roomType); - client - .to(adapter.room(spaceId, roomType)) - .emit('space:broadcast-awareness-update', message); + if (this.hasActiveDocSubscription(client, spaceType, spaceId, docId)) { + adapter.assertIn(spaceId); + const event = { ...message, sourceSocketId: client.id }; + this.emitActiveAwarenessUpdate(event); + this.event.broadcast('sync.awareness.updated', event); + } else { + const roomType = `${docId}:awareness` as const; + adapter.assertIn(spaceId, roomType); + client + .to(adapter.room(spaceId, roomType)) + .emit('space:broadcast-awareness-update', message); + } return {}; } diff --git a/packages/backend/server/src/core/workspaces/resolvers/doc.ts b/packages/backend/server/src/core/workspaces/resolvers/doc.ts index a51edd845e..6668c05422 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/doc.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/doc.ts @@ -44,7 +44,6 @@ import { type DotToUnderline, mapPermissionsToGraphqlPermissions, PermissionAccess, - PermissionService, } from '../../permission'; import { PublicUserType, WorkspaceUserType } from '../../user'; import { canUserExecuteLimitedActions } from '../abuse'; @@ -300,7 +299,6 @@ export class WorkspaceDocResolver { */ private readonly prisma: PrismaClient, private readonly ac: PermissionAccess, - private readonly permission: PermissionService, private readonly models: Models, private readonly cache: Cache, private readonly event: EventBus, @@ -409,12 +407,14 @@ export class WorkspaceDocResolver { @Parent() workspace: WorkspaceType, @Args('pagination', PaginationInput.decode) pagination: PaginationInput ): Promise { - const predicate = this.permission.docReadableSqlPredicate({ - userId: me.id, - workspaceId: workspace.id, - action: 'Doc.Read', - docIdColumn: Prisma.raw('"workspace_pages"."page_id"'), - }); + const readable = await this.runtime.filterReadableDocs( + me.id, + workspace.id, + await this.models.doc.listWorkspaceDocIds(workspace.id) + ); + const predicate = readable.length + ? Prisma.sql`"workspace_pages"."page_id" IN (${Prisma.join(readable)})` + : Prisma.sql`FALSE`; const [count, rows] = await this.models.doc.paginateDocInfoByUpdatedAt( workspace.id, pagination, diff --git a/packages/backend/server/src/data/__tests__/migrations.spec.ts b/packages/backend/server/src/data/__tests__/migrations.spec.ts index 622540f94d..65115f320e 100644 --- a/packages/backend/server/src/data/__tests__/migrations.spec.ts +++ b/packages/backend/server/src/data/__tests__/migrations.spec.ts @@ -1,12 +1,16 @@ +import { randomUUID } from 'node:crypto'; + import { ModuleRef } from '@nestjs/core'; import { PrismaClient } from '@prisma/client'; import ava, { TestFn } from 'ava'; import { createTestingModule, type TestingModule } from '../../__tests__/utils'; +import { BackendRuntimeProvider } from '../../core/backend-runtime'; import { Models } from '../../models'; import { BackfillPermissionProjection1765500000000 } from '../migrations/1765500000000-backfill-permission-projection'; import { BackfillTranscriptStorageKeys1786805802350 } from '../migrations/1786805802350-backfill-transcript-storage-keys'; import { ConvergeManagedProviderProfiles1786810000000 } from '../migrations/1786810000000-converge-managed-provider-profiles'; +import { MigrateLegacyContextBlobArtifacts1786820000000 } from '../migrations/1786820000000-migrate-legacy-context-blob-artifacts'; interface Context { module: TestingModule; @@ -207,3 +211,107 @@ test('managed provider migration preserves explicit profiles and converts legacy }) ); }); + +test('legacy context blob migration admits each blob once through the artifact runtime', async t => { + const user = await t.context.models.user.create({ + email: 'legacy-context@affine.pro', + }); + const workspace = await t.context.db.workspace.create({ + data: { accessPolicy: { create: {} } }, + }); + const session = await t.context.db.aiSession.create({ + data: { + userId: user.id, + workspaceId: workspace.id, + promptName: 'copilot', + }, + }); + const blobId = 'legacy-context-blob'; + const legacyTable = await t.context.db.$queryRaw<{ exists: boolean }[]>` + SELECT to_regclass('public.ai_contexts') IS NOT NULL AS exists + `; + const createdLegacyTable = !legacyTable[0]?.exists; + if (createdLegacyTable) { + const ref = { + get() { + throw new Error( + 'legacy context runtime should not be resolved without source tables' + ); + }, + } as unknown as ModuleRef; + await MigrateLegacyContextBlobArtifacts1786820000000.up(t.context.db, ref); + } + if (createdLegacyTable) { + await t.context.db.$executeRaw` + CREATE TABLE ai_contexts ( + id VARCHAR PRIMARY KEY, + session_id VARCHAR NOT NULL, + config JSON NOT NULL, + created_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ(3) NOT NULL + ) + `; + } + await t.context.db.blob.create({ + data: { + workspaceId: workspace.id, + key: blobId, + size: 12, + mime: 'text/plain', + status: 'completed', + }, + }); + await t.context.db.$executeRaw` + INSERT INTO ai_contexts (id, session_id, config, created_at, updated_at) + VALUES (${randomUUID()}, ${session.id}, ${JSON.stringify({ blobs: [blobId] })}::jsonb, now(), now()) + `; + + const calls: Array<{ + workspaceId: string; + blobId: string; + mimeType: string; + libraryOwned?: boolean; + }> = []; + const runtime = { + async ensureWorkspaceBlobArtifact(input: (typeof calls)[number]) { + calls.push(input); + await t.context.db.$executeRaw` + INSERT INTO workspace_artifacts ( + id, workspace_id, content_hash, canonical_media_type, size_bytes, + storage_scope, storage_key, status, ready_at + ) VALUES ( + ${randomUUID()}::uuid, ${input.workspaceId}, ${`hash-${input.blobId}`}, + ${input.mimeType}, 12, 'blob', + ${`${input.workspaceId}/${input.blobId}`}, 'ready', now() + ) + `; + return {}; + }, + }; + const ref = { + get(token: unknown) { + if (token === BackendRuntimeProvider) { + return runtime; + } + throw new Error('unexpected migration dependency'); + }, + } as unknown as ModuleRef; + + try { + await MigrateLegacyContextBlobArtifacts1786820000000.up(t.context.db, ref); + await MigrateLegacyContextBlobArtifacts1786820000000.up(t.context.db, ref); + } finally { + if (createdLegacyTable) { + await t.context.db.$executeRaw`DROP TABLE ai_contexts`; + } + } + + t.deepEqual(calls, [ + { + workspaceId: workspace.id, + blobId, + mimeType: 'text/plain', + libraryOwned: false, + }, + ]); +}); diff --git a/packages/backend/server/src/data/app.ts b/packages/backend/server/src/data/app.ts index 994c5999e7..2933700b9a 100644 --- a/packages/backend/server/src/data/app.ts +++ b/packages/backend/server/src/data/app.ts @@ -1,13 +1,12 @@ import { Module } from '@nestjs/common'; import { FunctionalityModules } from '../app.module'; -import { IndexerModule } from '../plugins/indexer'; import { CreateCommand } from './commands/create'; import { ImportConfigCommand } from './commands/import'; import { RevertCommand, RunCommand } from './commands/run'; @Module({ - imports: [...FunctionalityModules, IndexerModule], + imports: FunctionalityModules, providers: [CreateCommand, RunCommand, RevertCommand, ImportConfigCommand], }) export class CliAppModule {} diff --git a/packages/backend/server/src/data/commands/run.ts b/packages/backend/server/src/data/commands/run.ts index 2b047df6da..6a06c931a0 100644 --- a/packages/backend/server/src/data/commands/run.ts +++ b/packages/backend/server/src/data/commands/run.ts @@ -3,6 +3,8 @@ import { ModuleRef } from '@nestjs/core'; import { PrismaClient } from '@prisma/client'; import { once } from 'lodash-es'; +import { BackendRuntimeProvider } from '../../core/backend-runtime'; +import { StorageRuntimeProvider } from '../../core/storage-runtime'; import * as migrationImports from '../migrations'; interface Migration { @@ -13,6 +15,9 @@ interface Migration { order: number; } +const LEGACY_CONTEXT_BLOB_ARTIFACT_MIGRATION = + 'MigrateLegacyContextBlobArtifacts1786820000000'; + export const collectMigrations = once(() => { const migrations = Object.values(migrationImports).map(migration => { const order = Number(migration.name.match(/([\d]+)$/)?.[1]); @@ -43,6 +48,12 @@ export class RunCommand { ) {} async execute(): Promise { + await this.injector + .get(BackendRuntimeProvider, { strict: false }) + .runMigrations(); + await this.injector + .get(StorageRuntimeProvider, { strict: false }) + .runMigrations(); const migrations = collectMigrations(); const done: Migration[] = []; for (const migration of migrations) { @@ -85,6 +96,33 @@ export class RunCommand { await this.runMigration(migration); } + async admitLegacyContextBlobs(): Promise { + const tables = await this.db.$queryRaw< + Array<{ + contexts: string | null; + sessions: string | null; + blobs: string | null; + artifacts: string | null; + }> + >` + SELECT + to_regclass('public.ai_contexts')::text AS contexts, + to_regclass('public.ai_sessions_metadata')::text AS sessions, + to_regclass('public.blobs')::text AS blobs, + to_regclass('public.workspace_artifacts')::text AS artifacts + `; + + const schemaExists = Object.values(tables[0] ?? {}).every(Boolean); + if (!schemaExists) { + this.logger.log( + 'Skipping legacy context blob admission because its source schema is not present.' + ); + return; + } + + await this.runOne(LEGACY_CONTEXT_BLOB_ARTIFACT_MIGRATION); + } + private async runMigration(migration: Migration) { this.logger.log(`Running ${migration.name}...`); const record = await this.db.dataMigration.upsert({ diff --git a/packages/backend/server/src/data/migrations/1745211351719-create-indexer-tables.ts b/packages/backend/server/src/data/migrations/1745211351719-create-indexer-tables.ts deleted file mode 100644 index 6cbd29930c..0000000000 --- a/packages/backend/server/src/data/migrations/1745211351719-create-indexer-tables.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { ModuleRef } from '@nestjs/core'; -import { PrismaClient } from '@prisma/client'; - -import { IndexerService } from '../../plugins/indexer'; - -export class CreateIndexerTables1745211351719 { - static always = true; - - // do the migration - static async up(_db: PrismaClient, ref: ModuleRef) { - await ref.get(IndexerService, { strict: false }).createTables(); - } - - // revert the migration - static async down(_db: PrismaClient) {} -} diff --git a/packages/backend/server/src/data/migrations/1763800000000-rebuild-manticore-mixed-script-indexes.ts b/packages/backend/server/src/data/migrations/1763800000000-rebuild-manticore-mixed-script-indexes.ts deleted file mode 100644 index 627aa18100..0000000000 --- a/packages/backend/server/src/data/migrations/1763800000000-rebuild-manticore-mixed-script-indexes.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { ModuleRef } from '@nestjs/core'; -import { PrismaClient } from '@prisma/client'; - -import { IndexerService } from '../../plugins/indexer'; - -export class RebuildManticoreMixedScriptIndexes1763800000000 { - static async up(_db: PrismaClient, ref: ModuleRef) { - await ref.get(IndexerService, { strict: false }).rebuildManticoreIndexes(); - } - - static async down(_db: PrismaClient) {} -} diff --git a/packages/backend/server/src/data/migrations/1786820000000-migrate-legacy-context-blob-artifacts.ts b/packages/backend/server/src/data/migrations/1786820000000-migrate-legacy-context-blob-artifacts.ts new file mode 100644 index 0000000000..c5c1e68528 --- /dev/null +++ b/packages/backend/server/src/data/migrations/1786820000000-migrate-legacy-context-blob-artifacts.ts @@ -0,0 +1,79 @@ +import { ModuleRef } from '@nestjs/core'; +import { PrismaClient } from '@prisma/client'; + +import { BackendRuntimeProvider } from '../../core/backend-runtime'; + +type LegacyContextBlob = { + workspaceId: string; + blobId: string; + mimeType: string; +}; + +/** + * Convert the last product-owned references to workspace blobs into the + * artifact retention fact before the legacy context tables are removed. + * + * The runtime admission path is intentional here: a database row alone does + * not prove that the object still exists or that its metadata is truthful. + */ +export class MigrateLegacyContextBlobArtifacts1786820000000 { + static async up(db: PrismaClient, injector: ModuleRef) { + const tables = await db.$queryRaw< + Array<{ + contexts: string | null; + sessions: string | null; + blobs: string | null; + artifacts: string | null; + }> + >` + SELECT + to_regclass('public.ai_contexts')::text AS contexts, + to_regclass('public.ai_sessions_metadata')::text AS sessions, + to_regclass('public.blobs')::text AS blobs, + to_regclass('public.workspace_artifacts')::text AS artifacts + `; + + if (!Object.values(tables[0] ?? {}).every(Boolean)) { + return; + } + + const runtime = injector.get(BackendRuntimeProvider, { strict: false }); + const blobs = await db.$queryRaw` + SELECT DISTINCT + session.workspace_id AS "workspaceId", + blob.key AS "blobId", + blob.mime AS "mimeType" + FROM ai_contexts context + JOIN ai_sessions_metadata session ON session.id = context.session_id + JOIN blobs blob + ON blob.workspace_id = session.workspace_id + AND blob.deleted_at IS NULL + AND blob.status = 'completed' + WHERE jsonb_path_exists( + context.config::jsonb, + '$.** ? (@ == $blobKey)', + jsonb_build_object('blobKey', to_jsonb(blob.key::text)) + ) + AND NOT EXISTS ( + SELECT 1 + FROM workspace_artifacts artifact + WHERE artifact.workspace_id = session.workspace_id + AND artifact.storage_scope = 'blob' + AND artifact.storage_key = concat(session.workspace_id, '/', blob.key) + AND artifact.status = 'ready' + ) + ORDER BY session.workspace_id, blob.key + `; + + for (const blob of blobs) { + await runtime.ensureWorkspaceBlobArtifact({ + workspaceId: blob.workspaceId, + blobId: blob.blobId, + mimeType: blob.mimeType, + libraryOwned: false, + }); + } + } + + static async down(_db: PrismaClient, _injector: ModuleRef) {} +} diff --git a/packages/backend/server/src/data/migrations/index.ts b/packages/backend/server/src/data/migrations/index.ts index 452379ca72..de6430a519 100644 --- a/packages/backend/server/src/data/migrations/index.ts +++ b/packages/backend/server/src/data/migrations/index.ts @@ -1,10 +1,9 @@ export * from './1698398506533-guid'; export * from './1703756315970-unamed-account'; export * from './1721299086340-refresh-unnamed-user'; -export * from './1745211351719-create-indexer-tables'; export * from './1751966744168-correct-session-update-time'; -export * from './1763800000000-rebuild-manticore-mixed-script-indexes'; export * from './1765500000000-backfill-permission-projection'; export * from './1765600000000-backfill-entitlement-projection'; export * from './1786805802350-backfill-transcript-storage-keys'; export * from './1786810000000-converge-managed-provider-profiles'; +export * from './1786820000000-migrate-legacy-context-blob-artifacts'; diff --git a/packages/backend/server/src/env.ts b/packages/backend/server/src/env.ts index a0e44cd9d8..24beb76206 100644 --- a/packages/backend/server/src/env.ts +++ b/packages/backend/server/src/env.ts @@ -24,10 +24,17 @@ export enum Flavor { Sync = 'sync', Renderer = 'renderer', Front = 'front', - Doc = 'doc', + Worker = 'worker', Script = 'script', } +export enum ServerRole { + Frontend = 'frontend', + Api = 'api', + Worker = 'worker', + AllInOne = 'allinone', +} + export enum Namespace { Dev = 'dev', Beta = 'beta', @@ -101,6 +108,39 @@ export class Env implements AppEnv { return this.DEPLOYMENT_TYPE === DeploymentType.Selfhosted; } + get role(): ServerRole | undefined { + switch (this.FLAVOR) { + case Flavor.AllInOne: + return ServerRole.AllInOne; + case Flavor.Graphql: + return ServerRole.Api; + case Flavor.Worker: + return ServerRole.Worker; + case Flavor.Front: + case Flavor.Sync: + case Flavor.Renderer: + return ServerRole.Frontend; + case Flavor.Script: + return undefined; + } + } + + get isApi() { + return this.FLAVOR === Flavor.Graphql || this.FLAVOR === Flavor.AllInOne; + } + + get isWorker() { + return this.FLAVOR === Flavor.Worker || this.FLAVOR === Flavor.AllInOne; + } + + get isFrontend() { + return ( + this.FLAVOR === Flavor.Front || + this.FLAVOR === Flavor.Sync || + this.FLAVOR === Flavor.Renderer + ); + } + isFlavor(flavor: Flavor) { return this.FLAVOR === flavor || this.FLAVOR === Flavor.AllInOne; } @@ -111,7 +151,7 @@ export class Env implements AppEnv { sync: this.isFlavor(Flavor.Sync), renderer: this.isFlavor(Flavor.Renderer), front: this.FLAVOR === Flavor.Front, - doc: this.isFlavor(Flavor.Doc), + worker: this.isFlavor(Flavor.Worker), // Script in a special flavor, return true only when it is set explicitly script: this.FLAVOR === Flavor.Script, }; diff --git a/packages/backend/server/src/models/__tests__/comment.spec.ts b/packages/backend/server/src/models/__tests__/comment.spec.ts index 10ce7d587b..a78b57ecf8 100644 --- a/packages/backend/server/src/models/__tests__/comment.spec.ts +++ b/packages/backend/server/src/models/__tests__/comment.spec.ts @@ -100,6 +100,7 @@ test('should update a comment', async t => { userId: owner.id, }); + await waitNextMillisecond(); const comment2 = await models.comment.update({ id: comment1.id, content: { diff --git a/packages/backend/server/src/models/copilot-transcript-task.ts b/packages/backend/server/src/models/copilot-transcript-task.ts index d6e4556ef5..6a2daf47b7 100644 --- a/packages/backend/server/src/models/copilot-transcript-task.ts +++ b/packages/backend/server/src/models/copilot-transcript-task.ts @@ -125,23 +125,6 @@ export class CopilotTranscriptTaskModel extends BaseModel { return count === 1; } - async adoptLegacyDispatch( - id: string, - actionRunId: string | null, - dispatchGeneration: string - ) { - const { count } = await this.db.aiTranscriptTask.updateMany({ - where: { - id, - status: 'pending', - dispatchGeneration: null, - actionRunId, - }, - data: { dispatchGeneration }, - }); - return count === 1; - } - async attachActionRun( id: string, dispatchGeneration: string, diff --git a/packages/backend/server/src/models/doc.ts b/packages/backend/server/src/models/doc.ts index 6f9641e0db..b8a9d15a63 100644 --- a/packages/backend/server/src/models/doc.ts +++ b/packages/backend/server/src/models/doc.ts @@ -803,6 +803,14 @@ export class DocModel extends BaseModel { ] as const; } + async listWorkspaceDocIds(workspaceId: string) { + const rows = await this.db.workspaceDoc.findMany({ + where: { workspaceId }, + select: { docId: true }, + }); + return rows.map(row => row.docId); + } + async findEmptySummaryDocIds(workspaceId: string) { const rows = await this.db.workspaceDoc.findMany({ where: { diff --git a/packages/backend/server/src/native.ts b/packages/backend/server/src/native.ts index 72051e0af6..6628d231f4 100644 --- a/packages/backend/server/src/native.ts +++ b/packages/backend/server/src/native.ts @@ -45,6 +45,7 @@ import serverNativeModule, { type RemoteMimeTypeRequest, type ResolvedEntitlement, type ResolveEntitlementInput, + type RuntimeAggregateRequest, type RuntimeBlobCleanupExecuteResult, type RuntimeBlobCleanupPlanResult, type RuntimeBlobCleanupResult, @@ -64,6 +65,8 @@ import serverNativeModule, { type RuntimeObjectStoragePutOptions, type RuntimePresignedObjectRequest, type RuntimeRetrievalScope, + type RuntimeSearchQuery, + type RuntimeSearchRequest, type RuntimeTurnScopeSnapshot, type RuntimeVerificationTokenRecord, type RuntimeWorkspaceArtifact, @@ -144,6 +147,7 @@ export type { RemoteMimeTypeRequest, ResolvedEntitlement, ResolveEntitlementInput, + RuntimeAggregateRequest, RuntimeBlobCleanupExecuteResult, RuntimeBlobCleanupPlanResult, RuntimeBlobCleanupResult, @@ -163,6 +167,8 @@ export type { RuntimeObjectStoragePutOptions, RuntimePresignedObjectRequest, RuntimeRetrievalScope, + RuntimeSearchQuery, + RuntimeSearchRequest, RuntimeTurnScopeSnapshot, RuntimeVerificationTokenRecord, RuntimeWorkspaceArtifact, diff --git a/packages/backend/server/src/plugins/calendar/__tests__/service.spec.ts b/packages/backend/server/src/plugins/calendar/__tests__/service.spec.ts index d6aeb406c8..d42127c104 100644 --- a/packages/backend/server/src/plugins/calendar/__tests__/service.spec.ts +++ b/packages/backend/server/src/plugins/calendar/__tests__/service.spec.ts @@ -479,6 +479,32 @@ test('syncSubscription invalidates account when refresh token is invalid', async t.is(events.length, 0); }); +test('syncSubscription does not disable a calendar when token refresh returns 404', async t => { + const user = await module.create(Mockers.User); + const account = await createAccount(user.id, { + accessToken: 'expired-access-token', + expiresAt: new Date(Date.now() - 5 * 60 * 1000), + }); + const subscription = await createSubscription(account.id, { + syncToken: 'sync-token', + }); + + const provider = new MockCalendarProvider(); + mock.method(provider, 'refreshTokens', async () => { + throw new CalendarProviderRequestError({ + status: 404, + message: 'Token endpoint not found', + }); + }); + mock.method(providerFactory, 'get', () => provider); + + await calendarService.syncSubscription(subscription.id); + + const updated = await models.calendarSubscription.get(subscription.id); + t.is(updated?.enabled, true); + t.is(updated?.syncRetryCount, 1); +}); + test('syncSubscription disables subscription on provider 404', async t => { const user = await module.create(Mockers.User); const account = await createAccount(user.id); @@ -673,6 +699,44 @@ test('syncSubscription renews webhook channel when expiring', async t => { t.truthy(updated?.channelExpiration); }); +test('syncSubscription replaces a webhook channel that is already gone', async t => { + const user = await module.create(Mockers.User); + const account = await createAccount(user.id); + const subscription = await createSubscription(account.id, { + syncToken: 'sync-token', + customChannelId: 'missing-channel', + customResourceId: 'missing-resource', + channelExpiration: new Date(Date.now() + 60 * 60 * 1000), + }); + + const provider = new MockCalendarProvider(); + mock.method(provider, 'listEvents', async () => ({ + events: [], + nextSyncToken: 'next-sync', + })); + const stopMock = mock.method(provider, 'stopChannel', async () => { + throw new CalendarProviderRequestError({ + status: 404, + message: 'Channel not found', + }); + }); + const watchMock = mock.method(provider, 'watchCalendar', async () => ({ + channelId: 'replacement-channel', + resourceId: 'replacement-resource', + expiration: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000), + })); + mock.method(providerFactory, 'get', () => provider); + + await calendarService.syncSubscription(subscription.id); + + t.is(stopMock.mock.callCount(), 1); + t.is(watchMock.mock.callCount(), 1); + const updated = await models.calendarSubscription.get(subscription.id); + t.is(updated?.customChannelId, 'replacement-channel'); + t.is(updated?.customResourceId, 'replacement-resource'); + t.is(updated?.syncRetryCount, 0); +}); + test('syncSubscription falls back to polling when push is unsupported', async t => { const user = await module.create(Mockers.User); const account = await createAccount(user.id); diff --git a/packages/backend/server/src/plugins/calendar/service.ts b/packages/backend/server/src/plugins/calendar/service.ts index fb44d64283..09d105213f 100644 --- a/packages/backend/server/src/plugins/calendar/service.ts +++ b/packages/backend/server/src/plugins/calendar/service.ts @@ -403,6 +403,7 @@ export class CalendarService { account, provider, accessToken, + disableOnNotFound: true, }); return; } @@ -413,6 +414,7 @@ export class CalendarService { account, provider, accessToken, + disableOnNotFound: true, }); return; } @@ -933,10 +935,22 @@ export class CalendarService { subscription.customChannelId && subscription.customResourceId ) { - await provider.stopChannel({ - accessToken, - channelId: subscription.customChannelId, - resourceId: subscription.customResourceId, + try { + await provider.stopChannel({ + accessToken, + channelId: subscription.customChannelId, + resourceId: subscription.customResourceId, + }); + } catch (error) { + if (!this.isNotFoundError(error)) { + throw error; + } + } + + await this.models.calendarSubscription.updateChannel(subscription.id, { + customChannelId: null, + customResourceId: null, + channelExpiration: null, }); } @@ -986,8 +1000,9 @@ export class CalendarService { account: CalendarAccount; provider: CalendarProvider; accessToken?: string; + disableOnNotFound?: boolean; }) { - if (this.isSubscriptionMissingError(params.error)) { + if (params.disableOnNotFound && this.isNotFoundError(params.error)) { await this.disableSubscription({ subscriptionId: params.subscription.id, provider: params.provider, @@ -1021,7 +1036,7 @@ export class CalendarService { ); } - private isSubscriptionMissingError(error: unknown) { + private isNotFoundError(error: unknown) { if (!(error instanceof CalendarProviderRequestError)) { return false; } diff --git a/packages/backend/server/src/plugins/copilot/retrieval/document.ts b/packages/backend/server/src/plugins/copilot/retrieval/document.ts index 24241589d5..5ca0fd1493 100644 --- a/packages/backend/server/src/plugins/copilot/retrieval/document.ts +++ b/packages/backend/server/src/plugins/copilot/retrieval/document.ts @@ -1,6 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; -import { Config, SearchProviderNotFound } from '../../../base'; +import { SearchProviderNotFound } from '../../../base'; import { PermissionAccess } from '../../../core/permission'; import type { DocVisibility } from '../../../core/utils/blocksuite'; import { type DocChunkSimilarity, Models } from '../../../models'; @@ -70,7 +70,6 @@ function hasVectorProjectionMetadata(hit: DocChunkSimilarity) { @Injectable() export class DocumentRetrievalService { constructor( - private readonly config: Config, private readonly ac: PermissionAccess, private readonly indexer: IndexerService, @Inject(DOCUMENT_VECTOR_SEARCH) @@ -96,17 +95,15 @@ export class DocumentRetrievalService { byokLeaseId: options.byokLeaseId, }; const [lexicalAttempt, vectorAttempt] = await Promise.allSettled([ - this.config.indexer.enabled - ? this.indexer - .searchDocsByKeyword(workspaceId, query, { - limit: Math.max(limit * 3, 20), - docIds, - }) - .catch(error => { - if (error instanceof SearchProviderNotFound) return null; - throw error; - }) - : null, + this.indexer + .searchDocsByKeyword(userId, workspaceId, query, { + limit: Math.max(limit * 3, 20), + docIds, + }) + .catch(error => { + if (error instanceof SearchProviderNotFound) return null; + throw error; + }), this.context.canEmbedding ? this.context.matchWorkspaceDocCandidates( workspaceId, diff --git a/packages/backend/server/src/plugins/copilot/runtime/turn-orchestrator.ts b/packages/backend/server/src/plugins/copilot/runtime/turn-orchestrator.ts index f65333d339..cc73c83596 100644 --- a/packages/backend/server/src/plugins/copilot/runtime/turn-orchestrator.ts +++ b/packages/backend/server/src/plugins/copilot/runtime/turn-orchestrator.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; -import { BackendRuntimeEmbeddingJob } from '../../../core/backend-runtime'; +import { BackendRuntimeEmbeddingService } from '../../../core/backend-runtime'; import { type Turn } from '../core'; import { type ModelConditions, @@ -23,7 +23,7 @@ export class TurnOrchestrator { private readonly runtime: CapabilityRuntime, private readonly imageResults: ImageResultHost, private readonly turnPersistence: TurnPersistence, - private readonly embeddings: BackendRuntimeEmbeddingJob + private readonly embeddings: BackendRuntimeEmbeddingService ) {} private buildPromptParams(latestTurn?: Turn): Record { diff --git a/packages/backend/server/src/plugins/copilot/transcript/service.ts b/packages/backend/server/src/plugins/copilot/transcript/service.ts index 7103a7ac42..0b3d3b9859 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/service.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/service.ts @@ -512,7 +512,7 @@ export class CopilotTranscriptionService { async transcriptTask({ taskId, payload, - generation: queuedGeneration, + generation, retryOf, }: Jobs['copilot.transcript.task.submit']) { const task = await this.models.copilotTranscriptTask.get(taskId); @@ -520,17 +520,6 @@ export class CopilotTranscriptionService { throw new CopilotTranscriptionJobNotFound(); } let actionRunId = retryOf ?? null; - const generation = queuedGeneration ?? randomUUID(); - if ( - !queuedGeneration && - !(await this.models.copilotTranscriptTask.adoptLegacyDispatch( - taskId, - actionRunId, - generation - )) - ) { - return; - } const claimed = await this.models.copilotTranscriptTask.claimDispatch( taskId, generation, diff --git a/packages/backend/server/src/plugins/copilot/transcript/types.ts b/packages/backend/server/src/plugins/copilot/transcript/types.ts index e4d69ca2ce..176e04e9a2 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/types.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/types.ts @@ -53,7 +53,7 @@ declare global { 'copilot.transcript.task.submit': { taskId: string; payload: TranscriptionPayloadV2; - generation?: string; + generation: string; retryOf?: string; }; } diff --git a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json b/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json deleted file mode 100644 index 9583934ef0..0000000000 --- a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json +++ /dev/null @@ -1,13 +0,0 @@ -{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId1", "content" : "title1 hello, 这是一段包含中文的标题，hello 你好😄", "flavour" : "title", "blob" : "blob1", "ref_doc_id" : "refDocId1", "ref" : "ref1", "parent_flavour" : "parentFlavour1", "parent_block_id" : "parentBlockId1", "additional" : "additional1", "markdown_preview" : "markdownPreview1", "created_by_user_id" : "userId1", "updated_by_user_id" : "userId1", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-10T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId2", "content" : "title2 world, test searching morphology", "flavour" : "flavour2", "blob" : "blob2", "ref_doc_id" : "refDocId2", "ref" : "ref2", "parent_flavour" : "parentFlavour2", "parent_block_id" : "parentBlockId2", "additional" : "additional2", "markdown_preview" : "markdownPreview2", "created_by_user_id" : "userId2", "updated_by_user_id" : "userId2", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId3", "content" : "title3 hello update", "flavour" : "flavour3", "blob" : "blob3", "ref_doc_id" : "refDocId3", "ref" : "ref3", "parent_flavour" : "parentFlavour3", "parent_block_id" : "parentBlockId3", "additional" : "additional3", "markdown_preview" : "markdownPreview3", "created_by_user_id" : "userId3", "updated_by_user_id" : "userId3", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-09T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId4", "content" : "title4 hello", "flavour" : "flavour4", "blob" : "blob4", "ref_doc_id" : "refDocId4", "ref" : "ref4", "parent_flavour" : "parentFlavour4", "parent_block_id" : "parentBlockId4", "additional" : "additional4", "markdown_preview" : "markdownPreview4", "created_by_user_id" : "userId4", "updated_by_user_id" : "userId4", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId5", "content" : "title5 hello", "flavour" : "flavour5", "blob" : "blob5", "ref_doc_id" : "refDocId5", "ref" : "ref5", "parent_flavour" : "parentFlavour5", "parent_block_id" : "parentBlockId5", "additional" : "additional5", "markdown_preview" : "markdownPreview5", "created_by_user_id" : "userId5", "updated_by_user_id" : "userId5", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId6", "content" : "title6 hello", "flavour" : "flavour6", "blob" : "blob6", "ref_doc_id" : "refDocId6", "ref" : "ref6", "parent_flavour" : "parentFlavour6", "parent_block_id" : "parentBlockId6", "additional" : "additional6", "markdown_preview" : "markdownPreview6", "created_by_user_id" : "userId6", "updated_by_user_id" : "userId6", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId2", "doc_id" : "docId1", "block_id" : "blockId7", "content" : "title7 hello", "flavour" : "flavour7", "blob" : "blob7", "ref_doc_id" : "refDocId7", "ref" : "ref7", "parent_flavour" : "parentFlavour7", "parent_block_id" : "parentBlockId7", "additional" : "additional7", "markdown_preview" : "markdownPreview7", "created_by_user_id" : "userId7", "updated_by_user_id" : "userId7", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId9", "block_id" : "blockId9", "content" : "title9 hello affine issue hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour9", "parent_block_id" : "parentBlockId9", "additional" : "additional9", "markdown_preview" : "markdownPreview9", "created_by_user_id" : "userId9", "updated_by_user_id" : "userId9", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId10", "content" : "this is docId2 title content hello", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour10", "parent_block_id" : "parentBlockId10", "additional" : "additional10", "markdown_preview" : "markdownPreview10", "created_by_user_id" : "userId10", "updated_by_user_id" : "userId10", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId11", "content" : "this is docId2 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour11", "parent_block_id" : "parentBlockId11", "additional" : "additional11", "markdown_preview" : "markdownPreview11", "created_by_user_id" : "userId11", "updated_by_user_id" : "userId11", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId12", "content" : "this is docId2 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour12", "parent_block_id" : "parentBlockId12", "additional" : "additional12", "markdown_preview" : "markdownPreview12", "created_by_user_id" : "userId12", "updated_by_user_id" : "userId12", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId3", "block_id" : "blockId13", "content" : "this is docId3 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour13", "parent_block_id" : "parentBlockId13", "additional" : "additional13", "markdown_preview" : "markdownPreview13", "created_by_user_id" : "userId13", "updated_by_user_id" : "userId13", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId3", "block_id" : "blockId14", "content" : "this is docId3 title content world", "flavour" : "affine:database", "parent_flavour": "affine:database", "parent_block_id" : "parentBlockId14", "additional" : "additional14", "markdown_preview" : "markdownPreview14", "created_by_user_id" : "userId14", "updated_by_user_id" : "userId14", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"} diff --git a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json b/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json deleted file mode 100644 index e252622d95..0000000000 --- a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json +++ /dev/null @@ -1,11 +0,0 @@ -{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "title" : "title1 hello, 这是一段包含中文的标题，hello 你好😄", "summary" : "summary1", "journal" : "journal1", "created_by_user_id" : "userId1", "updated_by_user_id" : "userId1", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-10T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "title" : "title2 world, test searching morphology", "summary" : "summary2", "journal" : "journal2", "created_by_user_id" : "userId2", "updated_by_user_id" : "userId2", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId3", "title" : "title3 hello update", "summary" : "summary3", "journal" : "journal3", "created_by_user_id" : "userId3", "updated_by_user_id" : "userId3", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-09T06:04:13.278Z"} -{"workspace_id" : "workspaceId2", "doc_id" : "docId4", "title" : "title4 hello", "summary" : "summary4", "journal" : "journal4", "created_by_user_id" : "userId4", "updated_by_user_id" : "userId4", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId5", "title" : "title5 hello", "summary" : "summary5", "journal" : "journal5", "created_by_user_id" : "userId5", "updated_by_user_id" : "userId5", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId6", "title" : "title6 hello", "summary" : "summary6", "journal" : "journal6", "created_by_user_id" : "userId6", "updated_by_user_id" : "userId6", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId7", "title" : "title7 hello", "summary" : "summary7", "journal" : "journal7", "created_by_user_id" : "userId7", "updated_by_user_id" : "userId7", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId8", "title" : "title8 hello", "summary" : "summary8", "journal" : "journal8", "created_by_user_id" : "userId8", "updated_by_user_id" : "userId8", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId9", "title" : "title9 hello", "summary" : "summary9", "journal" : "journal9", "created_by_user_id" : "userId9", "updated_by_user_id" : "userId9", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId10", "title" : "title10 hello", "summary" : "summary10", "journal" : "journal10", "created_by_user_id" : "userId10", "updated_by_user_id" : "userId10", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{"workspace_id" : "workspaceId1", "doc_id" : "docId11", "title" : "title11 hello, old value", "summary" : "summary11", "journal" : "journal11", "created_by_user_id" : "userId11", "updated_by_user_id" : "userId11", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"} diff --git a/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.md deleted file mode 100644 index 0ea23cac01..0000000000 --- a/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.md +++ /dev/null @@ -1,567 +0,0 @@ -# Snapshot report for `src/plugins/indexer/__tests__/service.spec.ts` - -The actual snapshot is saved in `service.spec.ts.snap`. - -Generated by [AVA](https://avajs.dev). - -## should write block with array content work - -> Snapshot 1 - - [ - { - fields: { - content: [ - 'hello world', - ], - }, - }, - ] - -## should parse all query work - -> Snapshot 1 - - { - _source: [ - 'workspace_id', - 'doc_id', - ], - fields: [ - 'flavour', - 'doc_id', - 'ref_doc_id', - ], - query: { - match_all: {}, - }, - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'id', - ], - } - -## should parse exists query work - -> Snapshot 1 - - { - _source: [ - 'workspace_id', - 'doc_id', - ], - fields: [ - 'flavour', - 'doc_id', - 'ref_doc_id', - ], - query: { - exists: { - field: 'ref_doc_id', - }, - }, - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'id', - ], - } - -## should parse boost query work - -> Snapshot 1 - - { - _source: [ - 'workspace_id', - 'doc_id', - ], - fields: [ - 'flavour', - 'doc_id', - 'ref_doc_id', - ], - query: { - term: { - flavour: { - boost: 1.5, - value: 'affine:page', - }, - }, - }, - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'id', - ], - } - -## should parse match query work - -> Snapshot 1 - - { - _source: [ - 'workspace_id', - 'doc_id', - ], - fields: [ - 'flavour', - 'doc_id', - 'ref_doc_id', - 'parent_flavour', - 'parent_block_id', - 'additional', - 'markdown_preview', - 'created_by_user_id', - 'updated_by_user_id', - 'created_at', - 'updated_at', - ], - query: { - term: { - flavour: { - value: 'affine:page', - }, - }, - }, - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'id', - ], - } - -## should parse boolean query work - -> Snapshot 1 - - { - _source: [ - 'workspace_id', - 'doc_id', - ], - fields: [ - 'flavour', - 'doc_id', - 'ref_doc_id', - 'parent_flavour', - 'parent_block_id', - 'additional', - 'markdown_preview', - 'created_by_user_id', - 'updated_by_user_id', - 'created_at', - 'updated_at', - ], - query: { - bool: { - must: [ - { - term: { - workspace_id: { - value: 'workspaceId1', - }, - }, - }, - { - match: { - content: { - query: 'hello', - }, - }, - }, - { - bool: { - should: [ - { - match: { - content: { - query: 'hello', - }, - }, - }, - { - term: { - flavour: { - boost: 1.5, - value: 'affine:page', - }, - }, - }, - ], - }, - }, - ], - }, - }, - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'id', - ], - } - -## should parse search input highlight work - -> Snapshot 1 - - { - _source: [ - 'workspace_id', - 'doc_id', - ], - fields: [ - 'flavour', - 'doc_id', - 'ref_doc_id', - ], - highlight: { - fields: { - content: { - post_tags: [ - '', - ], - pre_tags: [ - '', - ], - }, - }, - }, - query: { - match_all: {}, - }, - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'id', - ], - } - -## should parse aggregate input highlight work - -> Snapshot 1 - - { - _source: [ - 'workspace_id', - 'doc_id', - ], - aggs: { - result: { - aggs: { - max_score: { - max: { - script: { - source: '_score', - }, - }, - }, - result: { - top_hits: { - _source: [ - 'workspace_id', - 'doc_id', - ], - fields: [ - 'flavour', - 'doc_id', - 'ref_doc_id', - ], - highlight: { - fields: { - content: { - post_tags: [ - '', - ], - pre_tags: [ - '', - ], - }, - }, - }, - }, - }, - }, - terms: { - field: 'flavour', - order: { - max_score: 'desc', - }, - size: undefined, - }, - }, - }, - query: { - match_all: {}, - }, - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'id', - ], - } - -## should search work - -> Snapshot 1 - - [ - { - fields: { - summary: [ - 'this is a test', - ], - title: [ - 'hello world', - ], - }, - highlights: { - title: [ - 'hello world', - ], - }, - }, - ] - -> Snapshot 2 - - [ - { - fields: { - summary: [ - '这是测试', - ], - title: [ - '你好世界', - ], - }, - highlights: { - title: [ - '你好 世界', - ], - }, - }, - ] - -## should search with exists query work - -> Snapshot 1 - - [ - { - fields: { - blockId: [ - 'blockId1', - ], - parentBlockId: [ - 'blockId2', - ], - }, - }, - ] - -## should search a doc summary work - -> Snapshot 1 - - [ - { - fields: { - summary: [ - 'hello world, this is a summary', - ], - }, - }, - ] - -## should aggregate with bool must_not query work - -> Snapshot 1 - - [ - { - count: 2, - hits: [ - { - fields: { - additional: [ - '{"foo": "bar3"}', - ], - markdownPreview: [ - 'hello world, this is a title', - ], - parentBlockId: [ - 'parentBlockId1', - ], - parentFlavour: [ - 'affine:database', - ], - }, - }, - { - fields: { - additional: [ - '{"foo": "bar3"}', - ], - markdownPreview: [ - 'hello world, this is a title', - ], - parentBlockId: [ - 'parentBlockId2', - ], - parentFlavour: [ - 'affine:database', - ], - }, - }, - ], - }, - { - count: 1, - hits: [ - { - fields: { - additional: [ - '{"foo": "bar3"}', - ], - markdownPreview: [ - 'hello world, this is a title', - ], - parentBlockId: [ - 'parentBlockId3', - ], - parentFlavour: [ - 'affine:database', - ], - }, - }, - ], - }, - ] - -## should index doc work - -> Snapshot 1 - - { - summary: [ - `We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊ - Airtable & Miro with their no-code programable datasheets␊ - ␊ - For developer or installation guides, please go to AFFiNE Development␊ - Blocks that assemble your next docs, tasks kanban or whiteboard␊ - ␊ - Trello with their Kanban␊ - Remnote & Capacities with their object-based tag system␊ - AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ␊ - There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step fu`, - ], - title: [ - 'Write, Draw, Plan all at Once.', - ], - } - -> Snapshot 2 - - [ - { - blockId: [ - 'VMx9lHw3TR', - ], - content: [ - 'For developers or installations guides, please go to AFFiNE Doc', - ], - flavour: [ - 'affine:paragraph', - ], - }, - { - blockId: [ - '9-K49otbCv', - ], - content: [ - 'For developer or installation guides, please go to AFFiNE Development', - ], - flavour: [ - 'affine:paragraph', - ], - }, - ] - -## should search blob names from doc snapshot work - -> Snapshot 1 - - Map { - 'ldZMrM4PDlsNG4Q4YvCsz623h6TKu4qI9_FpTqIypfw=' => 'test file name here.txt', - } - -## should search blob names work - -> Snapshot 1 - - [ - [ - 'blob1', - 'blob1 name.txt', - ], - [ - 'blob2', - 'blob2 name.md', - ], - [ - 'blob3', - 'blob3 name.docx', - ], - ] - -## should search docs by keyword work - -> Snapshot 1 - - [ - { - blockId: 'block1', - createdAt: Date 2025-06-20 00:00:00 UTC {}, - highlight: 'hello world', - title: 'hello world', - updatedAt: Date 2025-06-20 00:00:00 UTC {}, - }, - { - blockId: 'block2', - createdAt: Date 2025-06-20 00:00:01 UTC {}, - highlight: 'hello world 2', - title: 'hello world 2', - updatedAt: Date 2025-06-20 00:00:01 UTC {}, - }, - { - blockId: 'block3', - createdAt: Date 2025-06-20 00:00:02 UTC {}, - highlight: 'hello world 3', - title: 'hello world 3', - updatedAt: Date 2025-06-20 00:00:02 UTC {}, - }, - { - blockId: 'block4', - createdAt: Date 2025-06-20 00:00:03 UTC {}, - highlight: 'hello world 4', - title: '', - updatedAt: Date 2025-06-20 00:00:03 UTC {}, - }, - ] diff --git a/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.snap deleted file mode 100644 index 639fc57ea4..0000000000 Binary files a/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.snap and /dev/null differ diff --git a/packages/backend/server/src/plugins/indexer/__tests__/event.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/event.spec.ts index c7139ab346..941d1b2004 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/event.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/event.spec.ts @@ -1,11 +1,11 @@ import test from 'ava'; -import Sinon from 'sinon'; import { createModule } from '../../../__tests__/create-module'; -import { Config } from '../../../base'; +import { JobQueue } from '../../../base'; import { ConfigModule } from '../../../base/config'; import { IndexerEvent } from '../event'; import { IndexerModule } from '../index'; +import { IndexerScheduler } from '../scheduler'; const module = await createModule({ imports: [ @@ -18,29 +18,12 @@ const module = await createModule({ ], }); const indexerEvent = module.get(IndexerEvent); -const config = module.get(Config); +const indexerScheduler = new IndexerScheduler(module.get(JobQueue)); test.after.always(async () => { await module.close(); }); -test.afterEach.always(() => { - Sinon.restore(); -}); - -test('should not index workspace if indexer is disabled', async t => { - Sinon.stub(config.indexer, 'enabled').value(false); - const count = module.queue.count('indexer.indexWorkspace'); - - // @ts-expect-error ignore missing fields - await indexerEvent.indexWorkspace({ - workspaceId: 'test-workspace', - docId: 'test-workspace', - }); - - t.is(module.queue.count('indexer.indexWorkspace'), count); -}); - test('should index workspace when root snapshot is updated', async t => { // @ts-expect-error ignore missing fields await indexerEvent.indexWorkspace({ @@ -64,19 +47,19 @@ test('should not index workspace when non-root snapshot is updated', async t => t.is(module.queue.count('indexer.indexWorkspace'), count); }); -test('should not delete workspace if indexer is disabled', async t => { - Sinon.stub(config.indexer, 'enabled').value(false); - const count = module.queue.count('indexer.deleteWorkspace'); - - // @ts-expect-error ignore missing fields - await indexerEvent.deleteUserWorkspaces({ - ownedWorkspaces: ['test-workspace'], +test('should reindex documents after document access changes', async t => { + await indexerEvent.reindexDocOnGrantChange({ + workspaceId: 'test-workspace', + docId: 'test-doc', + }); + const { payload } = await module.queue.waitFor('indexer.indexDoc'); + t.deepEqual(payload, { + workspaceId: 'test-workspace', + docId: 'test-doc', }); - - t.is(module.queue.count('indexer.deleteWorkspace'), count); }); -test('should delete workspace if indexer is enabled', async t => { +test('should delete workspace', async t => { // @ts-expect-error ignore missing fields await indexerEvent.deleteUserWorkspaces({ ownedWorkspaces: ['test-workspace'], @@ -86,17 +69,8 @@ test('should delete workspace if indexer is enabled', async t => { t.is(payload.workspaceId, 'test-workspace'); }); -test('should not schedule auto index workspaces if indexer is disabled', async t => { - Sinon.stub(config.indexer, 'enabled').value(false); - const count = module.queue.count('indexer.autoIndexWorkspaces'); - - await indexerEvent.autoIndexWorkspaces(); - - t.is(module.queue.count('indexer.autoIndexWorkspaces'), count); -}); - test('should schedule auto index workspaces', async t => { - await indexerEvent.autoIndexWorkspaces(); + await indexerScheduler.autoIndexWorkspaces(); const { payload } = await module.queue.waitFor('indexer.autoIndexWorkspaces'); t.is(payload.lastIndexedWorkspaceSid, undefined); diff --git a/packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts index dff7e8acec..79ad9ca28a 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts @@ -1,26 +1,23 @@ import { randomUUID } from 'node:crypto'; -import { mock } from 'node:test'; import test from 'ava'; import Sinon from 'sinon'; import { createModule } from '../../../__tests__/create-module'; import { Mockers } from '../../../__tests__/mocks'; -import { Config, JOB_SIGNAL } from '../../../base'; +import { JOB_SIGNAL } from '../../../base'; import { ConfigModule } from '../../../base/config'; import { ServerConfigModule } from '../../../core/config'; import { DocReader } from '../../../core/doc'; import { Models } from '../../../models'; import { addDocToRootDoc } from '../../../native'; -import { SearchProviderFactory } from '../factory'; -import { IndexerModule, IndexerService } from '../index'; +import { IndexerModule, IndexerService, IndexerWorkerModule } from '../index'; import { IndexerJob } from '../job'; -import { ManticoresearchProvider } from '../providers'; -import { blockSQL, docSQL, SearchTable } from '../tables'; const module = await createModule({ imports: [ IndexerModule, + IndexerWorkerModule, ServerConfigModule, ConfigModule.override({ indexer: { @@ -32,11 +29,8 @@ const module = await createModule({ }); const indexerService = module.get(IndexerService); const indexerJob = module.get(IndexerJob); -const searchProviderFactory = module.get(SearchProviderFactory); -const manticoresearch = module.get(ManticoresearchProvider); const models = module.get(Models); const docReader = module.get(DocReader); -const config = module.get(Config); const user = await module.create(Mockers.User); const workspace = await module.create(Mockers.Workspace, { @@ -44,24 +38,12 @@ const workspace = await module.create(Mockers.Workspace, { owner: user, }); -test.before(async () => { - await manticoresearch.recreateTable(SearchTable.block, blockSQL); - await manticoresearch.recreateTable(SearchTable.doc, docSQL); -}); - test.after.always(async () => { await module.close(); }); test.afterEach.always(() => { Sinon.restore(); - mock.reset(); -}); - -test.beforeEach(() => { - mock.method(searchProviderFactory, 'get', () => { - return manticoresearch; - }); }); test('should handle indexer.indexDoc job', async t => { @@ -83,54 +65,19 @@ test('should handle indexer.deleteDoc job', async t => { }); test('should handle indexer.indexWorkspace job', async t => { - const count = module.queue.count('indexer.deleteDoc'); - const spy = Sinon.spy(indexerService, 'listDocIds'); + const spy = Sinon.stub(indexerService, 'reconcileWorkspace').resolves(); await indexerJob.indexWorkspace({ workspaceId: workspace.id, }); - t.is(spy.callCount, 1); - const { payload } = await module.queue.waitFor('indexer.indexDoc'); - t.is(payload.workspaceId, workspace.id); - t.is(payload.docId, '5nS9BSp3Px'); - // no delete job - t.is(module.queue.count('indexer.deleteDoc'), count); + t.true(spy.calledOnceWith(workspace.id)); // workspace should be indexed const ws = await models.workspace.get(workspace.id); t.is(ws!.indexed, true); }); -test('should not sync existing doc', async t => { - const count = module.queue.count('indexer.indexDoc'); - mock.method(indexerService, 'listDocIds', async () => { - return ['5nS9BSp3Px']; - }); - - await indexerJob.indexWorkspace({ - workspaceId: workspace.id, - }); - - t.is(module.queue.count('indexer.indexDoc'), count); -}); - -test('should delete dangling indexed docs absent from the root live set', async t => { - const count = module.queue.count('indexer.deleteDoc'); - mock.method(indexerService, 'listDocIds', async () => { - return ['mock-doc-id1', 'mock-doc-id2']; - }); - - await indexerJob.indexWorkspace({ - workspaceId: workspace.id, - }); - - const { payload } = await module.queue.waitFor('indexer.indexDoc'); - t.is(payload.workspaceId, workspace.id); - t.is(payload.docId, '5nS9BSp3Px'); - t.is(module.queue.count('indexer.deleteDoc'), count + 2); -}); - test('document cleanup reconcile deletes missing search state before ack', async t => { const deleteSpy = Sinon.spy(indexerService, 'deleteDoc'); const indexSpy = Sinon.spy(indexerService, 'indexDoc'); @@ -203,32 +150,6 @@ test('document cleanup reconcile reindexes restored doc before ack', async t => }); }); -test('document cleanup reconcile only acknowledges when indexer is disabled', async t => { - Sinon.stub(config.indexer, 'enabled').value(false); - const deleteSpy = Sinon.spy(indexerService, 'deleteDoc'); - const indexSpy = Sinon.spy(indexerService, 'indexDoc'); - const getDocSpy = Sinon.spy(docReader, 'getDoc'); - - await indexerJob.reconcileDocumentCleanup({ - workspaceId: workspace.id, - docId: 'disabled-doc', - cleanupVersion: 'version-disabled', - }); - - t.false(deleteSpy.called); - t.false(indexSpy.called); - t.false(getDocSpy.called); - const { payload } = await module.queue.waitFor( - 'backendRuntime.ackDocumentCleanupEffect' - ); - t.deepEqual(payload, { - workspaceId: workspace.id, - docId: 'disabled-doc', - cleanupVersion: 'version-disabled', - effect: 'search', - }); -}); - test('should handle indexer.deleteWorkspace job', async t => { const spy = Sinon.spy(indexerService, 'deleteWorkspace'); diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md deleted file mode 100644 index be39188569..0000000000 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md +++ /dev/null @@ -1,727 +0,0 @@ -# Snapshot report for `src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts` - -The actual snapshot is saved in `elasticsearch.spec.ts.snap`. - -Generated by [AVA](https://avajs.dev). - -## should batch write bugfix - -> Snapshot 1 - - [ - { - _id: 'workspaceId-batch-write-bugfix-for-elasticsearch/a/b1', - _source: { - doc_id: 'a', - workspace_id: 'workspaceId-batch-write-bugfix-for-elasticsearch', - }, - fields: { - block_id: [ - 'b1', - ], - doc_id: [ - 'a', - ], - workspace_id: [ - 'workspaceId-batch-write-bugfix-for-elasticsearch', - ], - }, - highlights: undefined, - }, - { - _id: 'workspaceId-batch-write-bugfix-for-elasticsearch/a/b2', - _source: { - doc_id: 'a', - workspace_id: 'workspaceId-batch-write-bugfix-for-elasticsearch', - }, - fields: { - block_id: [ - 'b2', - ], - doc_id: [ - 'a', - ], - workspace_id: [ - 'workspaceId-batch-write-bugfix-for-elasticsearch', - ], - }, - highlights: undefined, - }, - ] - -## should search block table query match url work - -> Snapshot 1 - - { - _id: 'workspaceId1/docId2/blockId8', - _source: { - doc_id: 'docId2', - workspace_id: 'workspaceId1', - }, - fields: { - additional: [ - 'additional8', - ], - content: [ - 'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2', - ], - created_at: [ - Date 2025-03-08 06:04:13 278ms UTC {}, - ], - doc_id: [ - 'docId2', - ], - markdown_preview: [ - 'markdownPreview8', - ], - parent_block_id: [ - 'parentBlockId8', - ], - parent_flavour: [ - 'parentFlavour8', - ], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - ref_doc_id: [ - 'docId1', - ], - updated_at: [ - Date 2025-03-08 06:04:13 278ms UTC {}, - ], - }, - highlights: { - content: [ - 'hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link', - 'https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link', - '-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%', - 'E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%', - 'AF%8D%E6%90%9C%E7%B4%A2', - ], - }, - } - -> Snapshot 2 - - { - _id: 'workspaceId1/docId2/blockId8', - _source: { - doc_id: 'docId2', - workspace_id: 'workspaceId1', - }, - fields: { - additional: [ - 'additional8', - ], - content: [ - 'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2', - ], - created_at: [ - Date 2025-03-08 06:04:13 278ms UTC {}, - ], - doc_id: [ - 'docId2', - ], - markdown_preview: [ - 'markdownPreview8', - ], - parent_block_id: [ - 'parentBlockId8', - ], - parent_flavour: [ - 'parentFlavour8', - ], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - ref_doc_id: [ - 'docId1', - ], - updated_at: [ - Date 2025-03-08 06:04:13 278ms UTC {}, - ], - }, - highlights: { - content: [ - 'hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https', - '://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%', - ], - }, - } - -## should search block table query content match cjk work - -> Snapshot 1 - - { - _id: 'workspaceId1/docId2-affine/blockId8', - _source: { - doc_id: 'docId2-affine', - workspace_id: 'workspaceId1', - }, - fields: { - content: [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - doc_id: [ - 'docId2-affine', - ], - flavour: [ - 'flavour8', - ], - }, - highlights: { - content: [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - }, - } - -> Snapshot 2 - - { - _id: 'workspaceId1/docId2-affine/blockId8', - _source: { - doc_id: 'docId2-affine', - workspace_id: 'workspaceId1', - }, - fields: { - content: [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - doc_id: [ - 'docId2-affine', - ], - flavour: [ - 'flavour8', - ], - }, - highlights: { - content: [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - }, - } - -## should search doc table query title match cjk work - -> Snapshot 1 - - { - _id: 'workspace-test-doc-title-cjk/doc-0', - _source: { - doc_id: 'doc-0', - workspace_id: 'workspace-test-doc-title-cjk', - }, - fields: { - doc_id: [ - 'doc-0', - ], - title: [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - }, - highlights: { - title: [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - }, - } - -> Snapshot 2 - - { - _id: 'workspace-test-doc-title-cjk/doc-0', - _source: { - doc_id: 'doc-0', - workspace_id: 'workspace-test-doc-title-cjk', - }, - fields: { - doc_id: [ - 'doc-0', - ], - title: [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - }, - highlights: { - title: [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - }, - } - -## should search doc table query title.autocomplete work - -> Snapshot 1 - - { - _id: 'workspace-test-doc-title-autocomplete/doc-0', - _source: { - doc_id: 'doc-0', - workspace_id: 'workspace-test-doc-title-autocomplete', - }, - fields: { - doc_id: [ - 'doc-0', - ], - title: [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - }, - highlights: { - 'title.autocomplete': [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - }, - } - -## should search query match ref_doc_id work - -> Snapshot 1 - - [ - { - fields: { - additional: [ - '{"foo": "bar0"}', - ], - block_id: [ - 'blockId1', - ], - doc_id: [ - 'doc-0', - ], - parent_block_id: [ - 'parentBlockId1', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc-1', - ], - }, - }, - { - fields: { - additional: [ - '{"foo": "bar1"}', - ], - block_id: [ - 'blockId-all', - ], - doc_id: [ - 'doc-0', - ], - parent_block_id: [ - 'parentBlockId2', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc-2', - 'doc-3', - 'doc-4', - 'doc-5', - 'doc-6', - 'doc-7', - 'doc-8', - 'doc-9', - 'doc-10', - 'doc-1', - ], - }, - }, - { - fields: { - additional: [ - '{"foo": "bar1"}', - ], - block_id: [ - 'blockId1-2', - ], - doc_id: [ - 'doc-0', - ], - parent_block_id: [ - 'parentBlockId2', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc-1', - 'doc-2', - ], - }, - }, - { - fields: { - additional: [ - '{"foo": "bar1"}', - ], - block_id: [ - 'blockId2-1', - ], - doc_id: [ - 'doc-0', - ], - parent_block_id: [ - 'parentBlockId2', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc-2', - 'doc-1', - ], - }, - }, - { - fields: { - additional: [ - '{"foo": "bar1"}', - ], - block_id: [ - 'blockId3-2-1-4', - ], - doc_id: [ - 'doc-0', - ], - parent_block_id: [ - 'parentBlockId2', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc-3', - 'doc-2', - 'doc-1', - 'doc-4', - ], - }, - }, - ] - -> Snapshot 2 - - [ - { - fields: { - additional: [ - '{"foo": "bar1"}', - ], - block_id: [ - 'blockId-all', - ], - doc_id: [ - 'doc-0', - ], - parent_block_id: [ - 'parentBlockId2', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc-2', - 'doc-3', - 'doc-4', - 'doc-5', - 'doc-6', - 'doc-7', - 'doc-8', - 'doc-9', - 'doc-10', - 'doc-1', - ], - }, - }, - { - fields: { - additional: [ - '{"foo": "bar3"}', - ], - block_id: [ - 'blockId4', - ], - doc_id: [ - 'doc-0', - ], - parent_block_id: [ - 'parentBlockId4', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc-10', - ], - }, - }, - ] - -## should search doc title support stemmer filter - -> Snapshot 1 - - { - _id: 'workspace-test-doc-title-stemmer-filter/doc-0', - _source: { - doc_id: 'doc-0', - workspace_id: 'workspace-test-doc-title-stemmer-filter', - }, - fields: { - doc_id: [ - 'doc-0', - ], - title: [ - 'Deploy on Windows by a designer', - ], - }, - highlights: { - title: [ - 'Deploy on Windows by a designer', - ], - }, - } - -> Snapshot 2 - - { - _id: 'workspace-test-doc-title-stemmer-filter/doc-0', - _source: { - doc_id: 'doc-0', - workspace_id: 'workspace-test-doc-title-stemmer-filter', - }, - fields: { - doc_id: [ - 'doc-0', - ], - title: [ - 'Deploy on Windows by a designer', - ], - }, - highlights: { - title: [ - 'Deploy on Windows by a designer', - ], - }, - } - -> Snapshot 3 - - { - _id: 'workspace-test-doc-title-stemmer-filter/doc-0', - _source: { - doc_id: 'doc-0', - workspace_id: 'workspace-test-doc-title-stemmer-filter', - }, - fields: { - doc_id: [ - 'doc-0', - ], - title: [ - 'Deploy on Windows by a designer', - ], - }, - highlights: { - title: [ - 'Deploy on Windows by a designer', - ], - }, - } - -## should return empty string field:summary value - -> Snapshot 1 - - [ - { - _id: 'workspaceId-search-query-return-empty-string-field-summary-value-for-elasticsearch/doc0', - _source: { - doc_id: 'doc0', - workspace_id: 'workspaceId-search-query-return-empty-string-field-summary-value-for-elasticsearch', - }, - fields: { - doc_id: [ - 'doc0', - ], - summary: [ - '', - ], - title: [ - '', - ], - }, - highlights: undefined, - }, - ] - -## should not return not exists field:ref_doc_id - -> Snapshot 1 - - [ - { - _id: 'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-elasticsearch/doc0/block0', - _source: { - doc_id: 'doc0', - workspace_id: 'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-elasticsearch', - }, - fields: { - block_id: [ - 'block0', - ], - doc_id: [ - 'doc0', - ], - }, - highlights: undefined, - }, - ] - -## should aggregate query work - -> Snapshot 1 - - [ - { - _id: 'workspaceId1/docId9/blockId9', - _source: { - doc_id: 'docId9', - workspace_id: 'workspaceId1', - }, - fields: { - block_id: [ - 'blockId9', - ], - flavour: [ - 'affine:page', - ], - }, - highlights: { - content: [ - 'title9 hello affine issue hello hello hello hello hello hello hello hello hello hello, hello hello hello', - 'hello hello hello hello hello', - ], - }, - }, - ] - -## should aggregate query return top score first - -> Snapshot 1 - - [ - { - count: 1, - hits: [ - { - _id: 'aggregate-test-workspace-top-score-max-first/doc-0/block-0', - _source: { - doc_id: 'doc-0', - workspace_id: 'aggregate-test-workspace-top-score-max-first', - }, - fields: { - block_id: [ - 'block-0', - ], - flavour: [ - 'affine:page', - ], - }, - highlights: { - content: [ - '0.15 - week.1进度', - ], - }, - }, - ], - key: 'doc-0', - }, - { - count: 2, - hits: [ - { - _id: 'aggregate-test-workspace-top-score-max-first/doc-10/block-10-1', - _source: { - doc_id: 'doc-10', - workspace_id: 'aggregate-test-workspace-top-score-max-first', - }, - fields: { - block_id: [ - 'block-10-1', - ], - flavour: [ - 'affine:paragraph', - ], - }, - highlights: { - content: [ - 'Example 1', - ], - }, - }, - { - _id: 'aggregate-test-workspace-top-score-max-first/doc-10/block-10-2', - _source: { - doc_id: 'doc-10', - workspace_id: 'aggregate-test-workspace-top-score-max-first', - }, - fields: { - block_id: [ - 'block-10-2', - ], - flavour: [ - 'affine:paragraph', - ], - }, - highlights: { - content: [ - 'Single substitution format 1', - ], - }, - }, - ], - key: 'doc-10', - }, - ] - -> Snapshot 2 - - [ - { - count: 1, - hits: [ - { - _id: 'aggregate-test-workspace-top-score-max-first/doc-0/block-0', - _source: { - doc_id: 'doc-0', - workspace_id: 'aggregate-test-workspace-top-score-max-first', - }, - fields: { - block_id: [ - 'block-0', - ], - flavour: [ - 'affine:page', - ], - }, - highlights: { - content: [ - '0.15 - week.1进度', - ], - }, - }, - ], - key: 'doc-0', - }, - ] diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap deleted file mode 100644 index b2ff26081a..0000000000 Binary files a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap and /dev/null differ diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md deleted file mode 100644 index fe7c7e98ba..0000000000 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md +++ /dev/null @@ -1,1053 +0,0 @@ -# Snapshot report for `src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts` - -The actual snapshot is saved in `manticoresearch.spec.ts.snap`. - -Generated by [AVA](https://avajs.dev). - -## should search doc title match chinese word segmentation - -> Snapshot 1 - - [ - { - _id: '5373363211628325828', - _source: { - doc_id: 'doc-chinese', - workspace_id: 'workspace-test-doc-title-chinese', - }, - fields: { - doc_id: [ - 'doc-chinese', - ], - title: [ - 'AFFiNE 是一个基于云端的笔记应用', - ], - }, - highlights: undefined, - }, - ] - -## should search block content match korean ngram - -> Snapshot 1 - - [ - { - _id: '1227635764506850985', - _source: { - doc_id: 'doc-korean', - workspace_id: 'workspace-test-block-content-korean', - }, - fields: { - block_id: [ - 'block-korean', - ], - content: [ - '다람쥐 헌 쳇바퀴에 타고파', - ], - }, - highlights: undefined, - }, - ] - -## should search block content match japanese kana ngram - -> Snapshot 1 - - [ - { - _id: '381498385699454292', - _source: { - doc_id: 'doc-japanese', - workspace_id: 'workspace-test-block-content-japanese', - }, - fields: { - block_id: [ - 'block-japanese', - ], - content: [ - 'いろはにほへと ちりぬるを', - ], - }, - highlights: undefined, - }, - ] - -## should write document work - -> Snapshot 1 - - { - block_id: [ - '', - ], - content: [ - 'hello world', - ], - flavour: [ - 'affine:page', - ], - flavour_indexed: [ - 'affine:page', - ], - parent_flavour: [ - 'affine:database', - ], - parent_flavour_indexed: [ - 'affine:database', - ], - } - -> Snapshot 2 - - { - block_id: [ - '', - ], - content: [ - 'hello world', - ], - flavour: [ - 'affine:page', - ], - ref_doc_id: [ - 'docId2', - ], - } - -> Snapshot 3 - - { - block_id: [ - '', - ], - content: [ - 'hello world', - ], - flavour: [ - 'affine:page', - ], - } - -## should handle ref_doc_id as string[] - -> Snapshot 1 - - [ - { - _id: '4676525419549473798', - _source: { - doc_id: 'doc-0', - ref: '{"foo": "bar"}', - ref_doc_id: 'docId2', - workspace_id: 'workspaceId-ref-doc-id-for-manticoresearch', - }, - fields: { - content: [ - 'hello world', - ], - flavour: [ - 'affine:page', - ], - ref: [ - '{"foo": "bar"}', - ], - ref_doc_id: [ - 'docId2', - ], - }, - highlights: undefined, - }, - { - _id: '4676526519061102009', - _source: { - doc_id: 'doc-0', - ref: '{"foo": "bar2"}', - ref_doc_id: 'docId2', - workspace_id: 'workspaceId-ref-doc-id-for-manticoresearch', - }, - fields: { - content: [ - 'hello world', - ], - flavour: [ - 'affine:text', - ], - ref: [ - '{"foo": "bar2"}', - ], - ref_doc_id: [ - 'docId2', - ], - }, - highlights: undefined, - }, - ] - -> Snapshot 2 - - [ - { - _id: '4676525419549473798', - _source: { - doc_id: 'doc-0', - ref: '["{\\"foo\\": \\"bar\\"}","{\\"foo\\": \\"baz\\"}"]', - ref_doc_id: '["docId2","docId3"]', - workspace_id: 'workspaceId-ref-doc-id-for-manticoresearch', - }, - fields: { - content: [ - 'hello world', - ], - flavour: [ - 'affine:page', - ], - ref: [ - '{"foo": "bar"}', - '{"foo": "baz"}', - ], - ref_doc_id: [ - 'docId2', - 'docId3', - ], - }, - highlights: undefined, - }, - { - _id: '4676526519061102009', - _source: { - doc_id: 'doc-0', - ref: '["{\\"foo\\": \\"bar2\\"}","{\\"foo\\": \\"baz2\\"}"]', - ref_doc_id: '["docId2","docId3"]', - workspace_id: 'workspaceId-ref-doc-id-for-manticoresearch', - }, - fields: { - content: [ - 'hello world', - ], - flavour: [ - 'affine:text', - ], - ref: [ - '{"foo": "bar2"}', - '{"foo": "baz2"}', - ], - ref_doc_id: [ - 'docId2', - 'docId3', - ], - }, - highlights: undefined, - }, - ] - -## should handle content as string[] - -> Snapshot 1 - - [ - { - _id: '8978714848978078536', - _source: { - doc_id: 'doc-0', - ref: '{"foo": "bar"}', - ref_doc_id: 'docId2', - workspace_id: 'workspaceId-content-as-string-array-for-manticoresearch', - }, - fields: { - content: [ - 'hello world', - ], - flavour: [ - 'affine:page', - ], - ref: [ - '{"foo": "bar"}', - ], - ref_doc_id: [ - 'docId2', - ], - }, - highlights: undefined, - }, - ] - -> Snapshot 2 - - [ - { - _id: '8978714848978078536', - _source: { - doc_id: 'doc-0', - ref: '{"foo": "bar"}', - ref_doc_id: 'docId2', - workspace_id: 'workspaceId-content-as-string-array-for-manticoresearch', - }, - fields: { - content: [ - 'hello world 2', - ], - flavour: [ - 'affine:page', - ], - ref: [ - '{"foo": "bar"}', - ], - ref_doc_id: [ - 'docId2', - ], - }, - highlights: undefined, - }, - ] - -## should handle blob as string[] - -> Snapshot 1 - - [ - { - _id: '8163498729658755634', - _source: { - blob: 'blob1', - doc_id: 'doc-0', - workspace_id: 'workspaceId-blob-as-string-array-for-manticoresearch', - }, - fields: { - blob: [ - 'blob1', - ], - content: [ - '', - ], - flavour: [ - 'affine:page', - ], - }, - highlights: undefined, - }, - ] - -> Snapshot 2 - - [ - { - _id: '8163498729658755634', - _source: { - blob: '["blob1","blob2"]', - doc_id: 'doc-0', - workspace_id: 'workspaceId-blob-as-string-array-for-manticoresearch', - }, - fields: { - blob: [ - 'blob1', - 'blob2', - ], - content: [ - '', - ], - flavour: [ - 'affine:page', - ], - }, - highlights: undefined, - }, - ] - -> Snapshot 3 - - [ - { - _id: '8163498729658755634', - _source: { - blob: 'blob3', - doc_id: 'doc-0', - workspace_id: 'workspaceId-blob-as-string-array-for-manticoresearch', - }, - fields: { - blob: [ - 'blob3', - ], - content: [ - '', - ], - flavour: [ - 'affine:page', - ], - }, - highlights: undefined, - }, - ] - -## should batch write bugfix - -> Snapshot 1 - - [ - { - _id: '8950102031541144623', - _source: { - doc_id: 'a', - workspace_id: 'workspaceId-batch-write-bugfix-for-manticoresearch', - }, - fields: { - block_id: [ - 'b1', - ], - content: [ - '2025-05-26', - ], - doc_id: [ - 'a', - ], - workspace_id: [ - 'workspaceId-batch-write-bugfix-for-manticoresearch', - ], - }, - highlights: undefined, - }, - { - _id: '8950103131052772834', - _source: { - doc_id: 'a', - workspace_id: 'workspaceId-batch-write-bugfix-for-manticoresearch', - }, - fields: { - block_id: [ - 'b2', - ], - content: [ - '', - ], - doc_id: [ - 'a', - ], - workspace_id: [ - 'workspaceId-batch-write-bugfix-for-manticoresearch', - ], - }, - highlights: undefined, - }, - ] - -## should search query all and get next cursor work - -> Snapshot 1 - - [ - { - _id: '1835975812913922715', - _score: 1, - _source: { - doc_id: 'doc-10', - workspace_id: 'workspaceId-search-query-all-and-get-next-cursor-for-manticoresearch', - }, - fields: { - block_id: [ - 'block-10', - ], - doc_id: [ - 'doc-10', - ], - flavour: [ - 'affine:page', - ], - workspace_id: [ - 'workspaceId-search-query-all-and-get-next-cursor-for-manticoresearch', - ], - }, - highlights: undefined, - }, - { - _id: '1859562045173936129', - _score: 1, - _source: { - doc_id: 'doc-19', - workspace_id: 'workspaceId-search-query-all-and-get-next-cursor-for-manticoresearch', - }, - fields: { - block_id: [ - 'block-19', - ], - doc_id: [ - 'doc-19', - ], - flavour: [ - 'affine:page', - ], - workspace_id: [ - 'workspaceId-search-query-all-and-get-next-cursor-for-manticoresearch', - ], - }, - highlights: undefined, - }, - ] - -## should filter by workspace_id work - -> Snapshot 1 - - [ - { - _id: '5890563618264835345', - _score: 1, - _source: { - doc_id: 'doc-0', - workspace_id: 'workspaceId-filter-by-workspace_id-for-manticoresearch', - }, - fields: { - block_id: [ - 'blockId1', - ], - doc_id: [ - 'doc-0', - ], - flavour: [ - 'affine:page', - ], - workspace_id: [ - 'workspaceId-filter-by-workspace_id-for-manticoresearch', - ], - }, - highlights: undefined, - }, - { - _id: '5890560319729950712', - _score: 1, - _source: { - doc_id: 'doc-0', - workspace_id: 'workspaceId-filter-by-workspace_id-for-manticoresearch', - }, - fields: { - block_id: [ - 'blockId2', - ], - doc_id: [ - 'doc-0', - ], - flavour: [ - 'affine:database', - ], - workspace_id: [ - 'workspaceId-filter-by-workspace_id-for-manticoresearch', - ], - }, - highlights: undefined, - }, - ] - -## should search query match url work - -> Snapshot 1 - - { - _id: '6109831083726758533', - _source: { - doc_id: 'docId2', - workspace_id: 'workspaceId1', - }, - fields: { - additional: [ - 'additional8', - ], - content: [ - 'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2', - ], - created_at: [ - Date 2025-03-08 06:04:13 UTC {}, - ], - doc_id: [ - 'docId2', - ], - markdown_preview: [ - 'markdownPreview8', - ], - parent_block_id: [ - 'parentBlockId8', - ], - parent_flavour: [ - 'parentFlavour8', - ], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - ref_doc_id: [ - 'docId1', - ], - updated_at: [ - Date 2025-03-08 06:04:13 UTC {}, - ], - }, - highlights: { - content: [ - ' hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4', - '%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6', - '%8E%A5%E5%AF%B9%E9%93%BE%E6', - '%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7', - ], - }, - } - -## should search query match ref_doc_id work - -> Snapshot 1 - - [ - { - _id: '7273541739182975606', - _source: { - doc_id: 'doc0', - parent_flavour: 'affine:database', - workspace_id: 'workspaceId-search-query-match-ref_doc_id-for-manticoresearch', - }, - fields: { - additional: [ - '{"foo": "bar0"}', - ], - block_id: [ - 'blockId1', - ], - doc_id: [ - 'doc0', - ], - parent_block_id: [ - 'parentBlockId1', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc1', - ], - }, - highlights: undefined, - }, - { - _id: '6397614322515597713', - _source: { - doc_id: 'doc0', - parent_flavour: 'affine:database', - workspace_id: 'workspaceId-search-query-match-ref_doc_id-for-manticoresearch', - }, - fields: { - additional: [ - '{"foo": "bar1"}', - ], - block_id: [ - 'blockId-all', - ], - doc_id: [ - 'doc0', - ], - parent_block_id: [ - 'parentBlockId2', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc2', - 'doc3', - 'doc4', - 'doc5', - 'doc6', - 'doc7', - 'doc8', - 'doc9', - 'doc10', - 'doc1', - ], - }, - highlights: undefined, - }, - { - _id: '6305665172360896969', - _source: { - doc_id: 'doc0', - parent_flavour: 'affine:database', - workspace_id: 'workspaceId-search-query-match-ref_doc_id-for-manticoresearch', - }, - fields: { - additional: [ - '{"foo": "bar1"}', - ], - block_id: [ - 'blockId1-2', - ], - doc_id: [ - 'doc0', - ], - parent_block_id: [ - 'parentBlockId2', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc1', - 'doc2', - ], - }, - highlights: undefined, - }, - { - _id: '5748459067614019233', - _source: { - doc_id: 'doc0', - parent_flavour: 'affine:database', - workspace_id: 'workspaceId-search-query-match-ref_doc_id-for-manticoresearch', - }, - fields: { - additional: [ - '{"foo": "bar1"}', - ], - block_id: [ - 'blockId2-1', - ], - doc_id: [ - 'doc0', - ], - parent_block_id: [ - 'parentBlockId2', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc2', - 'doc1', - ], - }, - highlights: undefined, - }, - { - _id: '6824370853640968276', - _source: { - doc_id: 'doc0', - parent_flavour: 'affine:database', - workspace_id: 'workspaceId-search-query-match-ref_doc_id-for-manticoresearch', - }, - fields: { - additional: [ - '{"foo": "bar1"}', - ], - block_id: [ - 'blockId3-2-1-4', - ], - doc_id: [ - 'doc0', - ], - parent_block_id: [ - 'parentBlockId2', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc3', - 'doc2', - 'doc1', - 'doc4', - ], - }, - highlights: undefined, - }, - ] - -> Snapshot 2 - - [ - { - _id: '6397614322515597713', - _source: { - doc_id: 'doc0', - workspace_id: 'workspaceId-search-query-match-ref_doc_id-for-manticoresearch', - }, - fields: { - additional: [ - '{"foo": "bar1"}', - ], - block_id: [ - 'blockId-all', - ], - doc_id: [ - 'doc0', - ], - parent_block_id: [ - 'parentBlockId2', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc2', - 'doc3', - 'doc4', - 'doc5', - 'doc6', - 'doc7', - 'doc8', - 'doc9', - 'doc10', - 'doc1', - ], - }, - highlights: undefined, - }, - { - _id: '7273547236741116661', - _source: { - doc_id: 'doc0', - workspace_id: 'workspaceId-search-query-match-ref_doc_id-for-manticoresearch', - }, - fields: { - additional: [ - '{"foo": "bar3"}', - ], - block_id: [ - 'blockId4', - ], - doc_id: [ - 'doc0', - ], - parent_block_id: [ - 'parentBlockId4', - ], - parent_flavour: [ - 'affine:database', - ], - ref_doc_id: [ - 'doc10', - ], - }, - highlights: undefined, - }, - ] - -## should return empty string field:summary value - -> Snapshot 1 - - [ - { - _id: '274027293861775228', - _source: { - doc_id: 'doc0', - workspace_id: 'workspaceId-search-query-return-empty-string-field-summary-value-for-manticoresearch', - }, - fields: { - doc_id: [ - 'doc0', - ], - summary: [ - '', - ], - title: [ - '', - ], - }, - highlights: undefined, - }, - ] - -## should not return not exists field:ref_doc_id - -> Snapshot 1 - - [ - { - _id: '2457631367295327017', - _source: { - doc_id: 'doc0', - workspace_id: 'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-manticoresearch', - }, - fields: { - block_id: [ - 'block0', - ], - doc_id: [ - 'doc0', - ], - }, - highlights: undefined, - }, - ] - -## should aggregate query return top score first - -> Snapshot 1 - - [ - { - count: 1, - hits: [ - { - _id: '6281444972018276017', - _source: { - doc_id: 'doc-0', - workspace_id: 'aggregate-test-workspace-top-score-max-first', - }, - fields: { - block_id: [ - 'block-0', - ], - flavour: [ - 'affine:page', - ], - }, - highlights: { - content: [ - '0.15 - week.1 进度', - ], - }, - }, - ], - key: 'doc-0', - }, - { - count: 2, - hits: [ - { - _id: '2160976319205307295', - _source: { - doc_id: 'doc-10', - workspace_id: 'aggregate-test-workspace-top-score-max-first', - }, - fields: { - block_id: [ - 'block-10-1', - ], - flavour: [ - 'affine:paragraph', - ], - }, - highlights: { - content: [ - 'Example 1', - ], - }, - }, - { - _id: '2160977418716935506', - _source: { - doc_id: 'doc-10', - workspace_id: 'aggregate-test-workspace-top-score-max-first', - }, - fields: { - block_id: [ - 'block-10-2', - ], - flavour: [ - 'affine:paragraph', - ], - }, - highlights: { - content: [ - 'Single substitution format 1', - ], - }, - }, - ], - key: 'doc-10', - }, - ] - -## should parse es query term work - -> Snapshot 1 - - { - equals: { - workspace_id: 'workspaceId1', - }, - } - -> Snapshot 2 - - { - equals: { - workspace_id: 'workspaceId1', - }, - } - -> Snapshot 3 - - { - match: { - flavour_indexed: { - boost: 1.5, - query: 'affine:page', - }, - }, - } - -> Snapshot 4 - - { - match: { - doc_id: { - boost: 1.5, - query: 'docId1', - }, - }, - } - -## should parse es query with custom term mapping field work - -> Snapshot 1 - - { - bool: { - must: [ - { - equals: { - workspace_id: 'workspaceId1', - }, - }, - { - equals: { - doc_id: 'docId1', - }, - }, - ], - }, - } - -> Snapshot 2 - - { - bool: { - must: { - equals: { - workspace_id: 'workspaceId1', - }, - }, - }, - } - -> Snapshot 3 - - { - equals: { - workspace_id: 'workspaceId1', - }, - } - -## should parse es query exists work - -> Snapshot 1 - - { - exists: { - field: 'parent_block_id_indexed', - }, - } - -> Snapshot 2 - - { - exists: { - field: 'ref_doc_id', - }, - } diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap deleted file mode 100644 index 5e308ddd22..0000000000 Binary files a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap and /dev/null differ diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts deleted file mode 100644 index 9d63629e23..0000000000 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts +++ /dev/null @@ -1,1909 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { readFile } from 'node:fs/promises'; -import path from 'node:path'; - -import _test from 'ava'; -import { omit, pick } from 'lodash-es'; - -import { - createModule, - TestingModule, -} from '../../../../__tests__/create-module'; -import { Mockers } from '../../../../__tests__/mocks'; -import { ConfigModule } from '../../../../base/config'; -import { User, Workspace } from '../../../../models'; -import { SearchProviderType } from '../../config'; -import { IndexerModule } from '../../index'; -import { AggregateQueryDSL, ElasticsearchProvider } from '../../providers'; -import { blockMapping, docMapping, SearchTable } from '../../tables'; - -const test = - process.env.AFFINE_INDEXER_SEARCH_PROVIDER === 'elasticsearch' - ? _test - : _test.skip; - -let module: TestingModule; -let searchProvider: ElasticsearchProvider; -let user: User; -let workspace: Workspace; - -_test.before(async () => { - module = await createModule({ - imports: [ - IndexerModule, - ConfigModule.override({ - indexer: { - enabled: true, - provider: { - type: SearchProviderType.Elasticsearch, - endpoint: 'http://localhost:9200', - username: 'elastic', - password: 'affine', - }, - }, - }), - ], - providers: [ElasticsearchProvider], - }); - searchProvider = module.get(ElasticsearchProvider); - user = await module.create(Mockers.User); - workspace = await module.create(Mockers.Workspace); - - await searchProvider.createTable( - SearchTable.block, - JSON.stringify(blockMapping) - ); - await searchProvider.createTable(SearchTable.doc, JSON.stringify(docMapping)); - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: randomUUID(), - doc_id: randomUUID(), - block_id: randomUUID(), - content: `hello world on search title, ${randomUUID()}`, - flavour: 'affine:page', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: randomUUID(), - doc_id: randomUUID(), - block_id: randomUUID(), - content: `hello world on search block content, ${randomUUID()}`, - flavour: 'other:flavour', - blob: randomUUID(), - ref_doc_id: randomUUID(), - ref: ['{"foo": "bar"}', '{"foo": "baz"}'], - parent_flavour: 'parent:flavour', - parent_block_id: randomUUID(), - additional: '{"foo": "bar"}', - markdown_preview: 'markdownPreview', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: 'workspaceId101', - doc_id: 'docId101', - block_id: 'blockId101', - content: 'hello world on search block content at 101', - flavour: 'other:flavour', - blob: 'blob101', - ref_doc_id: 'docId101', - ref: ['{"foo": "bar"}', '{"foo": "baz"}'], - parent_flavour: 'parent:flavour', - parent_block_id: 'blockId101', - additional: '{"foo": "bar"}', - markdown_preview: 'markdownPreview', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date('2025-04-19T08:19:36.160Z'), - updated_at: new Date('2025-04-19T08:19:36.160Z'), - }, - { - workspace_id: 'workspaceId1', - doc_id: 'docId2', - block_id: 'blockId8', - content: - 'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2', - flavour: 'flavour8', - ref_doc_id: 'docId1', - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - parent_flavour: 'parentFlavour8', - parent_block_id: 'parentBlockId8', - additional: 'additional8', - markdown_preview: 'markdownPreview8', - created_by_user_id: 'userId8', - updated_by_user_id: 'userId8', - created_at: new Date('2025-03-08T06:04:13.278Z'), - updated_at: new Date('2025-03-08T06:04:13.278Z'), - }, - { - workspace_id: 'workspaceId1', - doc_id: 'docId2-affine', - block_id: 'blockId8', - content: 'AFFiNE 是一个基于云端的笔记应用', - flavour: 'flavour8', - ref_doc_id: 'docId1', - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - parent_flavour: 'parentFlavour8', - parent_block_id: 'parentBlockId8', - additional: 'additional8', - markdown_preview: 'markdownPreview8', - created_by_user_id: 'userId8', - updated_by_user_id: 'userId8', - created_at: new Date('2025-03-08T06:04:13.278Z'), - updated_at: new Date('2025-03-08T06:04:13.278Z'), - }, - ], - { - refresh: true, - } - ); - const blocks = await readFile( - path.join(import.meta.dirname, '../__fixtures__/test-blocks.json'), - 'utf-8' - ); - const blockDocuments = blocks - .trim() - .split('\n') - .map(line => JSON.parse(line)); - await searchProvider.write(SearchTable.block, blockDocuments, { - refresh: true, - }); - - const docs = await readFile( - path.join(import.meta.dirname, '../__fixtures__/test-docs.json'), - 'utf-8' - ); - const docDocuments = docs - .trim() - .split('\n') - .map(line => JSON.parse(line)); - await searchProvider.write(SearchTable.doc, docDocuments, { - refresh: true, - }); -}); - -_test.after.always(async () => { - await searchProvider.deleteByQuery( - SearchTable.block, - { - term: { - workspace_id: workspace.id, - }, - }, - { - refresh: true, - } - ); - await searchProvider.deleteByQuery( - SearchTable.doc, - { - term: { - workspace_id: workspace.id, - }, - }, - { - refresh: true, - } - ); - - await module.close(); -}); - -test('should provider is elasticsearch', t => { - t.is(searchProvider.type, SearchProviderType.Elasticsearch); -}); - -// #region write - -test('should write document work', async t => { - const docId = randomUUID(); - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: 'hello world', - flavour: 'affine:page', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { match: { doc_id: docId } }, - fields: ['flavour', 'block_id', 'content', 'ref_doc_id'], - sort: ['_score'], - }); - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0].fields, { - flavour: ['affine:page'], - content: ['hello world'], - }); - t.deepEqual(result.nodes[0]._source, { - doc_id: docId, - workspace_id: workspace.id, - }); - - // set ref_doc_id to a string - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: 'hello world', - flavour: 'affine:page', - ref_doc_id: 'docId2', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { match: { doc_id: docId } }, - fields: ['flavour', 'block_id', 'content', 'ref_doc_id'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0].fields, { - flavour: ['affine:page'], - content: ['hello world'], - ref_doc_id: ['docId2'], - }); - - // not set ref_doc_id and replace the old value to null - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: 'hello world', - flavour: 'affine:page', - // ref_doc_id: 'docId2', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { match: { doc_id: docId } }, - fields: ['flavour', 'block_id', 'content', 'ref_doc_id'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0].fields, { - flavour: ['affine:page'], - content: ['hello world'], - }); -}); - -test('should handle ref_doc_id as string[]', async t => { - const docId = randomUUID(); - // set ref_doc_id to a string - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: 'hello world', - flavour: 'affine:page', - ref_doc_id: 'docId2', - ref: '{"foo": "bar"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'ref_doc_id', 'ref'], - query: { match: { doc_id: docId } }, - fields: ['flavour', 'content', 'ref_doc_id', 'ref'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0].fields, { - flavour: ['affine:page'], - content: ['hello world'], - ref_doc_id: ['docId2'], - ref: ['{"foo": "bar"}'], - }); - - t.deepEqual(result.nodes[0]._source, { - doc_id: docId, - workspace_id: workspace.id, - ref_doc_id: 'docId2', - ref: '{"foo": "bar"}', - }); - - // set ref_doc_id to a string[] - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: 'hello world', - flavour: 'affine:page', - ref_doc_id: ['docId2', 'docId3'], - ref: ['{"foo": "bar"}', '{"foo": "baz"}'], - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'ref_doc_id', 'ref'], - query: { match: { doc_id: docId } }, - fields: ['flavour', 'content', 'ref_doc_id', 'ref'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0].fields, { - flavour: ['affine:page'], - content: ['hello world'], - ref_doc_id: ['docId2', 'docId3'], - ref: ['{"foo": "bar"}', '{"foo": "baz"}'], - }); - t.deepEqual(result.nodes[0]._source, { - doc_id: docId, - workspace_id: workspace.id, - ref_doc_id: ['docId2', 'docId3'], - ref: ['{"foo": "bar"}', '{"foo": "baz"}'], - }); -}); - -test('should handle content as string[]', async t => { - const docId = randomUUID(); - // set content to a string - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: 'hello world', - flavour: 'affine:page', - ref_doc_id: 'docId2', - ref: '{"foo": "bar"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'ref_doc_id', 'ref'], - query: { match: { doc_id: docId } }, - fields: ['flavour', 'content', 'ref_doc_id', 'ref'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0].fields, { - flavour: ['affine:page'], - content: ['hello world'], - ref_doc_id: ['docId2'], - ref: ['{"foo": "bar"}'], - }); - t.deepEqual(result.nodes[0]._source, { - doc_id: docId, - workspace_id: workspace.id, - ref_doc_id: 'docId2', - ref: '{"foo": "bar"}', - }); - - // set content to a string[] - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: ['hello', 'world 2'], - flavour: 'affine:page', - ref_doc_id: 'docId2', - ref: '{"foo": "bar"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'ref_doc_id', 'ref'], - query: { match: { doc_id: docId } }, - fields: ['flavour', 'content', 'ref_doc_id', 'ref'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0].fields, { - flavour: ['affine:page'], - content: ['hello', 'world 2'], - ref_doc_id: ['docId2'], - ref: ['{"foo": "bar"}'], - }); - t.deepEqual(result.nodes[0]._source, { - doc_id: docId, - workspace_id: workspace.id, - ref_doc_id: 'docId2', - ref: '{"foo": "bar"}', - }); -}); - -test('should handle blob as string[]', async t => { - const docId = randomUUID(); - const blockId = randomUUID(); - // set blob to a string - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - block_id: blockId, - content: '', - flavour: 'affine:page', - blob: 'blob1', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'blob'], - query: { match: { doc_id: docId } }, - fields: ['flavour', 'content', 'blob'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0].fields, { - flavour: ['affine:page'], - blob: ['blob1'], - content: [''], - }); - t.deepEqual(result.nodes[0]._source, { - doc_id: docId, - workspace_id: workspace.id, - blob: 'blob1', - }); - - // set blob to a string[] - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - block_id: blockId, - content: '', - flavour: 'affine:page', - blob: ['blob1', 'blob2'], - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'blob'], - query: { match: { doc_id: docId } }, - fields: ['flavour', 'content', 'blob'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0].fields, { - flavour: ['affine:page'], - blob: ['blob1', 'blob2'], - content: [''], - }); - t.deepEqual(result.nodes[0]._source, { - doc_id: docId, - workspace_id: workspace.id, - blob: ['blob1', 'blob2'], - }); - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - block_id: blockId, - content: '', - flavour: 'affine:page', - blob: ['blob3'], - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'blob'], - query: { match: { doc_id: docId } }, - fields: ['flavour', 'content', 'blob'], - sort: ['_score'], - }); - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0].fields, { - flavour: ['affine:page'], - blob: ['blob3'], - content: [''], - }); - t.deepEqual(result.nodes[0]._source, { - doc_id: docId, - workspace_id: workspace.id, - blob: ['blob3'], - }); -}); - -test('should batch write bugfix', async t => { - const workspaceId = 'workspaceId-batch-write-bugfix-for-elasticsearch'; - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: 'a', - block_id: 'b1', - content: '2025-05-26', - flavour: 'affine:page', - additional: '{"displayMode":"edgeless"}', - created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', - updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', - created_at: '2025-05-26T05:16:23.128Z', - updated_at: '2025-05-26T05:15:53.091Z', - flavour_indexed: 'affine:page', - }, - { - workspace_id: workspaceId, - doc_id: 'a', - block_id: 'b2', - content: '', - flavour: 'affine:surface', - parent_flavour: 'affine:page', - parent_block_id: 'TcOGF6HSa7', - additional: '', - created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', - updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', - created_at: '2025-05-26T05:16:23.128Z', - updated_at: '2025-05-26T05:15:53.091Z', - flavour_indexed: 'affine:surface', - parent_flavour_indexed: 'affine:page', - parent_block_id_indexed: 'TcOGF6HSa7', - }, - ], - { - refresh: true, - } - ); - - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { - workspace_id: { - value: workspaceId, - }, - }, - }, - ], - }, - }, - fields: ['workspace_id', 'doc_id', 'block_id'], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); -}); - -// #endregion - -// #region search - -test('should search query all and get next cursor work', async t => { - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'doc_id', - 'block_id', - ], - query: { - match_all: {}, - }, - fields: ['flavour', 'doc_id', 'content', 'created_at', 'updated_at'], - size: 2, - }); - - t.truthy(result.total); - t.is(result.timedOut, false); - t.truthy(result.nextCursor); - t.is(typeof result.nextCursor, 'string'); - t.is(result.nodes.length, 2); - t.truthy(result.nodes[0]._id); - t.truthy(result.nodes[0]._score); - t.truthy(result.nodes[0].fields.flavour); - t.truthy(result.nodes[0].fields.doc_id); - t.truthy(result.nodes[0].fields.content); - t.truthy(result.nodes[0].fields.created_at); - t.truthy(result.nodes[0].fields.updated_at); - t.deepEqual(Object.keys(result.nodes[0]._source), ['workspace_id', 'doc_id']); - - // test cursor - const result2 = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'doc_id', - 'block_id', - ], - query: { - match_all: {}, - }, - fields: ['flavour', 'doc_id', 'content', 'created_at', 'updated_at'], - size: 10000, - cursor: result.nextCursor, - }); - - t.is(result2.total, result.total); - t.is(result2.timedOut, false); - t.truthy(result2.nextCursor); - t.is(typeof result2.nextCursor, 'string'); - t.true(result2.nodes.length < 10000); - - // next cursor should be empty - const result3 = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'doc_id', - 'block_id', - ], - query: { - match_all: {}, - }, - fields: ['flavour', 'doc_id', 'content', 'created_at', 'updated_at'], - size: 10000, - cursor: result2.nextCursor, - }); - - t.is(result3.total, result.total); - t.is(result3.timedOut, false); - t.falsy(result3.nextCursor); - t.is(result3.nodes.length, 0); -}); - -test('should search block table query match url work', async t => { - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - match: { - content: - 'https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2', - }, - }, - fields: [ - 'doc_id', - 'content', - 'ref', - 'ref_doc_id', - 'parent_flavour', - 'parent_block_id', - 'additional', - 'markdown_preview', - 'created_at', - 'updated_at', - ], - highlight: { - fields: { - content: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.true(result.total >= 1); - t.snapshot(omit(result.nodes[0], ['_score'])); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - match: { - content: 'https://linear.app', - }, - }, - fields: [ - 'doc_id', - 'content', - 'ref', - 'ref_doc_id', - 'parent_flavour', - 'parent_block_id', - 'additional', - 'markdown_preview', - 'created_at', - 'updated_at', - ], - highlight: { - fields: { - content: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.true(result.total >= 1); - t.snapshot(omit(result.nodes[0], ['_score'])); -}); - -test('should search block table query content match cjk work', async t => { - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - match: { - content: '笔记应用', - }, - }, - fields: ['flavour', 'doc_id', 'content'], - highlight: { - fields: { - content: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.is(result.total, 1); - t.snapshot(omit(result.nodes[0], ['_score'])); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - match: { - content: '记', - }, - }, - fields: ['flavour', 'doc_id', 'content'], - highlight: { - fields: { - content: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.is(result.total, 1); - t.snapshot(omit(result.nodes[0], ['_score'])); -}); - -test('should search doc table query title match cjk work', async t => { - const workspaceId = 'workspace-test-doc-title-cjk'; - await searchProvider.write( - SearchTable.doc, - [ - { - workspace_id: workspaceId, - doc_id: 'doc-0', - title: 'AFFiNE 是一个基于云端的笔记应用', - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.doc, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { match: { workspace_id: workspaceId } }, - { match: { title: '笔记应' } }, - ], - }, - }, - fields: ['doc_id', 'title'], - highlight: { - fields: { - title: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.is(result.total, 1); - t.snapshot(omit(result.nodes[0], ['_score'])); - - // match single chinese character - result = await searchProvider.search(SearchTable.doc, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { match: { workspace_id: workspaceId } }, - { match: { title: '笔' } }, - ], - }, - }, - fields: ['doc_id', 'title'], - highlight: { - fields: { - title: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.is(result.total, 1); - t.snapshot(omit(result.nodes[0], ['_score'])); -}); - -test('should search doc table query title.autocomplete work', async t => { - const docId = 'doc-0'; - const workspaceId = 'workspace-test-doc-title-autocomplete'; - await searchProvider.write( - SearchTable.doc, - [ - { - workspace_id: workspaceId, - doc_id: docId, - title: 'AFFiNE 是一个基于云端的笔记应用', - }, - ], - { - refresh: true, - } - ); - - const result = await searchProvider.search(SearchTable.doc, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { match: { workspace_id: workspaceId } }, - { match: { 'title.autocomplete': 'aff' } }, - ], - }, - }, - fields: ['doc_id', 'title'], - highlight: { - fields: { - 'title.autocomplete': { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.is(result.total, 1); - t.snapshot(omit(result.nodes[0], ['_score'])); -}); - -test('should search query match ref_doc_id work', async t => { - const docId = 'doc-0'; - const refDocId1 = 'doc-1'; - const refDocId2 = 'doc-2'; - const refDocId3 = 'doc-3'; - const refDocId4 = 'doc-4'; - const refDocId5 = 'doc-5'; - const refDocId6 = 'doc-6'; - const refDocId7 = 'doc-7'; - const refDocId8 = 'doc-8'; - const refDocId9 = 'doc-9'; - const refDocId10 = 'doc-10'; - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - block_id: 'blockId1', - content: 'hello world on search title blockId1', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId1', - ref_doc_id: refDocId1, - ref: '{"docId":"docId1","mode":"page"}', - additional: '{"foo": "bar0"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspace.id, - doc_id: docId, - block_id: 'blockId1-not-matched', - content: 'hello world on search title blockId1-not-matched', - flavour: 'affine:page', - parent_flavour: 'affine:database1', - parent_block_id: 'parentBlockId1', - ref_doc_id: refDocId1, - ref: '{"docId":"docId1","mode":"page"}', - additional: '{"foo": "bar0"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspace.id, - doc_id: docId, - block_id: 'blockId-all', - content: 'hello world on search title blockId-all', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId2', - ref_doc_id: [ - refDocId2, - refDocId3, - refDocId4, - refDocId5, - refDocId6, - refDocId7, - refDocId8, - refDocId9, - refDocId10, - refDocId1, - ], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - additional: '{"foo": "bar1"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspace.id, - doc_id: docId, - block_id: 'blockId1-2', - content: 'hello world on search title blockId1-2', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId2', - ref_doc_id: [refDocId1, refDocId2], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - additional: '{"foo": "bar1"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspace.id, - doc_id: docId, - block_id: 'blockId2-1', - content: 'hello world on search title blockId2-1', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId2', - ref_doc_id: [refDocId2, refDocId1], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - additional: '{"foo": "bar1"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspace.id, - doc_id: docId, - block_id: 'blockId3-2-1-4', - content: 'hello world on search title blockId3-2-1-4', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId2', - ref_doc_id: [refDocId3, refDocId2, refDocId1, refDocId4], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - additional: '{"foo": "bar1"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - // a link to the `refDocId1` document - { - workspace_id: workspace.id, - doc_id: refDocId1, - block_id: 'blockId3', - content: 'hello world on search title blockId3', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId3', - ref_doc_id: refDocId1, - ref: '{"docId":"docId1","mode":"page"}', - additional: '{"foo": "bar2"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspace.id, - doc_id: docId, - block_id: 'blockId4', - content: 'hello world on search title blockId4', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId4', - ref_doc_id: refDocId10, - ref: '{"docId":"docId2","mode":"page"}', - additional: '{"foo": "bar3"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspace.id, - doc_id: docId, - block_id: 'blockId1-text', - content: 'hello world on search title blockId1-text', - flavour: 'affine:text', - parent_flavour: 'affine:text', - parent_block_id: 'parentBlockId1', - ref_doc_id: refDocId1, - ref: '{"docId":"docId1","mode":"page"}', - additional: '{"foo": "bar0"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'parent_flavour'], - query: { - bool: { - must: [ - { - term: { workspace_id: { value: workspace.id } }, - }, - { - bool: { - must: [ - { - term: { parent_flavour: { value: 'affine:database' } }, - }, - { - // https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/array - // match: { ref_doc_id: { query: refDocId1 } }, - term: { ref_doc_id: { value: refDocId1 } }, - }, - // Ignore if it is a link to the `refDocId1` document - { - bool: { - must_not: { - term: { doc_id: { value: refDocId1 } }, - }, - }, - }, - ], - }, - }, - ], - }, - }, - fields: [ - 'doc_id', - 'block_id', - 'ref_doc_id', - 'parent_block_id', - 'additional', - 'parent_flavour', - ], - sort: ['_score'], - }); - - t.is(result.total, 5); - t.snapshot(result.nodes.map(node => pick(node, ['fields']))); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { workspace_id: { value: workspace.id } }, - }, - { - bool: { - must: [ - { - term: { parent_flavour: { value: 'affine:database' } }, - }, - { - term: { ref_doc_id: { value: refDocId10 } }, - }, - // Ignore if it is a link to the `refDocId1` document - { - bool: { - must_not: { - term: { doc_id: { value: refDocId1 } }, - }, - }, - }, - ], - }, - }, - ], - }, - }, - fields: [ - 'doc_id', - 'block_id', - 'ref_doc_id', - 'parent_block_id', - 'parent_flavour', - 'additional', - ], - sort: ['_score'], - }); - - t.is(result.total, 2); - t.snapshot(result.nodes.map(node => pick(node, ['fields']))); -}); - -test('should search doc title support stemmer filter', async t => { - const docId = 'doc-0'; - const workspaceId = 'workspace-test-doc-title-stemmer-filter'; - await searchProvider.write( - SearchTable.doc, - [ - { - workspace_id: workspaceId, - doc_id: docId, - title: 'Deploy on Windows by a designer', - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.doc, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { match: { workspace_id: workspaceId } }, - { match: { title: 'window' } }, - ], - }, - }, - fields: ['doc_id', 'title'], - highlight: { - fields: { - title: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.is(result.total, 1); - t.snapshot(omit(result.nodes[0], ['_score'])); - - result = await searchProvider.search(SearchTable.doc, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { match: { workspace_id: workspaceId } }, - { match: { title: 'windows' } }, - ], - }, - }, - fields: ['doc_id', 'title'], - highlight: { - fields: { - title: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.is(result.total, 1); - t.snapshot(omit(result.nodes[0], ['_score'])); - - result = await searchProvider.search(SearchTable.doc, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { match: { workspace_id: workspaceId } }, - { match: { title: 'design' } }, - ], - }, - }, - fields: ['doc_id', 'title'], - highlight: { - fields: { - title: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.is(result.total, 1); - t.snapshot(omit(result.nodes[0], ['_score'])); -}); - -test('should return empty string field:summary value', async t => { - const workspaceId = - 'workspaceId-search-query-return-empty-string-field-summary-value-for-elasticsearch'; - const docId = 'doc0'; - - await searchProvider.write( - SearchTable.doc, - [ - { - workspace_id: workspaceId, - doc_id: docId, - title: '', - summary: '', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.doc, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { workspace_id: { value: workspaceId } }, - }, - { - term: { - doc_id: { - value: docId, - }, - }, - }, - ], - }, - }, - fields: ['doc_id', 'title', 'summary'], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); -}); - -test('should not return not exists field:ref_doc_id', async t => { - const workspaceId = - 'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-elasticsearch'; - const docId = 'doc0'; - const blockId = 'block0'; - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId, - content: 'hello world on search title blockId1-text', - flavour: 'affine:text', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { workspace_id: { value: workspaceId } }, - }, - { - term: { - doc_id: { - value: docId, - }, - }, - }, - ], - }, - }, - fields: [ - 'doc_id', - 'block_id', - 'ref_doc_id', - 'parent_block_id', - 'additional', - 'parent_flavour', - ], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); -}); - -test('should created_at and updated_at is date type', async t => { - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'created_at', 'updated_at'], - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'doc_id', - 'block_id', - ], - query: { - match_all: {}, - }, - fields: ['created_at', 'updated_at'], - size: 2, - }); - - t.truthy(result.total); - t.truthy(result.nodes[0].fields.created_at); - t.truthy(result.nodes[0].fields.updated_at); - t.true( - result.nodes[0].fields.created_at[0] instanceof Date, - 'created_at should be date type, but got ' + - result.nodes[0].fields.created_at[0] - ); - t.true( - result.nodes[0].fields.updated_at[0] instanceof Date, - 'updated_at should be date type, but got ' + - result.nodes[0].fields.updated_at[0] - ); - t.true( - result.nodes[0]._source.created_at instanceof Date, - 'created_at should be date type, but got ' + - result.nodes[0]._source.created_at - ); - t.true( - result.nodes[0]._source.updated_at instanceof Date, - 'updated_at should be date type, but got ' + - result.nodes[0]._source.updated_at - ); -}); - -// #endregion - -// #region aggregate - -test('should aggregate query work', async t => { - const result = await searchProvider.aggregate(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - sort: ['_score', { updated_at: 'desc' }, 'doc_id', 'block_id'], - query: { - bool: { - must: [ - { - term: { - workspace_id: { - value: 'workspaceId1', - }, - }, - }, - { - bool: { - must: [ - { - match: { - content: 'hello', - }, - }, - { - bool: { - should: [ - { - match: { - content: 'hello', - }, - }, - { - term: { - flavour: { - value: 'affine:page', - boost: 1.5, - }, - }, - }, - ], - }, - }, - ], - }, - }, - ], - }, - }, - aggs: { - result: { - terms: { - field: 'doc_id', - order: { - max_score: 'desc', - }, - }, - aggs: { - max_score: { - max: { - script: { - source: '_score', - }, - }, - }, - result: { - top_hits: { - _source: ['workspace_id', 'doc_id'], - highlight: { - fields: { - content: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - fields: ['block_id', 'flavour'], - size: 2, - }, - }, - }, - }, - }, - }); - - t.truthy(result.total); - t.is(result.timedOut, false); - t.truthy(result.nextCursor); - t.true(result.buckets.length > 0); - t.truthy(result.buckets[0].key); - t.true(result.buckets[0].count > 0); - t.truthy(result.buckets[0].hits.nodes.length > 0); - t.deepEqual(Object.keys(result.buckets[0].hits.nodes[0]._source), [ - 'workspace_id', - 'doc_id', - ]); - t.snapshot(result.buckets[0].hits.nodes.map(node => omit(node, ['_score']))); -}); - -test('should aggregate query return top score first', async t => { - const workspaceId = 'aggregate-test-workspace-top-score-max-first'; - await searchProvider.deleteByQuery( - SearchTable.block, - { - bool: { - must: [{ term: { workspace_id: { value: workspaceId } } }], - }, - }, - { - refresh: true, - } - ); - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: 'doc-0', - block_id: 'block-0', - content: `0.15 - week.1进度`, - flavour: 'affine:page', - additional: '{"displayMode":"edgeless"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: 'doc-10', - block_id: 'block-10-1', - content: 'Example 1', - flavour: 'affine:paragraph', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: 'doc-10', - block_id: 'block-10-2', - content: 'Single substitution format 1', - flavour: 'affine:paragraph', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - const query = { - size: 50, - _source: ['workspace_id', 'doc_id'], - sort: ['_score', { updated_at: 'desc' }, 'doc_id', 'block_id'], - query: { - bool: { - must: [ - { - term: { - workspace_id: { - value: workspaceId, - }, - }, - }, - { - bool: { - must: [ - { - match: { - content: '0.15 week.1', - }, - }, - { - bool: { - should: [ - { - match: { - content: '0.15 week.1', - }, - }, - { - term: { - flavour: { - value: 'affine:page', - boost: 1.5, - }, - }, - }, - ], - }, - }, - ], - }, - }, - ], - }, - }, - aggs: { - result: { - terms: { - field: 'doc_id', - size: 100, - order: { - max_score: 'desc', - }, - }, - aggs: { - max_score: { - max: { - script: { - source: '_score', - }, - }, - }, - result: { - top_hits: { - _source: ['workspace_id', 'doc_id'], - highlight: { - fields: { - content: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - fields: ['block_id', 'flavour'], - size: 2, - }, - }, - }, - }, - }, - } as AggregateQueryDSL; - const result = await searchProvider.aggregate(SearchTable.block, query); - - t.truthy(result.total); - t.is(result.timedOut, false); - t.truthy(result.nextCursor); - t.true(result.buckets.length > 0); - t.truthy(result.buckets[0].key); - t.true(result.buckets[0].count > 0); - t.truthy(result.buckets[0].hits.nodes.length > 0); - t.deepEqual(Object.keys(result.buckets[0].hits.nodes[0]._source), [ - 'workspace_id', - 'doc_id', - ]); - t.snapshot( - result.buckets.map(bucket => ({ - key: bucket.key, - count: bucket.count, - hits: bucket.hits.nodes.map(node => omit(node, ['_score'])), - })) - ); - - // set size to 1 - query.aggs.result.terms.size = 1; - const result2 = await searchProvider.aggregate(SearchTable.block, query); - - t.is(result2.buckets.length, 1); - t.snapshot( - result2.buckets.map(bucket => ({ - key: bucket.key, - count: bucket.count, - hits: bucket.hits.nodes.map(node => omit(node, ['_score'])), - })) - ); - t.is(result2.buckets[0].hits.nodes.length, 1); -}); - -// #endregion - -// #region delete by query - -test('should delete by query work', async t => { - const docId = 'doc-delete-by-query'; - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - block_id: 'block-0', - content: `hello world on search title block-0`, - flavour: 'affine:page', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspace.id, - doc_id: docId, - block_id: 'block-1', - content: `hello world on search title block-1`, - flavour: 'other:flavour', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { - workspace_id: workspace.id, - }, - }, - { - term: { - doc_id: docId, - }, - }, - ], - }, - }, - fields: ['block_id'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 2); - - await searchProvider.deleteByQuery( - SearchTable.block, - { - bool: { - must: [ - { - term: { - workspace_id: workspace.id, - }, - }, - { - term: { - doc_id: docId, - }, - }, - ], - }, - }, - { - refresh: true, - } - ); - - const result2 = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { - workspace_id: workspace.id, - }, - }, - { - term: { - doc_id: docId, - }, - }, - ], - }, - }, - fields: ['block_id'], - sort: ['_score'], - }); - - t.is(result2.nodes.length, 0); -}); - -// #endregion diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts deleted file mode 100644 index 83b0a0d421..0000000000 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts +++ /dev/null @@ -1,1865 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { readFile } from 'node:fs/promises'; -import path from 'node:path'; - -import test from 'ava'; -import { omit } from 'lodash-es'; - -import { createModule } from '../../../../__tests__/create-module'; -import { Mockers } from '../../../../__tests__/mocks'; -import { ConfigModule } from '../../../../base/config'; -import { SearchProviderType } from '../../config'; -import { IndexerModule } from '../../index'; -import { ManticoresearchProvider } from '../../providers'; -import { blockSQL, docSQL, SearchTable } from '../../tables'; - -const module = await createModule({ - imports: [ - IndexerModule, - ConfigModule.override({ - indexer: { - enabled: true, - provider: { - type: SearchProviderType.Manticoresearch, - endpoint: 'http://localhost:9308', - }, - }, - }), - ], - providers: [ManticoresearchProvider], -}); -const searchProvider = module.get(ManticoresearchProvider); -const user = await module.create(Mockers.User); -const workspace = await module.create(Mockers.Workspace); - -test.before(async () => { - await searchProvider.recreateTable(SearchTable.block, blockSQL); - await searchProvider.recreateTable(SearchTable.doc, docSQL); - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: randomUUID(), - doc_id: randomUUID(), - block_id: randomUUID(), - content: `hello world on search title, ${randomUUID()}`, - flavour: 'affine:page', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: randomUUID(), - doc_id: randomUUID(), - block_id: randomUUID(), - content: `hello world on search block content, ${randomUUID()}`, - flavour: 'other:flavour', - blob: randomUUID(), - ref_doc_id: randomUUID(), - ref: ['{"foo": "bar"}', '{"foo": "baz"}'], - parent_flavour: 'parent:flavour', - parent_block_id: randomUUID(), - additional: '{"foo": "bar"}', - markdown_preview: 'markdownPreview', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: 'workspaceId101', - doc_id: 'docId101', - block_id: 'blockId101', - content: 'hello world on search block content at 101', - flavour: 'other:flavour', - blob: 'blob101', - ref_doc_id: 'docId101', - ref: ['{"foo": "bar"}', '{"foo": "baz"}'], - parent_flavour: 'parent:flavour', - parent_block_id: 'blockId101', - additional: '{"foo": "bar"}', - markdown_preview: 'markdownPreview', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date('2025-04-19T08:19:36.160Z'), - updated_at: new Date('2025-04-19T08:19:36.160Z'), - }, - { - workspace_id: 'workspaceId1', - doc_id: 'docId2', - block_id: 'blockId8', - content: - 'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2', - flavour: 'flavour8', - ref_doc_id: 'docId1', - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - parent_flavour: 'parentFlavour8', - parent_block_id: 'parentBlockId8', - additional: 'additional8', - markdown_preview: 'markdownPreview8', - created_by_user_id: 'userId8', - updated_by_user_id: 'userId8', - created_at: new Date('2025-03-08T06:04:13.278Z'), - updated_at: new Date('2025-03-08T06:04:13.278Z'), - }, - ], - { - refresh: true, - } - ); - const blocks = await readFile( - path.join(import.meta.dirname, '../__fixtures__/test-blocks.json'), - 'utf-8' - ); - const blockDocuments = blocks - .trim() - .split('\n') - .map(line => JSON.parse(line)); - await searchProvider.write(SearchTable.block, blockDocuments, { - refresh: true, - }); - - const docs = await readFile( - path.join(import.meta.dirname, '../__fixtures__/test-docs.json'), - 'utf-8' - ); - const docDocuments = docs - .trim() - .split('\n') - .map(line => JSON.parse(line)); - await searchProvider.write(SearchTable.doc, docDocuments, { - refresh: true, - }); -}); - -test.after.always(async () => { - await searchProvider.deleteByQuery( - SearchTable.block, - { - term: { workspace_id: workspace.id }, - }, - { - refresh: true, - } - ); - await searchProvider.deleteByQuery( - SearchTable.doc, - { - term: { workspace_id: workspace.id }, - }, - { - refresh: true, - } - ); - await module.close(); -}); - -test('should provider is manticoresearch', t => { - t.is(searchProvider.type, SearchProviderType.Manticoresearch); -}); - -test('should search doc title match chinese word segmentation', async t => { - const workspaceId = 'workspace-test-doc-title-chinese'; - const docId = 'doc-chinese'; - const title = 'AFFiNE 是一个基于云端的笔记应用'; - - await searchProvider.write( - SearchTable.doc, - [ - { - workspace_id: workspaceId, - doc_id: docId, - title, - }, - ], - { - refresh: true, - } - ); - - const result = await searchProvider.search(SearchTable.doc, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { term: { workspace_id: { value: workspaceId } } }, - { match: { title: '笔记' } }, - ], - }, - }, - fields: ['doc_id', 'title'], - sort: ['_score'], - }); - - t.true(result.total >= 1); - t.snapshot( - result.nodes - .filter(node => node._source.doc_id === docId) - .map(node => omit(node, ['_score'])) - ); -}); - -test('should search block content match korean ngram', async t => { - const workspaceId = 'workspace-test-block-content-korean'; - const docId = 'doc-korean'; - const blockId = 'block-korean'; - const content = '다람쥐 헌 쳇바퀴에 타고파'; - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId, - content, - flavour: 'affine:paragraph', - }, - ], - { - refresh: true, - } - ); - - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { term: { workspace_id: { value: workspaceId } } }, - { match: { content: '쥐' } }, - ], - }, - }, - fields: ['block_id', 'content'], - sort: ['_score'], - }); - - t.true(result.total >= 1); - t.snapshot( - result.nodes - .filter(node => node.fields.block_id?.[0] === blockId) - .map(node => omit(node, ['_score'])) - ); -}); - -test('should search block content match japanese kana ngram', async t => { - const workspaceId = 'workspace-test-block-content-japanese'; - const docId = 'doc-japanese'; - const blockId = 'block-japanese'; - const content = 'いろはにほへと ちりぬるを'; - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId, - content, - flavour: 'affine:paragraph', - }, - ], - { - refresh: true, - } - ); - - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { term: { workspace_id: { value: workspaceId } } }, - { match: { content: 'へ' } }, - ], - }, - }, - fields: ['block_id', 'content'], - sort: ['_score'], - }); - - t.true(result.total >= 1); - t.snapshot( - result.nodes - .filter(node => node.fields.block_id?.[0] === blockId) - .map(node => omit(node, ['_score'])) - ); -}); - -// #region write - -test('should write document work', async t => { - const docId = randomUUID(); - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: 'hello world', - flavour: 'affine:page', - parent_flavour: 'affine:database', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { term: { doc_id: { value: docId } } }, - fields: [ - 'flavour', - 'flavour_indexed', - 'parent_flavour', - 'parent_flavour_indexed', - 'block_id', - 'content', - 'ref_doc_id', - ], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.deepEqual(result.nodes[0]._source, { - doc_id: docId, - workspace_id: workspace.id, - }); - t.snapshot(result.nodes[0].fields); - - // set ref_doc_id to a string - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: 'hello world', - flavour: 'affine:page', - ref_doc_id: 'docId2', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { term: { doc_id: { value: docId } } }, - fields: ['flavour', 'block_id', 'content', 'ref_doc_id'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.snapshot(result.nodes[0].fields); - - // not set ref_doc_id and replace the old value to null - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: 'hello world', - flavour: 'affine:page', - // ref_doc_id: 'docId2', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { term: { doc_id: { value: docId } } }, - fields: ['flavour', 'block_id', 'content', 'ref_doc_id'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 1); - t.snapshot(result.nodes[0].fields); -}); - -test('should handle ref_doc_id as string[]', async t => { - const workspaceId = 'workspaceId-ref-doc-id-for-manticoresearch'; - const docId = 'doc-0'; - const blockId0 = 'block-0'; - const blockId1 = 'block-1'; - - // set ref_doc_id to a string - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId0, - content: 'hello world', - flavour: 'affine:page', - ref_doc_id: 'docId2', - ref: '{"foo": "bar"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId1, - content: 'hello world', - flavour: 'affine:text', - ref_doc_id: 'docId2', - ref: ['{"foo": "bar2"}'], - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date('2025-04-23T00:00:00.000Z'), - updated_at: new Date('2025-04-23T00:00:00.000Z'), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'ref_doc_id', 'ref'], - query: { - bool: { - must: [ - { term: { workspace_id: { value: workspaceId } } }, - { term: { doc_id: { value: docId } } }, - ], - }, - }, - fields: ['flavour', 'content', 'ref_doc_id', 'ref'], - sort: ['_score', { created_at: 'desc' }], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); - - // set ref_doc_id to a string[] - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId0, - content: 'hello world', - flavour: 'affine:page', - ref_doc_id: ['docId2', 'docId3'], - ref: ['{"foo": "bar"}', '{"foo": "baz"}'], - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId1, - content: 'hello world', - flavour: 'affine:text', - ref_doc_id: ['docId2', 'docId3'], - ref: ['{"foo": "bar2"}', '{"foo": "baz2"}'], - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date('2025-04-23T00:00:00.000Z'), - updated_at: new Date('2025-04-23T00:00:00.000Z'), - }, - ], - { - refresh: true, - } - ); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'ref_doc_id', 'ref'], - query: { - bool: { - must: [ - { term: { workspace_id: { value: workspaceId } } }, - { term: { doc_id: { value: docId } } }, - ], - }, - }, - fields: ['flavour', 'content', 'ref_doc_id', 'ref'], - sort: ['_score', { created_at: 'desc' }], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); -}); - -test('should handle content as string[]', async t => { - const workspaceId = 'workspaceId-content-as-string-array-for-manticoresearch'; - const docId = 'doc-0'; - const blockId = 'block-0'; - - // set content to a string - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId, - content: 'hello world', - flavour: 'affine:page', - ref_doc_id: 'docId2', - ref: '{"foo": "bar"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'ref_doc_id', 'ref'], - query: { - bool: { - must: [ - { term: { workspace_id: { value: workspaceId } } }, - { term: { doc_id: { value: docId } } }, - ], - }, - }, - fields: ['flavour', 'content', 'ref_doc_id', 'ref'], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); - - // set content to a string[] - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId, - content: ['hello', 'world 2'], - flavour: 'affine:page', - ref_doc_id: 'docId2', - ref: '{"foo": "bar"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'ref_doc_id', 'ref'], - query: { - bool: { - must: [ - { term: { workspace_id: { value: workspaceId } } }, - { term: { doc_id: { value: docId } } }, - ], - }, - }, - fields: ['flavour', 'content', 'ref_doc_id', 'ref'], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); -}); - -test('should handle blob as string[]', async t => { - const workspaceId = 'workspaceId-blob-as-string-array-for-manticoresearch'; - const docId = 'doc-0'; - const blockId = 'block-0'; - // set blob to a string - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId, - content: '', - flavour: 'affine:page', - blob: 'blob1', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'blob'], - query: { - bool: { - must: [ - { term: { workspace_id: { value: workspaceId } } }, - { term: { doc_id: { value: docId } } }, - ], - }, - }, - fields: ['flavour', 'content', 'blob'], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); - - // set blob to a string[] - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId, - content: '', - flavour: 'affine:page', - blob: ['blob1', 'blob2'], - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'blob'], - query: { - bool: { - must: [ - { term: { workspace_id: { value: workspaceId } } }, - { term: { doc_id: { value: docId } } }, - ], - }, - }, - fields: ['flavour', 'content', 'blob'], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId, - content: '', - flavour: 'affine:page', - blob: ['blob3'], - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'blob'], - query: { - bool: { - must: [ - { term: { workspace_id: { value: workspaceId } } }, - { term: { doc_id: { value: docId } } }, - ], - }, - }, - fields: ['flavour', 'content', 'blob'], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); -}); - -test('should batch write bugfix', async t => { - const workspaceId = 'workspaceId-batch-write-bugfix-for-manticoresearch'; - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: 'a', - block_id: 'b1', - content: '2025-05-26', - flavour: 'affine:page', - additional: '{"displayMode":"edgeless"}', - created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', - updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', - created_at: '2025-05-26T05:16:23.128Z', - updated_at: '2025-05-26T05:15:53.091Z', - flavour_indexed: 'affine:page', - }, - { - workspace_id: workspaceId, - doc_id: 'a', - block_id: 'b2', - content: '', - flavour: 'affine:surface', - parent_flavour: 'affine:page', - parent_block_id: 'TcOGF6HSa7', - additional: '', - created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', - updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', - created_at: '2025-05-26T05:16:23.128Z', - updated_at: '2025-05-26T05:15:53.091Z', - flavour_indexed: 'affine:surface', - parent_flavour_indexed: 'affine:page', - parent_block_id_indexed: 'TcOGF6HSa7', - }, - ], - { - refresh: true, - } - ); - - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { - workspace_id: { - value: workspaceId, - }, - }, - }, - ], - }, - }, - fields: ['workspace_id', 'doc_id', 'block_id', 'content'], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); -}); - -// #endregion - -// #region search - -test('should search query all and get next cursor work', async t => { - const workspaceId = - 'workspaceId-search-query-all-and-get-next-cursor-for-manticoresearch'; - await searchProvider.write( - SearchTable.block, - Array.from({ length: 20 }, (_, i) => ({ - workspace_id: workspaceId, - doc_id: `doc-${i}`, - block_id: `block-${i}`, - content: `hello world ${i}`, - flavour: 'affine:page', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - })), - { - refresh: true, - } - ); - - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'id', - ], - query: { - term: { - workspace_id: { - value: workspaceId, - }, - }, - }, - fields: ['flavour', 'workspace_id', 'doc_id', 'block_id'], - size: 2, - }); - - t.truthy(result.total); - t.is(result.timedOut, false); - t.truthy(result.nextCursor); - t.is(typeof result.nextCursor, 'string'); - t.snapshot(result.nodes); - t.is(result.nodes.length, 2); - - // test cursor - const result2 = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'id', - ], - query: { - term: { - workspace_id: { - value: workspaceId, - }, - }, - }, - fields: ['flavour', 'workspace_id', 'doc_id', 'block_id'], - size: 10000, - cursor: result.nextCursor, - }); - - t.is(result2.total, result.total - result.nodes.length); - t.is(result2.timedOut, false); - t.truthy(result2.nextCursor); - t.is(typeof result2.nextCursor, 'string'); - t.true(result2.nodes.length < 10000); - - // next cursor should be empty - const result3 = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'id', - ], - query: { - term: { - workspace_id: { - value: workspaceId, - }, - }, - }, - fields: ['flavour', 'workspace_id', 'doc_id', 'block_id'], - size: 10000, - cursor: result2.nextCursor, - }); - - t.is(result3.total, 0); - t.is(result3.timedOut, false); - t.falsy(result3.nextCursor); - t.is(result3.nodes.length, 0); -}); - -test('should filter by workspace_id work', async t => { - const workspaceId = 'workspaceId-filter-by-workspace_id-for-manticoresearch'; - const docId = 'doc-0'; - await searchProvider.write(SearchTable.block, [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: 'blockId1', - flavour: 'affine:page', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: docId, - block_id: 'blockId2', - flavour: 'affine:database', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ]); - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { - workspace_id: { - value: workspaceId, - }, - }, - }, - { - bool: { - must: [ - { - term: { - doc_id: { - value: docId, - }, - }, - }, - ], - }, - }, - ], - }, - }, - fields: ['flavour', 'workspace_id', 'doc_id', 'block_id'], - sort: ['_score'], - }); - - t.snapshot(result.nodes); - t.is(result.nodes.length, 2); -}); - -test('should search query match url work', async t => { - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - match: { - content: - 'https://linear.app/affine-design/issue/AF-1379/slash-commands', - }, - }, - fields: [ - 'doc_id', - 'content', - 'ref', - 'ref_doc_id', - 'parent_flavour', - 'parent_block_id', - 'additional', - 'markdown_preview', - 'created_at', - 'updated_at', - ], - highlight: { - fields: { - content: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - sort: ['_score'], - }); - - t.true(result.total >= 1); - t.snapshot(omit(result.nodes[0], ['_score'])); -}); - -test('should search query match ref_doc_id work', async t => { - const workspaceId = - 'workspaceId-search-query-match-ref_doc_id-for-manticoresearch'; - const docId = 'doc0'; - const refDocId1 = 'doc1'; - const refDocId2 = 'doc2'; - const refDocId3 = 'doc3'; - const refDocId4 = 'doc4'; - const refDocId5 = 'doc5'; - const refDocId6 = 'doc6'; - const refDocId7 = 'doc7'; - const refDocId8 = 'doc8'; - const refDocId9 = 'doc9'; - const refDocId10 = 'doc10'; - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: 'blockId1', - content: 'hello world on search title, blockId1', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId1', - ref_doc_id: refDocId1, - ref: '{"docId":"docId1","mode":"page"}', - additional: '{"foo": "bar0"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: docId, - block_id: 'blockId1-not-matched', - content: 'hello world on search title, blockId1-not-matched', - flavour: 'affine:page', - parent_flavour: 'affine:database1', - parent_block_id: 'parentBlockId1', - ref_doc_id: refDocId1, - ref: '{"docId":"docId1","mode":"page"}', - additional: '{"foo": "bar0"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: docId, - block_id: 'blockId-all', - content: 'hello world on search title, blockId-all', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId2', - ref_doc_id: [ - refDocId2, - refDocId3, - refDocId4, - refDocId5, - refDocId6, - refDocId7, - refDocId8, - refDocId9, - refDocId10, - refDocId1, - ], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - additional: '{"foo": "bar1"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: docId, - block_id: 'blockId1-2', - content: 'hello world on search title, blockId1-2', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId2', - ref_doc_id: [refDocId1, refDocId2], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - additional: '{"foo": "bar1"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: docId, - block_id: 'blockId2-1', - content: 'hello world on search title, blockId2-1', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId2', - ref_doc_id: [refDocId2, refDocId1], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - additional: '{"foo": "bar1"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: docId, - block_id: 'blockId3-2-1-4', - content: 'hello world on search title, blockId3-2-1-4', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId2', - ref_doc_id: [refDocId3, refDocId2, refDocId1, refDocId4], - ref: [ - '{"docId":"docId1","mode":"page"}', - '{"docId":"docId2","mode":"page"}', - ], - additional: '{"foo": "bar1"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - // a link to the `refDocId1` document - { - workspace_id: workspaceId, - doc_id: refDocId1, - block_id: 'blockId3', - content: 'hello world on search title, blockId3', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId3', - ref_doc_id: refDocId1, - ref: '{"docId":"docId1","mode":"page"}', - additional: '{"foo": "bar2"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: docId, - block_id: 'blockId4', - content: 'hello world on search title, blockId4', - flavour: 'affine:page', - parent_flavour: 'affine:database', - parent_block_id: 'parentBlockId4', - ref_doc_id: refDocId10, - ref: '{"docId":"docId2","mode":"page"}', - additional: '{"foo": "bar3"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: docId, - block_id: 'blockId1-text', - content: 'hello world on search title, blockId1-text', - flavour: 'affine:text', - parent_flavour: 'affine:text', - parent_block_id: 'parentBlockId1', - ref_doc_id: refDocId1, - ref: '{"docId":"docId1","mode":"page"}', - additional: '{"foo": "bar0"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'parent_flavour'], - query: { - bool: { - must: [ - { - term: { workspace_id: { value: workspaceId } }, - }, - { - bool: { - must: [ - { - term: { parent_flavour: { value: 'affine:database' } }, - }, - { - term: { ref_doc_id: { value: refDocId1 } }, - }, - // Ignore if it is a link to the `refDocId1` document - { - bool: { - must_not: { - term: { doc_id: { value: refDocId1 } }, - }, - }, - }, - ], - }, - }, - ], - }, - }, - fields: [ - 'doc_id', - 'block_id', - 'ref_doc_id', - 'parent_block_id', - 'additional', - 'parent_flavour', - ], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); - t.is(result.total, 5); - - result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { workspace_id: { value: workspaceId } }, - }, - { - bool: { - must: [ - { - term: { parent_flavour: { value: 'affine:database' } }, - }, - { - term: { ref_doc_id: { value: refDocId10 } }, - }, - // Ignore if it is a link to the `refDocId1` document - { - bool: { - must_not: { - term: { doc_id: { value: refDocId1 } }, - }, - }, - }, - ], - }, - }, - ], - }, - }, - fields: [ - 'doc_id', - 'block_id', - 'ref_doc_id', - 'parent_block_id', - 'parent_flavour', - 'additional', - ], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); - t.is(result.total, 2); -}); - -test('should return empty string field:summary value', async t => { - const workspaceId = - 'workspaceId-search-query-return-empty-string-field-summary-value-for-manticoresearch'; - const docId = 'doc0'; - - await searchProvider.write( - SearchTable.doc, - [ - { - workspace_id: workspaceId, - doc_id: docId, - title: '', - summary: '', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.doc, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { workspace_id: { value: workspaceId } }, - }, - { - term: { - doc_id: { - value: docId, - }, - }, - }, - ], - }, - }, - fields: ['doc_id', 'title', 'summary'], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); -}); - -test('should not return not exists field:ref_doc_id', async t => { - const workspaceId = - 'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-manticoresearch'; - const docId = 'doc0'; - const blockId = 'block0'; - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: docId, - block_id: blockId, - content: 'hello world on search title blockId1-text', - flavour: 'affine:text', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { workspace_id: { value: workspaceId } }, - }, - { - term: { - doc_id: { - value: docId, - }, - }, - }, - ], - }, - }, - fields: [ - 'doc_id', - 'block_id', - 'ref_doc_id', - 'parent_block_id', - 'additional', - 'parent_flavour', - ], - sort: ['_score'], - }); - - t.snapshot(result.nodes.map(node => omit(node, ['_score']))); -}); - -test('should created_at and updated_at is date type', async t => { - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id', 'created_at', 'updated_at'], - sort: [ - '_score', - { - updated_at: 'desc', - }, - 'doc_id', - 'block_id', - ], - query: { - match_all: {}, - }, - fields: ['created_at', 'updated_at'], - size: 2, - }); - - t.truthy(result.total); - t.truthy(result.nodes[0].fields.created_at); - t.truthy(result.nodes[0].fields.updated_at); - t.true( - result.nodes[0].fields.created_at[0] instanceof Date, - 'created_at should be date type, but got ' + - result.nodes[0].fields.created_at[0] - ); - t.true( - result.nodes[0].fields.updated_at[0] instanceof Date, - 'updated_at should be date type, but got ' + - result.nodes[0].fields.updated_at[0] - ); - t.true( - result.nodes[0]._source.created_at instanceof Date, - 'created_at should be date type, but got ' + - result.nodes[0]._source.created_at - ); - t.true( - result.nodes[0]._source.updated_at instanceof Date, - 'updated_at should be date type, but got ' + - result.nodes[0]._source.updated_at - ); -}); - -// #endregion - -// #region aggregate - -test('should aggregate query return top score first', async t => { - const workspaceId = 'aggregate-test-workspace-top-score-max-first'; - await searchProvider.deleteByQuery( - SearchTable.block, - { - bool: { - must: [{ term: { workspace_id: { value: workspaceId } } }], - }, - }, - { - refresh: true, - } - ); - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspaceId, - doc_id: 'doc-0', - block_id: 'block-0', - content: `0.15 - week.1进度`, - flavour: 'affine:page', - additional: '{"displayMode":"edgeless"}', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: 'doc-10', - block_id: 'block-10-1', - content: 'Example 1', - flavour: 'affine:paragraph', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspaceId, - doc_id: 'doc-10', - block_id: 'block-10-2', - content: 'Single substitution format 1', - flavour: 'affine:paragraph', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - const result = await searchProvider.aggregate(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - sort: ['_score', { updated_at: 'desc' }, 'doc_id', 'block_id'], - query: { - bool: { - must: [ - { - term: { - workspace_id: { - value: workspaceId, - }, - }, - }, - { - bool: { - must: [ - { - match: { - content: '0.15 week.1', - }, - }, - { - bool: { - should: [ - { - match: { - content: '0.15 week.1', - }, - }, - { - term: { - flavour: { - value: 'affine:page', - boost: 1.5, - }, - }, - }, - ], - }, - }, - ], - }, - }, - ], - }, - }, - aggs: { - result: { - terms: { - field: 'doc_id', - size: 100, - order: { - max_score: 'desc', - }, - }, - aggs: { - max_score: { - max: { - script: { - source: '_score', - }, - }, - }, - result: { - top_hits: { - _source: ['workspace_id', 'doc_id'], - highlight: { - fields: { - content: { - pre_tags: [''], - post_tags: [''], - }, - }, - }, - fields: ['block_id', 'flavour'], - size: 2, - }, - }, - }, - }, - }, - }); - - t.truthy(result.total); - t.is(result.timedOut, false); - t.true(result.buckets.length > 0); - t.truthy(result.buckets[0].key); - t.true(result.buckets[0].count > 0); - t.truthy(result.buckets[0].hits.nodes.length > 0); - t.deepEqual(Object.keys(result.buckets[0].hits.nodes[0]._source), [ - 'workspace_id', - 'doc_id', - ]); - t.snapshot( - result.buckets.map(bucket => ({ - key: bucket.key, - count: bucket.count, - hits: bucket.hits.nodes.map(node => omit(node, ['_score'])), - })) - ); -}); - -// #endregion - -// #region delete by query - -test('should delete by query work', async t => { - const docId = randomUUID(); - - await searchProvider.write( - SearchTable.block, - [ - { - workspace_id: workspace.id, - doc_id: docId, - content: `hello world on search title, ${randomUUID()}`, - flavour: 'affine:page', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - { - workspace_id: workspace.id, - doc_id: docId, - block_id: randomUUID(), - content: `hello world on search title, ${randomUUID()}`, - flavour: 'other:flavour', - created_by_user_id: user.id, - updated_by_user_id: user.id, - created_at: new Date(), - updated_at: new Date(), - }, - ], - { - refresh: true, - } - ); - - const result = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { - workspace_id: workspace.id, - }, - }, - { - term: { - doc_id: docId, - }, - }, - ], - }, - }, - fields: ['block_id'], - sort: ['_score'], - }); - - t.is(result.nodes.length, 2); - - await searchProvider.deleteByQuery( - SearchTable.block, - { - bool: { - must: [ - { - term: { - workspace_id: workspace.id, - }, - }, - { - term: { - doc_id: docId, - }, - }, - ], - }, - }, - { - refresh: true, - } - ); - - const result2 = await searchProvider.search(SearchTable.block, { - _source: ['workspace_id', 'doc_id'], - query: { - bool: { - must: [ - { - term: { - workspace_id: workspace.id, - }, - }, - { - term: { - doc_id: docId, - }, - }, - ], - }, - }, - fields: ['block_id'], - sort: ['_score'], - }); - - t.is(result2.nodes.length, 0); -}); - -// #endregion - -// #region parse es query - -test('should parse es query term work', async t => { - const query = { - term: { - workspace_id: { - value: 'workspaceId1', - }, - }, - }; - - // @ts-expect-error use private method - const result = searchProvider.parseESQuery(query); - - t.snapshot(result); - - const query2 = { - term: { - workspace_id: 'workspaceId1', - }, - }; - - // @ts-expect-error use private method - const result2 = searchProvider.parseESQuery(query2); - - t.snapshot(result2); - - const query3 = { - term: { - flavour: { - value: 'affine:page', - boost: 1.5, - }, - }, - }; - - // @ts-expect-error use private method - const result3 = searchProvider.parseESQuery(query3); - - t.snapshot(result3); - - const query4 = { - term: { - doc_id: { - value: 'docId1', - boost: 1.5, - }, - }, - }; - - // @ts-expect-error use private method - const result4 = searchProvider.parseESQuery(query4); - - t.snapshot(result4); -}); - -test('should parse es query with custom term mapping field work', async t => { - const query = { - bool: { - must: [ - { - term: { - workspace_id: { - value: 'workspaceId1', - }, - }, - }, - { - term: { - doc_id: { - value: 'docId1', - }, - }, - }, - ], - }, - }; - // @ts-expect-error use private method - const result = searchProvider.parseESQuery(query, { - termMappingField: 'equals', - }); - - t.snapshot(result); - - const query2 = { - bool: { - must: { - term: { - workspace_id: 'workspaceId1', - }, - }, - }, - }; - - // @ts-expect-error use private method - const result2 = searchProvider.parseESQuery(query2, { - termMappingField: 'equals', - }); - - t.snapshot(result2); - - const query3 = { - term: { - workspace_id: 'workspaceId1', - }, - }; - - // @ts-expect-error use private method - const result3 = searchProvider.parseESQuery(query3, { - termMappingField: 'equals', - }); - - t.snapshot(result3); -}); - -test('should parse es query with parent and nested must_not work', async t => { - const nestedMust = { - must: [ - { term: { workspace_id: 'workspaceId1' } }, - { bool: { must_not: { term: { doc_id: 'docId1' } } } }, - ], - }; - const parentMustNot = { term: { doc_id: 'docId2' } }; - const expectedMust = [{ equals: { workspace_id: 'workspaceId1' } }]; - const expectedMustNot = ['docId1', 'docId2']; - - const queryWithParentMustNotFirst = { - bool: { must_not: parentMustNot, must: nestedMust.must }, - }; - const queryWithParentMustNotLast = { - bool: { must: nestedMust.must, must_not: parentMustNot }, - }; - - // @ts-expect-error use private method - const result = searchProvider.parseESQuery(queryWithParentMustNotFirst); - // @ts-expect-error use private method - const result2 = searchProvider.parseESQuery(queryWithParentMustNotLast); - - t.deepEqual(result.bool.must, expectedMust); - t.deepEqual(result2.bool.must, expectedMust); - t.deepEqual( - result.bool.must_not.map((clause: any) => clause.equals.doc_id).sort(), - expectedMustNot - ); - t.deepEqual( - result2.bool.must_not.map((clause: any) => clause.equals.doc_id).sort(), - expectedMustNot - ); -}); - -test('should parse es query exists work', async t => { - const query = { - exists: { - field: 'parent_block_id', - }, - }; - - // @ts-expect-error use private method - const result = searchProvider.parseESQuery(query); - - t.snapshot(result); - - const query2 = { - exists: { - field: 'ref_doc_id', - }, - }; - - // @ts-expect-error use private method - const result2 = searchProvider.parseESQuery(query2); - - t.snapshot(result2); -}); - -// #endregion diff --git a/packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts index 75de465ab4..18f5c43252 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts @@ -1,2424 +1,197 @@ -import { randomUUID } from 'node:crypto'; -import { mock } from 'node:test'; - import test from 'ava'; -import { omit, pick } from 'lodash-es'; +import Sinon from 'sinon'; -import { createModule } from '../../../__tests__/create-module'; -import { Mockers } from '../../../__tests__/mocks'; -import { ConfigModule } from '../../../base/config'; -import { ServerConfigModule } from '../../../core/config'; -import { Models } from '../../../models'; -import { SearchProviderFactory } from '../factory'; -import { IndexerModule, IndexerService } from '../index'; -import { ManticoresearchProvider } from '../providers'; -import { UpsertDoc } from '../service'; -import { blockSQL, docSQL, SearchTable } from '../tables'; import { - AggregateInput, - SearchInput, - SearchQueryOccur, - SearchQueryType, -} from '../types'; + InternalServerError, + InvalidIndexerInput, + SearchProviderNotFound, + SpaceAccessDenied, + WorkspacePermissionNotFound, +} from '../../../base'; +import { BackendRuntimeProvider } from '../../../core/backend-runtime'; +import { ServerService } from '../../../core/config'; +import { Models } from '../../../models'; +import { IndexerService } from '../service'; +import { SearchQueryType, SearchTable } from '../types'; -const module = await createModule({ - imports: [ - IndexerModule, - ServerConfigModule, - ConfigModule.override({ - indexer: { - enabled: true, - }, - }), - ], - providers: [IndexerService], -}); -const indexerService = module.get(IndexerService); -const searchProviderFactory = module.get(SearchProviderFactory); -const manticoresearch = module.get(ManticoresearchProvider); -const models = module.get(Models); -const user = await module.create(Mockers.User); -const workspace = await module.create(Mockers.Workspace, { - snapshot: true, - owner: user, +test.afterEach.always(() => { + Sinon.restore(); }); -mock.method(searchProviderFactory, 'get', () => { - return manticoresearch; -}); - -test.after.always(async () => { - await module.close(); -}); - -test.before(async () => { - await manticoresearch.recreateTable(SearchTable.block, blockSQL); - await manticoresearch.recreateTable(SearchTable.doc, docSQL); -}); - -test.afterEach.always(async () => { - await indexerService.deleteByQuery( - SearchTable.doc, - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - refresh: true, - } - ); - await indexerService.deleteByQuery( - SearchTable.block, - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - refresh: true, - } - ); -}); - -// #region deleteByQuery() - -test('should deleteByQuery work', async t => { - const docId1 = randomUUID(); - const docId2 = randomUUID(); - await indexerService.write( - SearchTable.block, - [ - { - workspaceId: workspace.id, - docId: docId1, - blockId: randomUUID(), - content: 'hello world', - flavour: 'affine:page', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - workspaceId: workspace.id, - docId: docId2, - blockId: randomUUID(), - content: 'hello world', - flavour: 'affine:page', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } +test('reflects native search readiness in the Node feature flag', async t => { + const runtime = { + searchStatus: Sinon.stub(), + }; + runtime.searchStatus.onFirstCall().resolves({ ready: true }); + runtime.searchStatus.onSecondCall().resolves({ ready: false }); + const server = { + enableFeature: Sinon.stub(), + disableFeature: Sinon.stub(), + }; + const service = new IndexerService( + runtime as unknown as BackendRuntimeProvider, + {} as Models, + server as unknown as ServerService ); - let result = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.should, - queries: [ - { - type: SearchQueryType.match, - field: 'docId', - match: docId1, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId2, - }, - ], - }, - options: { - fields: ['docId'], - }, - }); + await service.onApplicationBootstrap(); + await service.onConfigChanged({ updates: { indexer: {} } } as never); - t.is(result.total, 2); - t.is(result.nodes.length, 2); - - await indexerService.deleteByQuery( - SearchTable.block, - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.should, - queries: [ - { - type: SearchQueryType.match, - field: 'docId', - match: docId1, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId2, - }, - ], - }, - { - refresh: true, - } - ); - - result = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'docId', - match: docId1, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId2, - }, - ], - }, - options: { - fields: ['docId'], - }, - }); - - t.is(result.total, 0); - t.is(result.nodes.length, 0); + t.true(server.enableFeature.calledOnce); + t.true(server.disableFeature.calledOnce); + t.is(runtime.searchStatus.callCount, 2); }); -// #endregion - -// #region write() - -test('should write throw error when field type wrong', async t => { - await t.throwsAsync( - indexerService.write(SearchTable.block, [ - { - workspaceId: workspace.id, - docId: 'docId1', - blockId: randomUUID(), - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - content: 'hello world', - flavour: 'affine:page', - // @ts-expect-error test error - refDocId: 123, - }, - ]), - { - message: /ref_doc_id/, - } +test('maps native search results and typed errors at the Node boundary', async t => { + const runtime = { + searchAuthorized: Sinon.stub(), + }; + const service = new IndexerService( + runtime as unknown as BackendRuntimeProvider, + {} as Models, + {} as ServerService ); -}); - -test('should write block with array content work', async t => { - const docId = randomUUID(); - const blockId = randomUUID(); - await indexerService.write( - SearchTable.block, - [ - { - workspaceId: workspace.id, - docId, - blockId, - content: ['hello', 'world'], - flavour: 'affine:page', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); - - const result = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - type: SearchQueryType.match, - field: 'content', - match: 'hello world', - }, - ], - }, - options: { - fields: ['content'], - }, - }); - - t.is(result.total, 1); - t.is(result.nodes.length, 1); - t.snapshot( - result.nodes.map(node => ({ - fields: node.fields, - })) - ); -}); - -test('should write 10k docs work', async t => { - const docCount = 10000; - const docs: UpsertDoc[] = []; - for (let i = 0; i < docCount; i++) { - docs.push({ - workspaceId: workspace.id, - docId: randomUUID(), - title: `hello world ${i} ${randomUUID()}`, - summary: `this is a test ${i} ${randomUUID()}`, - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }); - } - await indexerService.write(SearchTable.doc, docs); - - // cleanup - await indexerService.deleteByQuery( - SearchTable.doc, - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - refresh: true, - } - ); - - t.pass(); -}); - -test('should write ref as string[] work', async t => { - const docIds = [randomUUID(), randomUUID(), randomUUID()]; - - await indexerService.write( - SearchTable.block, - [ - { - docId: docIds[0], - workspaceId: workspace.id, - content: 'test1', - flavour: 'markdown', - blockId: randomUUID(), - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date('2025-04-22T00:00:00.000Z'), - updatedAt: new Date('2025-04-22T00:00:00.000Z'), - }, - { - docId: docIds[1], - workspaceId: workspace.id, - content: 'test2', - flavour: 'markdown', - blockId: randomUUID(), - refDocId: [docIds[0]], - ref: ['{"foo": "bar1"}'], - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date('2021-04-22T00:00:00.000Z'), - updatedAt: new Date('2021-04-22T00:00:00.000Z'), - }, - { - docId: docIds[2], - workspaceId: workspace.id, - content: 'test3', - flavour: 'markdown', - blockId: randomUUID(), - refDocId: [docIds[0], docIds[2]], - ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'], - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date('2025-03-22T00:00:00.000Z'), - updatedAt: new Date('2025-03-22T00:00:00.000Z'), - }, - { - docId: docIds[0], - workspaceId: workspace.id, - content: 'test4', - flavour: 'markdown', - blockId: randomUUID(), - refDocId: [docIds[0], docIds[2]], - ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'], - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date('2025-04-22T00:00:00.000Z'), - updatedAt: new Date('2025-04-22T00:00:00.000Z'), - }, - ], - { - refresh: true, - } - ); - - t.pass(); -}); - -// #endregion - -// #region parseInput() - -test('should parse all query work', async t => { const input = { table: SearchTable.block, - query: { type: SearchQueryType.all }, - options: { - fields: ['flavour', 'docId', 'refDocId'], - }, + query: { type: SearchQueryType.match, field: 'content', match: 'hello' }, + options: { fields: ['docId', 'createdAt'] }, }; - - const result = indexerService.parseInput(input); - - t.snapshot(result); -}); - -test('should parse exists query work', async t => { - const input = { - table: SearchTable.block, - query: { type: SearchQueryType.exists, field: 'refDocId' }, - options: { - fields: ['flavour', 'docId', 'refDocId'], - }, - }; - - const result = indexerService.parseInput(input); - - t.snapshot(result); -}); - -test('should parse boost query work', async t => { - const input = { - table: SearchTable.block, - query: { - type: SearchQueryType.boost, - boost: 1.5, - query: { - type: SearchQueryType.match, - field: 'flavour', - match: 'affine:page', - }, - }, - options: { - fields: ['flavour', 'docId', 'refDocId'], - }, - }; - - const result = indexerService.parseInput(input); - - t.snapshot(result); -}); - -test('should parse match query work', async t => { - const input = { - table: SearchTable.block, - query: { - type: SearchQueryType.match, - field: 'flavour', - match: 'affine:page', - }, - options: { - fields: [ - 'flavour', - 'docId', - 'refDocId', - 'parentFlavour', - 'parentBlockId', - 'additional', - 'markdownPreview', - 'createdByUserId', - 'updatedByUserId', - 'createdAt', - 'updatedAt', - ], - }, - }; - - const result = indexerService.parseInput(input); - - t.snapshot(result); -}); - -test('should parse boolean query work', async t => { - const input = { - table: SearchTable.block, - query: { - type: 'boolean', - occur: 'must', - queries: [ + runtime.searchAuthorized.resolves({ + ok: true, + value: { + total: 1, + nodes: [ { - type: 'match', - field: 'workspaceId', - match: 'workspaceId1', - }, - { - type: 'match', - field: 'content', - match: 'hello', - }, - { - type: 'boolean', - occur: 'should', - queries: [ - { - type: 'match', - field: 'content', - match: 'hello', - }, - { - type: 'boost', - boost: 1.5, - query: { - type: 'match', - field: 'flavour', - match: 'affine:page', - }, - }, - ], - }, - ], - }, - options: { - fields: [ - 'flavour', - 'docId', - 'refDocId', - 'parentFlavour', - 'parentBlockId', - 'additional', - 'markdownPreview', - 'createdByUserId', - 'updatedByUserId', - 'createdAt', - 'updatedAt', - ], - }, - }; - - const result = indexerService.parseInput(input as SearchInput); - - t.snapshot(result); -}); - -test('should parse search input highlight work', async t => { - const input = { - table: SearchTable.block, - query: { - type: SearchQueryType.all, - }, - options: { - fields: ['flavour', 'docId', 'refDocId'], - highlights: [{ field: 'content', before: '', end: '' }], - }, - }; - - const result = indexerService.parseInput(input as SearchInput); - - t.snapshot(result); -}); - -test('should parse aggregate input highlight work', async t => { - const input = { - table: SearchTable.doc, - field: 'flavour', - query: { - type: SearchQueryType.all, - }, - options: { - hits: { - fields: ['flavour', 'docId', 'refDocId'], - highlights: [{ field: 'content', before: '', end: '' }], - }, - }, - }; - - const result = indexerService.parseInput(input as AggregateInput); - - t.snapshot(result); -}); - -// #endregion - -// #region search() - -test('should search work', async t => { - const docId1 = randomUUID(); - const docId2 = randomUUID(); - await indexerService.write( - SearchTable.doc, - [ - { - workspaceId: workspace.id, - title: 'hello world', - summary: 'this is a test', - docId: docId1, - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - workspaceId: workspace.id, - title: '你好世界', - summary: '这是测试', - docId: docId2, - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - type: SearchQueryType.match, - field: 'title', - match: 'hello hello', - }, - ], - }, - options: { - fields: ['workspaceId', 'docId', 'title', 'summary'], - highlights: [{ field: 'title', before: '', end: '' }], - }, - }); - - t.truthy(result.nextCursor); - t.is(result.total, 1); - t.is(result.nodes.length, 1); - t.snapshot( - result.nodes.map(node => ({ - fields: omit(node.fields, 'workspaceId', 'docId'), - highlights: node.highlights, - })) - ); - t.deepEqual(result.nodes[0]._source, { - workspaceId: workspace.id, - docId: docId1, - }); - - result = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - type: SearchQueryType.match, - field: 'title', - match: '你好你好', - }, - ], - }, - options: { - fields: ['workspaceId', 'docId', 'title', 'summary'], - highlights: [{ field: 'title', before: '', end: '' }], - }, - }); - - t.truthy(result.nextCursor); - t.is(result.total, 1); - t.is(result.nodes.length, 1); - t.snapshot( - result.nodes.map(node => ({ - fields: omit(node.fields, 'workspaceId', 'docId'), - highlights: node.highlights, - })) - ); - t.deepEqual(result.nodes[0]._source, { - workspaceId: workspace.id, - docId: docId2, - }); -}); - -test('should throw error when limit is greater than 10000', async t => { - await t.throwsAsync( - indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.all, - }, - options: { - fields: ['workspaceId', 'docId', 'title', 'summary'], - pagination: { - limit: 10001, - }, - }, - }), - { - message: 'Invalid indexer input: limit must be less than 10000', - } - ); -}); - -test('should search with exists query work', async t => { - const docId1 = randomUUID(); - const docId2 = randomUUID(); - const docId3 = randomUUID(); - await indexerService.write( - SearchTable.block, - [ - { - workspaceId: workspace.id, - docId: docId1, - blockId: 'blockId1', - content: 'hello world', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - flavour: 'affine:page', - parentBlockId: 'blockId2', - }, - { - workspaceId: workspace.id, - docId: docId2, - blockId: 'blockId2', - content: 'hello world', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date('2025-04-24T00:00:00.000Z'), - flavour: 'affine:page', - refDocId: [docId1], - ref: ['{"type": "affine:page", "id": "docId1"}'], - }, - { - workspaceId: workspace.id, - docId: docId3, - blockId: 'blockId3', - content: 'hello world', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - flavour: 'affine:page', - refDocId: [docId2, docId1], - ref: [ - '{"type": "affine:page", "id": "docId2"}', - '{"type": "affine:page", "id": "docId1"}', - ], - }, - ], - { - refresh: true, - } - ); - - const result = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.should, - queries: [docId1, docId2, docId3].map(docId => ({ - type: SearchQueryType.match, - field: 'docId', - match: docId, - })), - }, - { - type: SearchQueryType.exists, - field: 'refDocId', - }, - ], - }, - ], - }, - options: { - fields: ['blockId', 'refDocId', 'ref'], - }, - }); - - t.is(result.total, 2); - t.is(result.nodes.length, 2); - t.deepEqual(result.nodes[0].fields, { - blockId: ['blockId3'], - refDocId: [docId2, docId1], - ref: [ - '{"type": "affine:page", "id": "docId2"}', - '{"type": "affine:page", "id": "docId1"}', - ], - }); - t.deepEqual(result.nodes[1].fields, { - blockId: ['blockId2'], - refDocId: [docId1], - ref: ['{"type": "affine:page", "id": "docId1"}'], - }); - - const result2 = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.should, - queries: [docId1, docId2, docId3].map(docId => ({ - type: SearchQueryType.match, - field: 'docId', - match: docId, - })), - }, - { - type: SearchQueryType.exists, - field: 'parentBlockId', - }, - ], - }, - ], - }, - options: { - fields: ['blockId', 'refDocId', 'ref', 'parentBlockId'], - }, - }); - - t.is(result2.total, 1); - t.is(result2.nodes.length, 1); - t.snapshot( - result2.nodes.map(node => ({ - fields: node.fields, - })) - ); -}); - -test('should get all title and docId from doc table', async t => { - const docIds: string[] = []; - for (let i = 0; i < 10101; i++) { - docIds.push(randomUUID()); - } - await indexerService.write( - SearchTable.doc, - docIds.map(docId => ({ - workspaceId: workspace.id, - docId, - title: `hello world ${docId}`, - summary: `this is a test ${docId}`, - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - })), - { - refresh: true, - } - ); - - let result = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - type: SearchQueryType.all, - }, - ], - }, - options: { - fields: ['title', 'docId'], - pagination: { - limit: 10000, - }, - }, - }); - - const searchDocIds: string[] = []; - for (const node of result.nodes) { - searchDocIds.push(node.fields.docId[0] as string); - } - while (result.nextCursor) { - result = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, + id: 'node', + score: 1, + fields: { + workspace_id: ['workspace'], + doc_id: ['doc'], + created_at: [2_000], }, - { - type: SearchQueryType.all, - }, - ], - }, - options: { - fields: ['title', 'docId'], - pagination: { - limit: 10000, - cursor: result.nextCursor, + highlights: { markdown_preview: ['hello'] }, }, - }, - }); - for (const node of result.nodes) { - searchDocIds.push(node.fields.docId[0] as string); - } + ], + }, + }); + + const result = await service.search('actor', 'workspace', input); + t.deepEqual(result.nodes[0]._source, { + workspaceId: 'workspace', + docId: 'doc', + }); + t.true(result.nodes[0].fields.createdAt[0] instanceof Date); + t.deepEqual(result.nodes[0].highlights, { + markdownPreview: ['hello'], + }); + + for (const [errorCode, expected] of [ + ['workspace_denied', SpaceAccessDenied], + ['invalid_request', InvalidIndexerInput], + ['unsupported_query', InvalidIndexerInput], + ['provider_unavailable', SearchProviderNotFound], + ['permission_unavailable', WorkspacePermissionNotFound], + ['unexpected', InternalServerError], + ] as const) { + runtime.searchAuthorized.resolves({ ok: false, errorCode }); + const error = await t.throwsAsync( + service.search('actor', 'workspace', input) + ); + t.true(error instanceof expected, errorCode); } - - t.is(searchDocIds.length, docIds.length); - t.deepEqual(searchDocIds.sort(), docIds.sort()); }); -test('should search with bool must multiple conditions query work', async t => { - const docId1 = randomUUID(); - const docId2 = randomUUID(); - const docId3 = randomUUID(); - const blockId1 = randomUUID(); - const blockId2 = randomUUID(); - const blockId3 = randomUUID(); - const blockId4 = randomUUID(); - const blockId5 = randomUUID(); - await indexerService.write( - SearchTable.block, - [ - // ref to docId1, ignore current docId1 - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId1, - blockId: blockId1, - refDocId: [docId1], - ref: ['{"foo": "bar1"}'], - content: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - // ref to docId1, docId2, ignore current docId1 - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId1, - blockId: blockId2, - refDocId: [docId1, docId2], - ref: ['{"foo": "bar1"}', '{"foo": "bar2"}'], - content: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - // matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId2, - blockId: blockId3, - refDocId: [docId1, docId2], - ref: ['{"foo": "bar1"}', '{"foo": "bar2"}'], - content: 'hello world, this is a title', - parentBlockId: 'parentBlockId1', - parentFlavour: 'affine:database', - additional: '{"foo": "bar3"}', - markdownPreview: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date('2025-04-26T00:00:00.000Z'), - }, - // matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId2, - blockId: blockId4, - refDocId: [docId1], - ref: ['{"foo": "bar1"}'], - content: 'hello world, this is a title', - parentBlockId: 'parentBlockId2', - parentFlavour: 'affine:database', - additional: '{"foo": "bar3"}', - markdownPreview: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date('2025-04-25T00:00:00.000Z'), - }, - // matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId3, - blockId: blockId5, - refDocId: [docId2, docId1, docId3], - ref: ['{"foo": "bar2"}', '{"foo": "bar1"}', '{"foo": "bar3"}'], - content: 'hello world, this is a title', - parentBlockId: 'parentBlockId3', - parentFlavour: 'affine:database', - additional: '{"foo": "bar3"}', - markdownPreview: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date('2025-04-24T00:00:00.000Z'), - }, - // not matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId3, - blockId: 'blockId6', - refDocId: [docId2, docId3], - ref: ['{"foo": "bar2"}', '{"foo": "bar3"}'], - content: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - // not matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId3, - blockId: 'blockId7', - content: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - // not matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId2, - blockId: 'blockId8', - refDocId: [docId1], - ref: ['{"foo": "bar1"}'], - content: 'hello world, this is a title', - parentBlockId: 'parentBlockId2', - parentFlavour: 'affine:text', - additional: '{"foo": "bar3"}', - markdownPreview: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date('2025-04-25T00:00:00.000Z'), - }, - ], - { - refresh: true, - } - ); - - const result = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'refDocId', - match: docId1, - }, - { - type: SearchQueryType.match, - field: 'parentFlavour', - match: 'affine:database', - }, - // Ignore if it is a link to the current document. - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must_not, - queries: [ - { - type: SearchQueryType.match, - field: 'docId', - match: docId1, - }, - ], - }, - ], - }, - options: { - fields: ['docId', 'blockId', 'parentBlockId', 'additional'], - pagination: { - limit: 100, - }, - }, - }); - - t.is(result.total, 3); - t.is(result.nodes.length, 3); - t.deepEqual(result.nodes[0].fields, { - docId: [docId2], - blockId: [blockId3], - parentBlockId: ['parentBlockId1'], - additional: ['{"foo": "bar3"}'], - }); - t.deepEqual(result.nodes[1].fields, { - docId: [docId2], - blockId: [blockId4], - parentBlockId: ['parentBlockId2'], - additional: ['{"foo": "bar3"}'], - }); - t.deepEqual(result.nodes[2].fields, { - docId: [docId3], - blockId: [blockId5], - parentBlockId: ['parentBlockId3'], - additional: ['{"foo": "bar3"}'], - }); -}); - -test('should search a doc summary work', async t => { - const docId1 = randomUUID(); - await indexerService.write( - SearchTable.doc, - [ - { - workspaceId: workspace.id, - docId: docId1, - title: 'hello world, this is a title', - summary: 'hello world, this is a summary', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); - - const result = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId1, - }, - ], - }, - options: { - fields: ['summary'], - }, - }); - - t.is(result.total, 1); - t.is(result.nodes.length, 1); - t.snapshot( - result.nodes.map(node => ({ - fields: node.fields, - })) - ); -}); - -// #endregion - -// #region aggregate() - -test('should aggregate work', async t => { - const docId1 = randomUUID(); - const docId2 = randomUUID(); - const blockId1 = randomUUID(); - const blockId2 = randomUUID(); - const blockId3 = randomUUID(); - await indexerService.write( - SearchTable.block, - [ - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId1, - blockId: blockId3, - content: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - workspaceId: workspace.id, - flavour: 'affine:text', - docId: docId1, - blockId: blockId1, - content: 'hello world, this is a block', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - workspaceId: workspace.id, - flavour: 'affine:text', - docId: docId1, - blockId: randomUUID(), - content: 'this is a block', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - workspaceId: workspace.id, - flavour: 'affine:text', - docId: docId2, - blockId: blockId2, - content: 'hello world, this is a test block', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - // not match - { - workspaceId: workspace.id, - flavour: 'affine:database', - docId: docId2, - blockId: randomUUID(), - content: 'this is a test block', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); - - const result = await indexerService.aggregate({ - table: SearchTable.block, - field: 'docId', - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'content', - match: 'hello', - }, - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.should, - queries: [ +test('searchDocs keeps filtering and enrichment in Node', async t => { + const runtime = { + aggregateAuthorized: Sinon.stub().resolves({ + ok: true, + value: { + total: 1, + hasMore: false, + buckets: [ + { + key: 'doc', + count: 1, + hits: { + nodes: [ { - type: SearchQueryType.match, - field: 'content', - match: 'hello', - }, - { - type: SearchQueryType.boost, - boost: 1.5, - query: { - type: SearchQueryType.match, - field: 'flavour', - match: 'affine:page', + id: 'block', + score: 1, + fields: { + workspace_id: ['workspace'], + doc_id: ['doc'], + block_id: ['block'], + unit_id: ['unit'], + projection_version: [1], + source_hash: ['hash'], + visibility: ['visible'], + source_block_id: ['source-block'], + flavour: ['affine:paragraph'], + content: ['body'], + created_at: [2_000], + updated_at: [3_000], + created_by_user_id: ['creator'], + updated_by_user_id: ['updater'], }, + highlights: { content: ['body'] }, }, ], }, - ], - }, - ], - }, - options: { - hits: { - fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], - highlights: [{ field: 'content', before: '', end: '' }], - }, - }, - }); - - t.is(result.total, 3); - t.is(result.buckets.length, 2); - t.deepEqual(result.buckets[0].key, docId1); - t.is(result.buckets[0].count, 2); - // match affine:page first - t.deepEqual(result.buckets[0].hits.nodes[0].fields, { - workspaceId: [workspace.id], - docId: [docId1], - blockId: [blockId3], - content: ['hello world, this is a title'], - flavour: ['affine:page'], - }); - t.deepEqual(result.buckets[0].hits.nodes[0].highlights, { - content: ['hello world, this is a title'], - }); - t.deepEqual(result.buckets[0].hits.nodes[0]._source, { - workspaceId: workspace.id, - docId: docId1, - }); - t.deepEqual(result.buckets[0].hits.nodes[1].fields, { - workspaceId: [workspace.id], - docId: [docId1], - blockId: [blockId1], - content: ['hello world, this is a block'], - flavour: ['affine:text'], - }); - t.deepEqual(result.buckets[0].hits.nodes[1].highlights, { - content: ['hello world, this is a block'], - }); - t.deepEqual(result.buckets[0].hits.nodes[1]._source, { - workspaceId: workspace.id, - docId: docId1, - }); - t.deepEqual(result.buckets[1].key, docId2); - t.is(result.buckets[1].count, 1); - t.deepEqual(result.buckets[1].hits.nodes[0].fields, { - workspaceId: [workspace.id], - docId: [docId2], - blockId: [blockId2], - content: ['hello world, this is a test block'], - flavour: ['affine:text'], - }); - t.deepEqual(result.buckets[1].hits.nodes[0].highlights, { - content: ['hello world, this is a test block'], - }); - t.deepEqual(result.buckets[1].hits.nodes[0]._source, { - workspaceId: workspace.id, - docId: docId2, - }); -}); - -test('should aggregate with bool must_not query work', async t => { - const docId1 = randomUUID(); - const docId2 = randomUUID(); - const docId3 = randomUUID(); - const blockId1 = randomUUID(); - const blockId2 = randomUUID(); - const blockId3 = randomUUID(); - const blockId4 = randomUUID(); - const blockId5 = randomUUID(); - await indexerService.write( - SearchTable.block, - [ - // ref to docId1, ignore current docId1 - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId1, - blockId: blockId1, - refDocId: [docId1], - ref: ['{"foo": "bar1"}'], - content: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - // ref to docId1, docId2, ignore current docId1 - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId1, - blockId: blockId2, - refDocId: [docId1, docId2], - ref: ['{"foo": "bar1"}', '{"foo": "bar2"}'], - content: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - // matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId2, - blockId: blockId3, - refDocId: [docId1, docId2], - ref: ['{"foo": "bar1"}', '{"foo": "bar2"}'], - content: 'hello world, this is a title', - parentBlockId: 'parentBlockId1', - parentFlavour: 'affine:database', - additional: '{"foo": "bar3"}', - markdownPreview: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date('2025-04-26T00:00:00.000Z'), - }, - // matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId2, - blockId: blockId4, - refDocId: [docId1], - ref: ['{"foo": "bar1"}'], - content: 'hello world, this is a title', - parentBlockId: 'parentBlockId2', - parentFlavour: 'affine:database', - additional: '{"foo": "bar3"}', - markdownPreview: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date('2025-04-25T00:00:00.000Z'), - }, - // matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId3, - blockId: blockId5, - refDocId: [docId2, docId1, docId3], - ref: ['{"foo": "bar2"}', '{"foo": "bar1"}', '{"foo": "bar3"}'], - content: 'hello world, this is a title', - parentBlockId: 'parentBlockId3', - parentFlavour: 'affine:database', - additional: '{"foo": "bar3"}', - markdownPreview: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date('2025-04-24T00:00:00.000Z'), - }, - // not matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId3, - blockId: 'blockId6', - refDocId: [docId2, docId3], - ref: ['{"foo": "bar2"}', '{"foo": "bar3"}'], - content: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - // not matched - { - workspaceId: workspace.id, - flavour: 'affine:page', - docId: docId3, - blockId: 'blockId7', - content: 'hello world, this is a title', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); - - const result = await indexerService.aggregate({ - table: SearchTable.block, - field: 'docId', - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'refDocId', - match: docId1, - }, - // Ignore if it is a link to the current document. - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must_not, - queries: [ - { - type: SearchQueryType.match, - field: 'docId', - match: docId1, - }, - ], - }, - ], - }, - options: { - pagination: { - limit: 100, - }, - hits: { - fields: [ - 'docId', - 'blockId', - 'parentBlockId', - 'parentFlavour', - 'additional', - 'markdownPreview', + }, ], - pagination: { - limit: 5, - }, - }, - }, - }); - - t.is(result.total, 3); - t.is(result.buckets.length, 2); - - t.is(result.buckets[0].key, docId2); - t.is(result.buckets[0].count, 2); - t.deepEqual( - pick(result.buckets[0].hits.nodes[0].fields, 'docId', 'blockId'), - { - docId: [docId2], - blockId: [blockId3], - } - ); - t.deepEqual( - pick(result.buckets[0].hits.nodes[1].fields, 'docId', 'blockId'), - { - docId: [docId2], - blockId: [blockId4], - } - ); - - t.is(result.buckets[1].key, docId3); - t.is(result.buckets[1].count, 1); - t.deepEqual( - pick(result.buckets[1].hits.nodes[0].fields, 'docId', 'blockId'), - { - docId: [docId3], - blockId: [blockId5], - } - ); - - t.snapshot( - result.buckets.map(bucket => ({ - count: bucket.count, - hits: bucket.hits.nodes.map(node => ({ - fields: omit(node.fields, 'docId', 'blockId'), - })), - })) - ); -}); - -test('should throw error when field is not allowed in aggregate input', async t => { - await t.throwsAsync( - indexerService.aggregate({ - table: SearchTable.block, - field: 'workspaceId', - query: { - type: SearchQueryType.all, - }, - options: { - hits: { - fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], - }, }, }), - { - message: - 'Invalid indexer input: aggregate field "workspaceId" is not allowed', - } - ); -}); - -// #endregion - -// #region deleteWorkspace() - -test('should delete workspace work', async t => { - const workspaceId = randomUUID(); - const docId1 = randomUUID(); - const docId2 = randomUUID(); - await indexerService.write( - SearchTable.doc, - [ - { - workspaceId, - docId: docId1, - title: 'hello world', - summary: 'this is a test', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - workspaceId, - docId: docId2, - title: 'hello world', - summary: 'this is a test', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); - await indexerService.write( - SearchTable.block, - [ - { - workspaceId, - docId: docId1, - blockId: randomUUID(), - content: 'hello world', - flavour: 'affine:text', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, + }; + const creator = { id: 'creator', name: 'Creator' }; + const updater = { id: 'updater', name: 'Updater' }; + const models = { + doc: { + findMetas: Sinon.stub().resolves([ + { docId: 'doc', title: 'Fallback title' }, + ]), }, - options: { - fields: ['workspaceId', 'docId', 'title', 'summary'], - }, - }); - - t.is(result.total, 2); - t.is(result.nodes.length, 2); - - let result2 = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - options: { - fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], - }, - }); - - t.is(result2.total, 1); - t.is(result2.nodes.length, 1); - - await indexerService.deleteWorkspace(workspaceId, { - refresh: true, - }); - - result = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - options: { - fields: ['workspaceId', 'docId', 'title', 'summary'], - }, - }); - t.is(result.total, 0); - t.is(result.nodes.length, 0); - - result2 = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - options: { - fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], - }, - }); - - t.is(result2.total, 0); - t.is(result2.nodes.length, 0); -}); - -// #endregion - -// #region deleteDoc() - -test('should delete doc work', async t => { - const workspaceId = randomUUID(); - const docId1 = randomUUID(); - const docId2 = randomUUID(); - await indexerService.write( - SearchTable.doc, - [ - { - workspaceId, - docId: docId1, - title: 'hello world', - summary: 'this is a test', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - workspaceId, - docId: docId2, - title: 'hello world', - summary: 'this is a test', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); - await indexerService.write( - SearchTable.block, - [ - { - workspaceId, - docId: docId1, - blockId: randomUUID(), - content: 'hello world', - flavour: 'affine:text', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - workspaceId, - docId: docId2, - blockId: randomUUID(), - content: 'hello world', - flavour: 'affine:text', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); - - let result1 = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId1, - }, - ], - }, - options: { - fields: ['workspaceId', 'docId', 'title', 'summary'], - }, - }); - - t.is(result1.total, 1); - t.is(result1.nodes.length, 1); - t.deepEqual(result1.nodes[0].fields.docId, [docId1]); - - let result2 = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId2, - }, - ], - }, - options: { - fields: ['workspaceId', 'docId', 'title', 'summary'], - }, - }); - - t.is(result2.total, 1); - t.is(result2.nodes.length, 1); - t.deepEqual(result2.nodes[0].fields.docId, [docId2]); - - let result3 = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId1, - }, - ], - }, - options: { - fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], - }, - }); - t.is(result3.total, 1); - t.is(result3.nodes.length, 1); - t.deepEqual(result3.nodes[0].fields.docId, [docId1]); - - let result4 = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId2, - }, - ], - }, - options: { - fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], - }, - }); - t.is(result4.total, 1); - t.is(result4.nodes.length, 1); - t.deepEqual(result4.nodes[0].fields.docId, [docId2]); - - await indexerService.deleteDoc(workspaceId, docId1, { - refresh: true, - }); - - // make sure the docId1 is deleted - result1 = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId1, - }, - ], - }, - options: { - fields: ['workspaceId', 'docId', 'title', 'summary'], - }, - }); - - t.is(result1.total, 0); - t.is(result1.nodes.length, 0); - - // make sure the docId2 is not deleted - result2 = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId2, - }, - ], - }, - options: { - fields: ['workspaceId', 'docId', 'title', 'summary'], - }, - }); - - t.is(result2.total, 1); - t.is(result2.nodes.length, 1); - t.deepEqual(result2.nodes[0].fields.docId, [docId2]); - - // make sure the docId1 block is deleted - result3 = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId1, - }, - ], - }, - options: { - fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], - }, - }); - - t.is(result3.total, 0); - t.is(result3.nodes.length, 0); - - // docId2 block should not be deleted - result4 = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId2, - }, - ], - }, - options: { - fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], - }, - }); - - t.is(result4.total, 1); - t.is(result4.nodes.length, 1); - t.deepEqual(result4.nodes[0].fields.docId, [docId2]); -}); - -// #endregion - -// #region listDocIds() - -test('should list doc ids work', async t => { - const workspaceId = randomUUID(); - const docs = []; - const docCount = 20011; - for (let i = 0; i < docCount; i++) { - docs.push({ - workspaceId, - docId: randomUUID(), - title: `hello world ${i} ${randomUUID()}`, - summary: `this is a test ${i} ${randomUUID()}`, - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }); - } - await indexerService.write(SearchTable.doc, docs, { - refresh: true, - }); - - const docIds = await indexerService.listDocIds(workspaceId); - - t.is(docIds.length, docCount); - t.deepEqual(docIds.sort(), docs.map(doc => doc.docId).sort()); - - await indexerService.deleteWorkspace(workspaceId, { - refresh: true, - }); - const docIds2 = await indexerService.listDocIds(workspaceId); - - t.is(docIds2.length, 0); -}); - -// #endregion - -// #region indexDoc() - -test('should index doc work', async t => { - const docSnapshot = await module.create(Mockers.DocSnapshot, { - workspaceId: workspace.id, - user, - }); - - await indexerService.indexDoc(workspace.id, docSnapshot.id, { - refresh: true, - }); - - const result = await indexerService.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.match, - field: 'docId', - match: docSnapshot.id, - }, - options: { - fields: ['workspaceId', 'docId', 'title', 'summary'], - }, - }); - - t.is(result.total, 1); - t.deepEqual(result.nodes[0].fields.workspaceId, [workspace.id]); - t.deepEqual(result.nodes[0].fields.docId, [docSnapshot.id]); - t.snapshot(omit(result.nodes[0].fields, ['workspaceId', 'docId'])); - - // search blocks - const result2 = await indexerService.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - { - type: SearchQueryType.match, - field: 'content', - match: - 'For developers or installations guides, please go to AFFiNE Doc', - }, - ], - }, - options: { - fields: [ - 'workspaceId', - 'docId', - 'blockId', - 'unitId', - 'projectionVersion', - 'sourceHash', - 'content', - 'flavour', - ], - highlights: [ - { - field: 'content', - before: '', - end: '', - }, - ], - pagination: { - limit: 2, - }, - }, - }); - - t.is(result2.nodes.length, 2); - t.true( - result2.nodes.every( - node => - node.fields.unitId.length === 1 && - node.fields.projectionVersion[0] === 1 && - node.fields.sourceHash.length === 1 - ) - ); - t.is(new Set(result2.nodes.map(node => node.fields.sourceHash[0])).size, 1); - t.snapshot( - result2.nodes.map(node => - omit(node.fields, [ - 'workspaceId', - 'docId', - 'unitId', - 'projectionVersion', - 'sourceHash', - ]) - ) - ); -}); -// #endregion - -// #region searchBlobNames() - -test('should search blob names from doc snapshot work', async t => { - const docSnapshot = await module.create(Mockers.DocSnapshot, { - workspaceId: workspace.id, - user, - snapshotFile: 'test-doc-with-blob.snapshot.bin', - }); - - await indexerService.indexDoc(workspace.id, docSnapshot.id, { - refresh: true, - }); - - const blobNameMap = await indexerService.searchBlobNames(workspace.id, [ - 'ldZMrM4PDlsNG4Q4YvCsz623h6TKu4qI9_FpTqIypfw=', - ]); - - t.snapshot(blobNameMap); -}); - -test('should search blob names work', async t => { - const workspaceId = randomUUID(); - const blobId1 = 'blob1'; - const blobId2 = 'blob2'; - const blobId3 = 'blob3'; - const blobId4 = 'blob4'; - - await indexerService.write( - SearchTable.block, - [ - { - workspaceId, - blob: blobId1, - content: 'blob1 name.txt', - flavour: 'affine:attachment', - docId: randomUUID(), - blockId: randomUUID(), - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - workspaceId, - blob: blobId2, - content: 'blob2 name.md', - flavour: 'affine:attachment', - docId: randomUUID(), - blockId: randomUUID(), - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - { - workspaceId, - blob: blobId3, - content: 'blob3 name.docx', - flavour: 'affine:attachment', - docId: randomUUID(), - blockId: randomUUID(), - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - // no attachment - { - workspaceId, - blob: blobId3, - content: 'mock blob3 content', - flavour: 'affine:page', - docId: randomUUID(), - blockId: randomUUID(), - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - { - refresh: true, - } - ); - - const blobNameMap = await indexerService.searchBlobNames(workspaceId, [ - blobId1, - blobId2, - blobId3, - blobId4, - ]); - - t.is(blobNameMap.size, 3); - t.snapshot( - Array.from(blobNameMap.entries()).sort((a, b) => a[0].localeCompare(b[0])) - ); -}); - -// #endregion - -// #region searchDocsByKeyword() - -test('should search docs by keyword work', async t => { - const workspaceId = workspace.id; - const docId1 = randomUUID(); - const docId2 = randomUUID(); - const docId3 = randomUUID(); - const docId4 = randomUUID(); - - await module.create(Mockers.DocMeta, { - workspaceId, - docId: docId1, - title: 'hello world 1', - }); - await module.create(Mockers.DocMeta, { - workspaceId, - docId: docId2, - title: 'hello world 2', - }); - await module.create(Mockers.DocMeta, { - workspaceId, - docId: docId3, - title: 'hello world 3', - }); - - await indexerService.write( - SearchTable.block, - [ - { - workspaceId, - docId: docId1, - blockId: 'block1', - content: 'hello world', - flavour: 'affine:page', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date('2025-06-20T00:00:00.000Z'), - updatedAt: new Date('2025-06-20T00:00:00.000Z'), - }, - { - workspaceId, - docId: docId2, - blockId: 'block2', - content: 'hello world 2', - flavour: 'affine:text', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date('2025-06-20T00:00:01.000Z'), - updatedAt: new Date('2025-06-20T00:00:01.000Z'), - }, - { - workspaceId, - docId: docId3, - blockId: 'block3', - content: 'hello world 3', - flavour: 'affine:text', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date('2025-06-20T00:00:02.000Z'), - updatedAt: new Date('2025-06-20T00:00:02.000Z'), - }, - { - workspaceId, - docId: docId4, - blockId: 'block4', - content: 'hello world 4', - flavour: 'affine:text', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date('2025-06-20T00:00:03.000Z'), - updatedAt: new Date('2025-06-20T00:00:03.000Z'), - }, - ], - { - refresh: true, - } - ); - - const rows = await indexerService.searchDocsByKeyword(workspaceId, 'hello'); - - t.is(rows.length, 4); - t.snapshot( - rows - .map(row => - omit(row, [ - 'docId', - 'createdByUserId', - 'updatedByUserId', - 'createdByUser', - 'updatedByUser', + user: { + getPublicUsersMap: Sinon.stub().resolves( + new Map([ + ['creator', creator], + ['updater', updater], ]) - ) - .sort((a, b) => a.blockId.localeCompare(b.blockId)) - ); -}); - -test('should search docs by keyword with doc id filter', async t => { - const workspaceId = workspace.id; - const docId1 = randomUUID(); - const docId2 = randomUUID(); - - await module.create(Mockers.DocMeta, { - workspaceId, - docId: docId1, - title: 'hello filtered 1', - }); - await module.create(Mockers.DocMeta, { - workspaceId, - docId: docId2, - title: 'hello filtered 2', - }); - - await indexerService.write( - SearchTable.block, - [ - { - workspaceId, - docId: docId1, - blockId: 'filtered-block1', - content: 'hello filtered', - flavour: 'affine:text', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date('2025-06-20T00:00:00.000Z'), - updatedAt: new Date('2025-06-20T00:00:00.000Z'), - }, - { - workspaceId, - docId: docId2, - blockId: 'filtered-block2', - content: 'hello filtered', - flavour: 'affine:text', - createdByUserId: user.id, - updatedByUserId: user.id, - createdAt: new Date('2025-06-20T00:00:01.000Z'), - updatedAt: new Date('2025-06-20T00:00:01.000Z'), - }, - ], - { - refresh: true, - } - ); - - const rows = await indexerService.searchDocsByKeyword( - workspaceId, - 'hello filtered', - { - docIds: [docId2], - } + ), + }, + }; + const service = new IndexerService( + runtime as unknown as BackendRuntimeProvider, + models as unknown as Models, + {} as ServerService ); t.deepEqual( - rows.map(row => row.docId), - [docId2] + await service.searchDocsByKeyword('actor', 'workspace', 'body', { + docIds: [], + }), + [] + ); + t.false(runtime.aggregateAuthorized.called); + const docs = await service.searchDocsByKeyword('actor', 'workspace', 'body', { + limit: 5, + docIds: ['doc'], + }); + + t.is(docs[0].docId, 'doc'); + t.is(docs[0].blockId, 'source-block'); + t.is(docs[0].title, 'Fallback title'); + t.is(docs[0].highlight, 'body'); + t.deepEqual(docs[0].createdByUser, creator); + t.deepEqual(docs[0].updatedByUser, updater); + const request = runtime.aggregateAuthorized.firstCall.args[2]; + t.is(request.options.pagination?.limit, 5); + t.true(JSON.stringify(request.query).includes('doc')); + t.true( + models.doc.findMetas.calledOnceWithExactly( + [{ workspaceId: 'workspace', docId: 'doc' }], + { + select: { title: true }, + } + ) ); }); - -// #endregion - -test('should rebuild manticore indexes and requeue workspaces', async t => { - const workspace1 = await module.create(Mockers.Workspace, { - indexed: true, - }); - const workspace2 = await module.create(Mockers.Workspace, { - indexed: true, - }); - const queueCount = module.queue.count('indexer.indexWorkspace'); - - await indexerService.rebuildManticoreIndexes(); - - const queuedWorkspaceIds = new Set( - module.queue.add - .getCalls() - .filter(call => call.args[0] === 'indexer.indexWorkspace') - .slice(queueCount) - .map(call => call.args[1].workspaceId) - ); - - t.true(queuedWorkspaceIds.has(workspace1.id)); - t.true(queuedWorkspaceIds.has(workspace2.id)); - - t.is((await models.workspace.get(workspace1.id))?.indexed, false); - t.is((await models.workspace.get(workspace2.id))?.indexed, false); -}); diff --git a/packages/backend/server/src/plugins/indexer/config.ts b/packages/backend/server/src/plugins/indexer/config.ts index b50b011f4f..804682b17f 100644 --- a/packages/backend/server/src/plugins/indexer/config.ts +++ b/packages/backend/server/src/plugins/indexer/config.ts @@ -3,6 +3,7 @@ import { z } from 'zod'; import { defineModuleConfig } from '../../base'; export enum SearchProviderType { + Embedded = 'embedded', Manticoresearch = 'manticoresearch', Elasticsearch = 'elasticsearch', } @@ -34,14 +35,14 @@ defineModuleConfig('indexer', { env: ['AFFINE_INDEXER_ENABLED', 'boolean'], }, 'provider.type': { - desc: 'Indexer search service provider name', - default: SearchProviderType.Manticoresearch, + desc: 'Indexer search provider. Self-hosted uses the embedded provider by default; remote providers require an endpoint.', + default: SearchProviderType.Embedded, shape: SearchProviderTypeSchema, env: ['AFFINE_INDEXER_SEARCH_PROVIDER', 'string'], }, 'provider.endpoint': { - desc: 'Indexer search service endpoint', - default: 'http://localhost:9308', + desc: 'Remote indexer endpoint. Not used by the embedded provider.', + default: '', env: ['AFFINE_INDEXER_SEARCH_ENDPOINT', 'string'], validate: val => { // allow to be nullable and empty string diff --git a/packages/backend/server/src/plugins/indexer/event.ts b/packages/backend/server/src/plugins/indexer/event.ts index 1b57085558..b9317c2145 100644 --- a/packages/backend/server/src/plugins/indexer/event.ts +++ b/packages/backend/server/src/plugins/indexer/event.ts @@ -1,21 +1,45 @@ import { Injectable } from '@nestjs/common'; -import { Cron, CronExpression } from '@nestjs/schedule'; -import { Config, JobQueue, OnEvent } from '../../base'; +import { JobQueue, OnEvent } from '../../base'; @Injectable() export class IndexerEvent { - constructor( - private readonly queue: JobQueue, - private readonly config: Config - ) {} + constructor(private readonly queue: JobQueue) {} + + @OnEvent('doc.grants.changed') + async reindexDocOnGrantChange({ + workspaceId, + docId, + }: Events['doc.grants.changed']) { + await this.indexDoc({ workspaceId, docId }); + } + + @OnEvent('doc.owner.changed') + async reindexDocOnOwnerChange({ + workspaceId, + docId, + }: Events['doc.owner.changed']) { + await this.indexDoc({ workspaceId, docId }); + } + + @OnEvent('doc.default_role.changed') + async reindexDocOnDefaultRoleChange({ + workspaceId, + docId, + }: Events['doc.default_role.changed']) { + await this.indexDoc({ workspaceId, docId }); + } + + @OnEvent('doc.public_state.changed') + async reindexDocOnPublicStateChange({ + workspaceId, + docId, + }: Events['doc.public_state.changed']) { + await this.indexDoc({ workspaceId, docId }); + } @OnEvent('doc.updated') async indexDoc({ workspaceId, docId }: Events['doc.updated']) { - if (!this.config.indexer.enabled) { - return; - } - await this.queue.add( 'indexer.indexDoc', { @@ -31,10 +55,6 @@ export class IndexerEvent { @OnEvent('doc.snapshot.updated') async indexWorkspace({ workspaceId, docId }: Events['doc.snapshot.updated']) { - if (!this.config.indexer.enabled) { - return; - } - if (workspaceId !== docId) { return; } @@ -48,10 +68,6 @@ export class IndexerEvent { @OnEvent('user.deleted') async deleteUserWorkspaces(payload: Events['user.deleted']) { - if (!this.config.indexer.enabled) { - return; - } - for (const workspace of payload.ownedWorkspaces) { await this.queue.add( 'indexer.deleteWorkspace', @@ -65,21 +81,4 @@ export class IndexerEvent { ); } } - - @Cron(CronExpression.EVERY_30_SECONDS) - async autoIndexWorkspaces() { - if (!this.config.indexer.enabled) { - return; - } - - await this.queue.add( - 'indexer.autoIndexWorkspaces', - {}, - { - // make sure only one job is running at a time - delay: 30 * 1000, - jobId: 'autoIndexWorkspaces', - } - ); - } } diff --git a/packages/backend/server/src/plugins/indexer/factory.ts b/packages/backend/server/src/plugins/indexer/factory.ts deleted file mode 100644 index 00710b6676..0000000000 --- a/packages/backend/server/src/plugins/indexer/factory.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; - -import { SearchProviderNotFound } from '../../base'; -import { ServerFeature, ServerService } from '../../core'; -import { SearchProviderType } from './config'; -import type { SearchProvider } from './providers/def'; - -@Injectable() -export class SearchProviderFactory { - constructor(private readonly server: ServerService) {} - - private readonly logger = new Logger(SearchProviderFactory.name); - readonly #providers = new Map(); - #providerType: SearchProviderType | undefined; - - get(): SearchProvider { - const provider = - this.#providerType && this.#providers.get(this.#providerType); - if (!provider) { - throw new SearchProviderNotFound(); - } - return provider; - } - - register(provider: SearchProvider) { - if (this.#providers.has(provider.type)) { - return; - } - this.#providerType = provider.type; - this.#providers.set(provider.type, provider); - this.logger.log(`Search provider [${provider.type}] registered.`); - this.server.enableFeature(ServerFeature.Indexer); - } - - unregister(provider: SearchProvider) { - if (!this.#providers.has(provider.type)) { - return; - } - this.#providers.delete(provider.type); - this.logger.log(`Search provider [${provider.type}] unregistered.`); - if (this.#providers.size === 0) { - this.server.disableFeature(ServerFeature.Indexer); - } - } -} diff --git a/packages/backend/server/src/plugins/indexer/index.ts b/packages/backend/server/src/plugins/indexer/index.ts index 44d655e97a..9b9641bb99 100644 --- a/packages/backend/server/src/plugins/indexer/index.ts +++ b/packages/backend/server/src/plugins/indexer/index.ts @@ -5,30 +5,41 @@ import { Module } from '@nestjs/common'; import { ServerConfigModule } from '../../core/config'; import { DocStorageModule } from '../../core/doc'; import { PermissionModule } from '../../core/permission'; -import { QuotaServiceModule } from '../../core/quota'; import { IndexerEvent } from './event'; -import { SearchProviderFactory } from './factory'; import { IndexerJob } from './job'; -import { SearchProviders } from './providers'; import { IndexerResolver } from './resolver'; +import { IndexerScheduler } from './scheduler'; import { IndexerService } from './service'; +const INDEXER_SHARED_IMPORTS = [ + ServerConfigModule, + DocStorageModule, + PermissionModule, +]; + @Module({ - imports: [ - ServerConfigModule, - DocStorageModule, - PermissionModule, - QuotaServiceModule, - ], - providers: [ - IndexerResolver, - IndexerService, - IndexerJob, - IndexerEvent, - SearchProviderFactory, - ...SearchProviders, - ], - exports: [IndexerService, SearchProviderFactory], + imports: INDEXER_SHARED_IMPORTS, + providers: [IndexerService], + exports: [IndexerService], +}) +export class IndexerServiceModule {} + +@Module({ + imports: [IndexerServiceModule], + providers: [IndexerEvent], +}) +export class IndexerProducerModule {} + +@Module({ + imports: [IndexerServiceModule, DocStorageModule, PermissionModule], + providers: [IndexerJob, IndexerScheduler], +}) +export class IndexerWorkerModule {} + +@Module({ + imports: [IndexerServiceModule, DocStorageModule, PermissionModule], + providers: [IndexerResolver, IndexerEvent], + exports: [IndexerServiceModule], }) export class IndexerModule {} diff --git a/packages/backend/server/src/plugins/indexer/job.ts b/packages/backend/server/src/plugins/indexer/job.ts index 0ce3d914a7..738324f3a3 100644 --- a/packages/backend/server/src/plugins/indexer/job.ts +++ b/packages/backend/server/src/plugins/indexer/job.ts @@ -47,24 +47,17 @@ export class IndexerJob { @OnJob('indexer.indexDoc') async indexDoc({ workspaceId, docId }: Jobs['indexer.indexDoc']) { - if (!this.config.indexer.enabled) { - return; - } - // delete the 'indexer.deleteDoc' job from the queue await this.queue.remove( `deleteDoc/${workspaceId}/${docId}`, 'indexer.deleteDoc' ); await this.service.indexDoc(workspaceId, docId); + await this.enqueueBlobRefProjection(workspaceId, docId); } @OnJob('indexer.deleteDoc') async deleteDoc({ workspaceId, docId }: Jobs['indexer.deleteDoc']) { - if (!this.config.indexer.enabled) { - return; - } - // delete the 'indexer.updateDoc' job from the queue await this.queue.remove( `indexDoc/${workspaceId}/${docId}`, @@ -79,24 +72,21 @@ export class IndexerJob { docId, cleanupVersion, }: Jobs['indexer.reconcileDocumentCleanup']) { - if (this.config.indexer.enabled) { - const root = await this.doc.getDoc(workspaceId, workspaceId); - if (!root) { - throw new Error(`workspace root ${workspaceId} not found`); - } - const live = readAllDocIdsFromWorkspaceSnapshot(root.bin, true).includes( - docId - ); - if (live) { - if (!(await this.doc.getDoc(workspaceId, docId))) { - throw new Error( - `restored document ${workspaceId}/${docId} not found` - ); - } - await this.service.indexDoc(workspaceId, docId); - } else { - await this.service.deleteDoc(workspaceId, docId); + const root = await this.doc.getDoc(workspaceId, workspaceId); + if (!root) { + throw new Error(`workspace root ${workspaceId} not found`); + } + const live = readAllDocIdsFromWorkspaceSnapshot(root.bin, true).includes( + docId + ); + if (live) { + if (!(await this.doc.getDoc(workspaceId, docId))) { + throw new Error(`restored document ${workspaceId}/${docId} not found`); } + await this.service.indexDoc(workspaceId, docId); + await this.enqueueBlobRefProjection(workspaceId, docId); + } else { + await this.service.deleteDoc(workspaceId, docId); } await this.queue.add('backendRuntime.ackDocumentCleanupEffect', { workspaceId, @@ -108,10 +98,6 @@ export class IndexerJob { @OnJob('indexer.indexWorkspace') async indexWorkspace({ workspaceId }: Jobs['indexer.indexWorkspace']) { - if (!this.config.indexer.enabled) { - return; - } - await this.queue.remove(workspaceId, 'indexer.deleteWorkspace'); const workspace = await this.models.workspace.get(workspaceId); if (!workspace) { @@ -119,71 +105,17 @@ export class IndexerJob { return; } - const root = await this.doc.getDoc(workspaceId, workspaceId); - if (!root) { - this.logger.warn(`workspace snapshot ${workspaceId} not found`); - return; - } - - const docIdsInWorkspace = readAllDocIdsFromWorkspaceSnapshot(root.bin); - const docIdsInIndexer = await this.service.listDocIds(workspaceId); - - const docIdsInWorkspaceSet = new Set(docIdsInWorkspace); - const docIdsInIndexerSet = new Set(docIdsInIndexer); - // diff the docIdsInWorkspace and docIdsInIndexer, if the workspace is not indexed, all the docIdsInWorkspace should be indexed - const missingDocIds = workspace.indexed - ? docIdsInWorkspace.filter(docId => !docIdsInIndexerSet.has(docId)) - : docIdsInWorkspace; - const deletedDocIds = docIdsInIndexer.filter( - docId => !docIdsInWorkspaceSet.has(docId) - ); - for (const docId of deletedDocIds) { - await this.queue.add( - 'indexer.deleteDoc', - { - workspaceId, - docId, - }, - { - jobId: `deleteDoc/${workspaceId}/${docId}`, - // the deleteDoc job should be higher priority than the indexDoc job - priority: 0, - } - ); - } - for (const docId of missingDocIds) { - await this.queue.add( - 'indexer.indexDoc', - { - workspaceId, - docId, - }, - { - jobId: `indexDoc/${workspaceId}/${docId}`, - priority: 100, - } - ); - } + await this.service.reconcileWorkspace(workspaceId); if (!workspace.indexed) { await this.models.workspace.update(workspaceId, { indexed: true, }); } - if (!missingDocIds.length && !deletedDocIds.length) { - this.logger.verbose(`workspace ${workspaceId} is already indexed`); - return; - } - this.logger.log( - `indexed workspace ${workspaceId} with ${missingDocIds.length} missing docs and ${deletedDocIds.length} deleted docs` - ); + this.logger.log(`reconciled workspace ${workspaceId}`); } @OnJob('indexer.deleteWorkspace') async deleteWorkspace({ workspaceId }: Jobs['indexer.deleteWorkspace']) { - if (!this.config.indexer.enabled) { - return; - } - await this.queue.remove( `indexWorkspace/${workspaceId}`, 'indexer.indexWorkspace' @@ -193,10 +125,6 @@ export class IndexerJob { @OnJob('indexer.autoIndexWorkspaces') async autoIndexWorkspaces(payload: Jobs['indexer.autoIndexWorkspaces']) { - if (!this.config.indexer.enabled) { - return; - } - const startSid = payload.lastIndexedWorkspaceSid ?? 0; const workspaces = await this.models.workspace.list( { sid: { gt: startSid } }, @@ -210,9 +138,6 @@ export class IndexerJob { } let addedCount = 0; for (const workspace of workspaces) { - if (workspace.indexed) { - continue; - } const snapshotMeta = await this.models.doc.getSnapshot( workspace.id, workspace.id, @@ -246,4 +171,22 @@ export class IndexerJob { payload.lastIndexedWorkspaceSid = nextSid; return JOB_SIGNAL.Repeat; } + + private async enqueueBlobRefProjection(workspaceId: string, docId: string) { + const snapshot = await this.models.doc.getSnapshot(workspaceId, docId, { + select: { updatedAt: true }, + }); + if (!snapshot) { + return; + } + const sourceRevision = snapshot.updatedAt.getTime(); + await this.queue.add( + 'backendRuntime.projectWorkspaceDocBlobRefs', + { workspaceId, docId, sourceRevision }, + { + jobId: `doc:blob-ref-projection:${workspaceId}:${docId}:${sourceRevision}`, + priority: 100, + } + ); + } } diff --git a/packages/backend/server/src/plugins/indexer/providers/def.ts b/packages/backend/server/src/plugins/indexer/providers/def.ts deleted file mode 100644 index 50dfdc7c46..0000000000 --- a/packages/backend/server/src/plugins/indexer/providers/def.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { Inject, Injectable, Logger } from '@nestjs/common'; - -import { Config, OnEvent } from '../../../base'; -import { SearchProviderType } from '../config'; -import { SearchProviderFactory } from '../factory'; -import { SearchTable } from '../tables'; - -export interface SearchNode { - _id: string; - _score: number; - _source: Record; - fields: Record; - highlights?: Record; -} - -export interface SearchResult { - took: number; - timedOut: boolean; - total: number; - nodes: SearchNode[]; - nextCursor?: string; -} - -export interface AggregateBucket { - key: string; - count: number; - hits: { - nodes: SearchNode[]; - }; -} - -export interface AggregateResult { - took: number; - timedOut: boolean; - total: number; - buckets: AggregateBucket[]; - nextCursor?: string; -} - -export interface BaseQueryDSL { - _source: string[]; - sort: unknown[]; - query: Record; - size?: number; - from?: number; - cursor?: string; -} - -export interface HighlightDSL { - pre_tags: string[]; - post_tags: string[]; -} - -export interface SearchQueryDSL extends BaseQueryDSL { - fields: string[]; - highlight?: { - fields: Record; - }; -} - -export interface TopHitsDSL extends Omit< - SearchQueryDSL, - 'query' | 'sort' | 'from' | 'cursor' -> {} - -export interface AggregateQueryDSL extends BaseQueryDSL { - aggs: { - result: { - terms: { - field: string; - size?: number; - order: { - max_score: 'desc'; - }; - }; - aggs: { - max_score: { - max: { - script: { - source: '_score'; - }; - }; - }; - result: { - top_hits: TopHitsDSL; - }; - }; - }; - }; -} - -export interface OperationOptions { - refresh?: boolean; -} - -@Injectable() -export abstract class SearchProvider { - abstract type: SearchProviderType; - /** - * Create a new search index table. - */ - abstract createTable(table: SearchTable, mapping: string): Promise; - /** - * Search documents from the search index table. - */ - abstract search( - table: SearchTable, - dsl: SearchQueryDSL - ): Promise; - /** - * Aggregate documents from the search index table. - */ - abstract aggregate( - table: SearchTable, - dsl: AggregateQueryDSL - ): Promise; - /** - * Write documents to the search index table. - * If the document already exists, it will be replaced. - * If the document does not exist, it will be created. - */ - abstract write( - table: SearchTable, - documents: Record[], - options?: OperationOptions - ): Promise; - /** - * Delete documents from the search index table. - */ - abstract deleteByQuery( - table: SearchTable, - query: Record, - options?: OperationOptions - ): Promise; - - protected readonly logger = new Logger(this.constructor.name); - - @Inject() private readonly factory!: SearchProviderFactory; - @Inject() private readonly AFFiNEConfig!: Config; - - protected get config() { - return this.AFFiNEConfig.indexer; - } - - protected get configured() { - return this.config.enabled && this.config.provider.type === this.type; - } - - @OnEvent('config.init') - onConfigInit() { - this.setup(); - } - - @OnEvent('config.changed') - onConfigUpdated(event: Events['config.changed']) { - if ('indexer' in event.updates) { - this.setup(); - } - } - - protected setup() { - if (this.configured) { - this.factory.register(this); - } else { - this.factory.unregister(this); - } - } -} diff --git a/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts b/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts deleted file mode 100644 index 18970b7d22..0000000000 --- a/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts +++ /dev/null @@ -1,365 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { - InternalServerError, - InvalidSearchProviderRequest, - safeFetch, -} from '../../../base'; -import { SearchProviderType } from '../config'; -import { DateFieldNames, SearchTable, SearchTableUniqueId } from '../tables'; -import { - AggregateQueryDSL, - AggregateResult, - OperationOptions, - SearchProvider, - SearchQueryDSL, - SearchResult, -} from './def'; - -interface ESSearchResponse { - took: number; - timed_out: boolean; - hits: { - total: { - value: number; - }; - hits: { - _index: string; - _id: string; - _score: number; - _source: Record; - fields: Record; - highlight?: Record; - sort: unknown[]; - }[]; - }; -} - -interface ESAggregateResponse extends ESSearchResponse { - aggregations: { - result: { - buckets: { - key: string; - doc_count: number; - result: { - hits: { - total: { - value: number; - }; - max_score: number; - hits: { - _index: string; - _id: string; - _score: number; - _source: Record; - fields: Record; - highlight?: Record; - }[]; - }; - }; - }[]; - }; - }; -} - -const INDEXER_FETCH_OPTIONS = { - timeoutMs: 30_000, - maxRedirects: 0, - maxBytes: 50 * 1024 * 1024, - allowedHeaders: ['authorization', 'content-type'], - allowPrivateTargetOrigin: true, -}; - -@Injectable() -export class ElasticsearchProvider extends SearchProvider { - type = SearchProviderType.Elasticsearch; - - /** - * @see https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-create - */ - override async createTable( - table: SearchTable, - mapping: string - ): Promise { - const url = `${this.config.provider.endpoint}/${table}`; - try { - const result = await this.request('PUT', url, mapping); - this.logger.log( - `created table ${table}, result: ${JSON.stringify(result)}` - ); - } catch (err) { - if ( - err instanceof InvalidSearchProviderRequest && - (err.data.type === 'resource_already_exists_exception' || - (err.data.type === 'invalid_index_name_exception' && - err.data.reason.includes('already exists as alias'))) - ) { - this.logger.debug(`table ${table} already exists`); - } else { - throw err; - } - } - } - - override async write( - table: SearchTable, - documents: Record[], - options?: OperationOptions - ): Promise { - const start = Date.now(); - const records: string[] = []; - for (const document of documents) { - // @ts-expect-error ignore document type check - const id = SearchTableUniqueId[table](document); - records.push( - JSON.stringify({ - index: { - _index: table, - _id: id, - }, - }) - ); - records.push(JSON.stringify(document)); - } - const url = new URL(`${this.config.provider.endpoint}/_bulk`); - if (options?.refresh) { - url.searchParams.set('refresh', 'true'); - } - await this.requestBulk(url.toString(), records); - this.logger.debug( - `wrote ${documents.length} documents to ${table} in ${Date.now() - start}ms` - ); - } - - /** - * @see https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete-by-query - */ - override async deleteByQuery( - table: T, - query: Record, - options?: OperationOptions - ): Promise { - const start = Date.now(); - const url = new URL( - `${this.config.provider.endpoint}/${table}/_delete_by_query` - ); - if (options?.refresh) { - url.searchParams.set('refresh', 'true'); - } - const result = await this.request( - 'POST', - url.toString(), - JSON.stringify({ query }), - 'application/json', - // ignore 409 error: version_conflict_engine_exception, version conflict, required seqNo [255898790], primary term [3]. current document has seqNo [256133002] and primary term [3] - [409] - ); - this.logger.debug( - `deleted by query ${table} ${JSON.stringify(query)} in ${Date.now() - start}ms, result: ${JSON.stringify(result).substring(0, 500)}` - ); - } - - override async search( - table: SearchTable, - dsl: SearchQueryDSL - ): Promise { - const body = this.#convertToSearchBody(dsl); - const data = (await this.requestSearch(table, body)) as ESSearchResponse; - return { - took: data.took, - timedOut: data.timed_out, - total: data.hits.total.value, - nextCursor: this.#encodeCursor(data.hits.hits.at(-1)?.sort), - nodes: data.hits.hits.map(hit => ({ - _id: hit._id, - _score: hit._score, - _source: this.formatDateFields(hit._source), - fields: this.formatDateFields(hit.fields), - highlights: hit.highlight, - })), - }; - } - - override async aggregate( - table: SearchTable, - dsl: AggregateQueryDSL - ): Promise { - const body = this.#convertToSearchBody(dsl); - const data = (await this.requestSearch(table, body)) as ESAggregateResponse; - const buckets = data.aggregations.result.buckets; - return { - took: data.took, - timedOut: data.timed_out, - total: data.hits.total.value, - nextCursor: this.#encodeCursor(data.hits.hits.at(-1)?.sort), - buckets: buckets.map(bucket => ({ - key: bucket.key, - count: bucket.doc_count, - hits: { - nodes: bucket.result.hits.hits.map(hit => ({ - _id: hit._id, - _score: hit._score, - _source: this.formatDateFields(hit._source), - fields: this.formatDateFields(hit.fields), - highlights: hit.highlight, - })), - }, - })), - }; - } - - protected formatDateFields>( - fieldsOrSource: T - ): T { - for (const fieldName of DateFieldNames) { - let values = fieldsOrSource[fieldName]; - if (!values) { - continue; - } - if (Array.isArray(values)) { - // { created_at: ['2025-06-20T03:02:43.442Z'] } => { created_at: [new Date('2025-06-20T03:02:43.442Z')] } - values = values.map(this.formatDateValue); - } else { - // { created_at: '2025-06-20T03:02:43.442Z' } => { created_at: new Date('2025-06-20T03:02:43.442Z') } - values = this.formatDateValue(values); - } - // @ts-expect-error ignore type check - fieldsOrSource[fieldName] = values; - } - return fieldsOrSource; - } - - /** - * elasticsearch return date value as string, we need to convert it to Date object - */ - protected formatDateValue(value: unknown) { - if (value && typeof value === 'string') { - return new Date(value); - } - return value; - } - - protected async requestSearch(table: SearchTable, body: Record) { - const url = `${this.config.provider.endpoint}/${table}/_search`; - const jsonBody = JSON.stringify(body); - const start = Date.now(); - try { - return await this.request('POST', url, jsonBody); - } finally { - const duration = Date.now() - start; - // log slow search - if (duration > 1000) { - this.logger.warn( - `Slow search on ${table} in ${duration}ms, DSL: ${jsonBody}` - ); - } else { - this.logger.verbose( - `search ${table} in ${duration}ms, DSL: ${jsonBody}` - ); - } - } - } - - /** - * @see https://www.elastic.co/docs/api/doc/elasticsearch-serverless/operation/operation-bulk-2 - */ - protected async requestBulk(url: string, records: string[]) { - return await this.request( - 'POST', - url.toString(), - records.join('\n') + '\n', - 'application/x-ndjson' - ); - } - - protected async request( - method: 'POST' | 'PUT', - url: string, - body: string, - contentType = 'application/json', - ignoreErrorStatus?: number[] - ) { - const headers = { - 'Content-Type': contentType, - } as Record; - if (this.config.provider.apiKey) { - headers.Authorization = `ApiKey ${this.config.provider.apiKey}`; - } else if (this.config.provider.password) { - headers.Authorization = `Basic ${Buffer.from(`${this.config.provider.username}:${this.config.provider.password}`).toString('base64')}`; - } - const response = await safeFetch( - url, - { method, body, headers }, - INDEXER_FETCH_OPTIONS - ); - const data = await response.json(); - if (ignoreErrorStatus?.includes(response.status)) { - return data; - } - - // handle error, status >= 400 - // { - // "error": { - // "root_cause": [ - // { - // "type": "illegal_argument_exception", - // "reason": "The bulk request must be terminated by a newline [\\n]" - // } - // ], - // "type": "illegal_argument_exception", - // "reason": "The bulk request must be terminated by a newline [\\n]" - // }, - // "status": 400 - // } - if (response.status >= 500) { - this.logger.error( - `request error, url: ${url}, body: ${body}, response status: ${response.status}, response body: ${JSON.stringify(data, null, 2)}` - ); - throw new InternalServerError(); - } - if (response.status >= 400) { - this.logger.warn( - `request failed, url: ${url}, body: ${body}, response status: ${response.status}, response body: ${JSON.stringify(data, null, 2)}` - ); - const errorData = data as { - error?: { type: string; reason: string } | string; - }; - let reason = ''; - let type = ''; - if (typeof errorData.error === 'string') { - reason = errorData.error; - } else if (errorData.error) { - reason = errorData.error.reason; - type = errorData.error.type; - } else { - reason = `unknown error, status ${response.status}, please check the response body`; - } - throw new InvalidSearchProviderRequest({ - reason, - type, - }); - } - return data; - } - - #convertToSearchBody(dsl: SearchQueryDSL | AggregateQueryDSL) { - const data: Record = { - ...dsl, - }; - if (dsl.cursor) { - data.cursor = undefined; - data.search_after = this.#decodeCursor(dsl.cursor); - } - return data; - } - - #decodeCursor(cursor: string) { - return JSON.parse(Buffer.from(cursor, 'base64').toString('utf-8')); - } - - #encodeCursor(cursor?: unknown[]) { - return cursor - ? Buffer.from(JSON.stringify(cursor)).toString('base64') - : undefined; - } -} diff --git a/packages/backend/server/src/plugins/indexer/providers/index.ts b/packages/backend/server/src/plugins/indexer/providers/index.ts deleted file mode 100644 index b5c9d5196c..0000000000 --- a/packages/backend/server/src/plugins/indexer/providers/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { ElasticsearchProvider } from './elasticsearch'; -import { ManticoresearchProvider } from './manticoresearch'; - -export const SearchProviders = [ManticoresearchProvider, ElasticsearchProvider]; - -export * from './def'; -export * from './elasticsearch'; -export * from './manticoresearch'; diff --git a/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts b/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts deleted file mode 100644 index ad999c2ff2..0000000000 --- a/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts +++ /dev/null @@ -1,515 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { omit } from 'lodash-es'; - -import { InternalServerError, safeFetch } from '../../../base'; -import { SearchProviderType } from '../config'; -import { SearchTable } from '../tables'; -import { - AggregateQueryDSL, - AggregateResult, - HighlightDSL, - OperationOptions, - SearchNode, - SearchQueryDSL, - SearchResult, -} from './def'; -import { ElasticsearchProvider } from './elasticsearch'; - -interface MSSearchResponse { - took: number; - timed_out: boolean; - hits: { - total: number; - hits: { - _index: string; - _id: string; - _score: number; - _source: Record; - highlight?: Record; - sort: unknown[]; - }[]; - }; - scroll: string; -} - -const INDEXER_FETCH_OPTIONS = { - timeoutMs: 30_000, - maxRedirects: 0, - maxBytes: 50 * 1024 * 1024, - allowedHeaders: ['authorization', 'content-type'], - allowPrivateTargetOrigin: true, -}; - -const SupportIndexedAttributes = [ - 'flavour', - 'parent_flavour', - 'parent_block_id', -]; - -const SupportExactTermFields = new Set([ - 'workspace_id', - 'doc_id', - 'block_id', - 'flavour', - 'parent_flavour', - 'parent_block_id', - 'created_by_user_id', - 'updated_by_user_id', -]); - -const ConvertEmptyStringToNullValueFields = new Set([ - 'ref_doc_id', - 'ref', - 'blob', - 'additional', - 'parent_block_id', - 'parent_flavour', -]); - -function boolClauses(value: unknown) { - if (value === undefined) return []; - return Array.isArray(value) ? value : [value]; -} - -@Injectable() -export class ManticoresearchProvider extends ElasticsearchProvider { - override type = SearchProviderType.Manticoresearch; - - override async createTable( - table: SearchTable, - mapping: string - ): Promise { - const text = await this.#executeSQL(mapping); - this.logger.log(`created table ${table}, response: ${text}`); - } - - async dropTable(table: SearchTable): Promise { - const text = await this.#executeSQL(`DROP TABLE IF EXISTS ${table}`); - this.logger.log(`dropped table ${table}, response: ${text}`); - } - - async recreateTable(table: SearchTable, mapping: string): Promise { - await this.dropTable(table); - await this.createTable(table, mapping); - } - - override async write( - table: SearchTable, - documents: Record[], - options?: OperationOptions - ): Promise { - if (table === SearchTable.block) { - documents = documents.map(document => ({ - ...document, - // convert content `string[]` to `string` - // because manticoresearch full text search does not support `string[]` - content: Array.isArray(document.content) - ? document.content.join(' ') - : document.content, - // convert one item array to string in `blob`, `ref`, `ref_doc_id` - blob: this.#formatArrayValue(document.blob), - ref: this.#formatArrayValue(document.ref), - ref_doc_id: this.#formatArrayValue(document.ref_doc_id), - // add extra indexed attributes - ...SupportIndexedAttributes.reduce( - (acc, attribute) => { - acc[`${attribute}_indexed`] = document[attribute]; - return acc; - }, - {} as Record - ), - })); - } - - await super.write(table, documents, options); - } - - /** - * @see https://manual.manticoresearch.com/Data_creation_and_modification/Deleting_documents?static=true&client=JSON#Deleting-documents - */ - override async deleteByQuery( - table: T, - query: Record, - options?: OperationOptions - ): Promise { - const start = Date.now(); - const url = new URL(`${this.config.provider.endpoint}/delete`); - if (options?.refresh) { - url.searchParams.set('refresh', 'true'); - } - const body = JSON.stringify({ - table, - // term not work on delete query, so we need to use equals instead - query: this.parseESQuery(query, { termMappingField: 'equals' }), - }); - const result = await this.request('POST', url.toString(), body); - this.logger.debug( - `deleted by query ${body} in ${Date.now() - start}ms, result: ${JSON.stringify(result)}` - ); - } - - override async search( - table: SearchTable, - dsl: SearchQueryDSL - ): Promise { - const body = this.#convertToSearchBody(dsl); - const data = (await this.requestSearch(table, body)) as MSSearchResponse; - return { - took: data.took, - timedOut: data.timed_out, - total: data.hits.total, - nextCursor: data.scroll, - nodes: data.hits.hits.map(hit => ({ - _id: hit._id, - _score: hit._score, - _source: this.formatDateFields( - this.#formatSource(dsl._source, hit._source) - ), - fields: this.formatDateFields( - this.#formatFieldsFromSource(dsl.fields, hit._source) - ), - highlights: this.#formatHighlights( - dsl.highlight?.fields, - hit.highlight - ), - })), - }; - } - - override async aggregate( - table: SearchTable, - dsl: AggregateQueryDSL - ): Promise { - const aggs = dsl.aggs; - const topHits = aggs.result.aggs.result.top_hits; - const groupByField = aggs.result.terms.field; - const searchDSL = { - ...omit(dsl, 'aggs'), - // add groupByField to fields if not already in - fields: topHits.fields.includes(groupByField) - ? topHits.fields - : [...topHits.fields, groupByField], - highlight: topHits.highlight, - }; - const body = this.#convertToSearchBody(searchDSL); - const data = (await this.requestSearch(table, body)) as MSSearchResponse; - - // calculate the aggregate buckets - const bucketsMap = new Map(); - for (const hit of data.hits.hits) { - const key = hit._source[groupByField] as string; - const node = { - _id: hit._id, - _score: hit._score, - _source: this.formatDateFields( - this.#formatSource(topHits._source, hit._source) - ), - fields: this.formatDateFields( - this.#formatFieldsFromSource(topHits.fields, hit._source) - ), - highlights: this.#formatHighlights( - topHits.highlight?.fields, - hit.highlight - ), - }; - if (bucketsMap.has(key)) { - bucketsMap.get(key)?.push(node); - } else { - bucketsMap.set(key, [node]); - } - } - return { - took: data.took, - timedOut: data.timed_out, - total: data.hits.total, - nextCursor: data.scroll, - buckets: Array.from(bucketsMap.entries()).map(([key, nodes]) => ({ - key, - count: nodes.length, - hits: { - nodes: topHits.size ? nodes.slice(0, topHits.size) : nodes, - }, - })), - }; - } - - #convertToSearchBody(dsl: SearchQueryDSL) { - const data: Record = { - ...dsl, - query: this.parseESQuery(dsl.query), - fields: undefined, - _source: [...new Set([...dsl._source, ...dsl.fields])], - }; - - // https://manual.manticoresearch.com/Searching/Pagination#Pagination-of-search-results - // use scroll - if (dsl.cursor) { - data.cursor = undefined; - data.options = { - scroll: dsl.cursor, - }; - } else { - data.options = { - scroll: true, - }; - } - - // if highlight provided, add all fields to highlight - // "highlight":{"fields":{"title":{"pre_tags":[""],"post_tags":[""]}} - // to - // "highlight":{"pre_tags":[""],"post_tags":[""]} - if (dsl.highlight) { - const firstOptions = Object.values(dsl.highlight.fields)[0]; - data.highlight = firstOptions; - } - return data; - } - - /** - * manticoresearch return date value as timestamp, we need to convert it to Date object - */ - protected override formatDateValue(value: unknown) { - if (value && typeof value === 'number') { - // 1750389254 => new Date(1750389254 * 1000) - return new Date(value * 1000); - } - if (value && typeof value === 'string') { - const timestamp = Date.parse(value); - if (!Number.isNaN(timestamp)) { - return new Date(timestamp); - } - } - return value; - } - - private parseESQuery( - query: Record, - options?: { - termMappingField?: string; - parentNodes?: Record[]; - } - ) { - let node: Record = {}; - if (query.bool) { - node.bool = {}; - for (const occur in query.bool) { - const conditions = query.bool[occur]; - if (Array.isArray(conditions)) { - const parsedConditions: Record[] = []; - const existing = node.bool[occur]; - node.bool[occur] = parsedConditions; - // { must: [ { term: [Object] }, { bool: [Object] } ] } - // { - // must: [ { term: [Object] }, { term: [Object] }, { bool: [Object] } ] - // } - for (const item of conditions) { - this.parseESQuery(item, { - ...options, - parentNodes: parsedConditions, - }); - } - if (existing !== undefined) { - node.bool[occur] = [...boolClauses(existing), ...parsedConditions]; - } - if (occur === 'must') { - for (let i = node.bool.must.length - 1; i >= 0; i--) { - const child = node.bool.must[i]; - const childBool = child.bool; - if ( - childBool && - Object.keys(childBool).length === 1 && - childBool.must_not !== undefined - ) { - node.bool.must.splice(i, 1); - node.bool.must_not = [ - ...boolClauses(node.bool.must_not), - ...boolClauses(childBool.must_not), - ]; - } - } - } - } else { - // { - // must_not: { term: { doc_id: 'docId' } } - // } - const parsed = this.parseESQuery(conditions, { - termMappingField: options?.termMappingField, - }); - if (node.bool[occur] !== undefined) { - node.bool[occur] = [...boolClauses(node.bool[occur]), parsed]; - } else { - node.bool[occur] = parsed; - } - } - } - } else if (query.term) { - // { - // term: { - // workspace_id: { - // value: 'workspaceId1' - // } - // } - // } - // to - // { - // term: { - // workspace_id: 'workspaceId1' - // } - // } - let field = Object.keys(query.term)[0]; - let termField = - options?.termMappingField ?? - (SupportExactTermFields.has(field) ? 'equals' : 'term'); - let value = query.term[field]; - if (typeof value === 'object' && 'value' in value) { - if ('boost' in value) { - // { - // term: { - // flavour: { - // value: 'affine:page', - // boost: 1.5, - // }, - // }, - // } - // to - // { - // match: { - // flavour_indexed: { - // query: 'affine:page', - // boost: 1.5, - // }, - // }, - // } - if (SupportIndexedAttributes.includes(field)) { - field = `${field}_indexed`; - } - termField = 'match'; - value = { - query: value.value, - boost: value.boost, - }; - } else { - value = value.value; - } - } - node = { - [termField]: { - [field]: value, - }, - }; - } else if (query.exists) { - let field = query.exists.field; - if (SupportIndexedAttributes.includes(field)) { - // override the field to indexed field - field = `${field}_indexed`; - } - node = { - ...query, - exists: { - ...query.exists, - field, - }, - }; - } else { - node = { - ...query, - }; - } - if (options?.parentNodes) { - options.parentNodes.push(node); - } - // this.logger.verbose(`parsed es query ${JSON.stringify(query, null, 2)} to ${JSON.stringify(node, null, 2)}`); - return node; - } - - /** - * Format fields from source to match the expected format for ManticoreSearch - */ - #formatFieldsFromSource(fields: string[], source: Record) { - return fields.reduce( - (acc, field) => { - let value = source[field]; - if (ConvertEmptyStringToNullValueFields.has(field) && value === '') { - value = null; - } - if (value !== null && value !== undefined) { - // special handle `ref_doc_id`, `ref`, `blob` as string[] - if ( - (field === 'ref_doc_id' || field === 'ref' || field === 'blob') && - typeof value === 'string' && - value.startsWith('["') - ) { - //'["b5ed7e73-b792-4a80-8727-c009c5b50116","573ccd98-72be-4a43-9e75-fdc67231bcb4"]' - // to - // ['b5ed7e73-b792-4a80-8727-c009c5b50116', '573ccd98-72be-4a43-9e75-fdc67231bcb4'] - // or - // '["{\"foo\": \"bar\"}","{\"foo\": \"baz\"}"]' - // to - // [{foo: 'bar'}, {foo: 'baz'}] - value = JSON.parse(value as string); - } - acc[field] = Array.isArray(value) ? value : [value]; - } - return acc; - }, - {} as Record - ); - } - - #formatHighlights( - highlightFields?: Record, - highlights?: Record - ) { - if (!highlightFields || !highlights) { - return undefined; - } - return this.#formatFieldsFromSource( - Object.keys(highlightFields), - highlights - ); - } - - #formatSource(fields: string[], source: Record) { - return fields.reduce( - (acc, field) => { - acc[field] = source[field]; - return acc; - }, - {} as Record - ); - } - - #formatArrayValue(value: unknown | unknown[]) { - if (Array.isArray(value)) { - if (value.length === 1) { - return value[0]; - } - return JSON.stringify(value); - } - return value; - } - - async #executeSQL(sql: string) { - const url = `${this.config.provider.endpoint}/cli`; - const headers: Record = { - 'Content-Type': 'text/plain', - }; - if (this.config.provider.apiKey) { - headers.Authorization = `ApiKey ${this.config.provider.apiKey}`; - } else if (this.config.provider.password) { - headers.Authorization = `Basic ${Buffer.from(`${this.config.provider.username}:${this.config.provider.password}`).toString('base64')}`; - } - - const response = await safeFetch( - url, - { method: 'POST', body: sql, headers }, - INDEXER_FETCH_OPTIONS - ); - const text = (await response.text()).trim(); - if (!response.ok) { - this.logger.error(`failed to execute SQL "${sql}", response: ${text}`); - throw new InternalServerError(); - } - return text; - } -} diff --git a/packages/backend/server/src/plugins/indexer/resolver.ts b/packages/backend/server/src/plugins/indexer/resolver.ts index baf888a7a8..268e75ff46 100644 --- a/packages/backend/server/src/plugins/indexer/resolver.ts +++ b/packages/backend/server/src/plugins/indexer/resolver.ts @@ -1,9 +1,7 @@ import { Args, Parent, ResolveField, Resolver } from '@nestjs/graphql'; -import { Prisma, PrismaClient } from '@prisma/client'; import { CurrentUser } from '../../core/auth'; -import { PermissionAccess, PermissionService } from '../../core/permission'; -import { QuotaStateService } from '../../core/quota/state'; +import { PermissionAccess } from '../../core/permission'; import { UserType } from '../../core/user'; import { WorkspaceType } from '../../core/workspaces'; import { IndexerService } from './service'; @@ -13,9 +11,6 @@ import { SearchDocObjectType, SearchDocsInput, SearchInput, - SearchQuery, - SearchQueryOccur, - SearchQueryType, SearchResultObjectType, } from './types'; @@ -23,10 +18,7 @@ import { export class IndexerResolver { constructor( private readonly indexer: IndexerService, - private readonly ac: PermissionAccess, - private readonly db: PrismaClient, - private readonly permission: PermissionService, - private readonly quotaState: QuotaStateService + private readonly ac: PermissionAccess ) {} @ResolveField(() => SearchResultObjectType, { @@ -39,18 +31,7 @@ export class IndexerResolver { ): Promise { // currentUser can read the workspace await this.ac.user(me.id).workspace(workspace.id).assert('Workspace.Read'); - this.#addWorkspaceFilter(workspace, input); - if (!(await this.#addReadableDocFilter(workspace, me, input))) { - return { - nodes: [], - pagination: { - count: 0, - hasMore: false, - }, - }; - } - - const result = await this.indexer.search(input); + const result = await this.indexer.search(me.id, workspace.id, input); return { nodes: result.nodes, pagination: { @@ -71,24 +52,12 @@ export class IndexerResolver { ): Promise { // currentUser can read the workspace await this.ac.user(me.id).workspace(workspace.id).assert('Workspace.Read'); - this.#addWorkspaceFilter(workspace, input); - if (!(await this.#addReadableDocFilter(workspace, me, input))) { - return { - buckets: [], - pagination: { - count: 0, - hasMore: false, - }, - }; - } - - const result = await this.indexer.aggregate(input); + const result = await this.indexer.aggregate(me.id, workspace.id, input); return { buckets: result.buckets, pagination: { count: result.total, - hasMore: result.buckets.length > 0, - nextCursor: result.nextCursor, + hasMore: result.hasMore, }, }; } @@ -101,107 +70,13 @@ export class IndexerResolver { @Parent() workspace: WorkspaceType, @Args('input') input: SearchDocsInput ): Promise { - const readableDocIds = await this.#readableDocIdsForSearch(workspace, me); const docs = await this.indexer.searchDocsByKeyword( + me.id, workspace.id, input.keyword, - { - limit: input.limit, - docIds: readableDocIds ?? undefined, - } + { limit: input.limit } ); return docs; } - - #addWorkspaceFilter( - workspace: WorkspaceType, - input: SearchInput | AggregateInput - ) { - // filter by workspace id - input.query = { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspace.id, - }, - input.query, - ], - }; - } - - async #addReadableDocFilter( - workspace: WorkspaceType, - user: UserType, - input: SearchInput | AggregateInput - ) { - const docIds = await this.#readableDocIdsForSearch(workspace, user); - if (docIds === null) { - return true; - } - - if (docIds.length === 0) { - return false; - } - - input.query = { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [input.query, this.#docIdFilterQuery(docIds)], - }; - return true; - } - - async #readableDocIdsForSearch(workspace: WorkspaceType, user: UserType) { - const state = await this.quotaState.reconcileWorkspaceQuotaState( - workspace.id - ); - const isTeamWorkspace = - state.plan === 'team' || state.plan === 'selfhost_team'; - if (!isTeamWorkspace) { - return null; - } - - return await this.#listReadableDocIds(workspace, user); - } - - #docIdFilterQuery(docIds: string[]): SearchQuery { - return { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.should, - queries: docIds.map(docId => ({ - type: SearchQueryType.match, - field: 'docId', - match: docId, - })), - }; - } - - async #listReadableDocIds(workspace: WorkspaceType, user: UserType) { - const input = { - userId: user.id, - workspaceId: workspace.id, - action: 'Doc.Read', - docIdColumn: Prisma.raw('candidate_docs.doc_id'), - } as const; - const predicate = this.permission.docReadableSqlPredicate(input); - const rows = await this.db.$queryRaw<{ docId: string }[]>` - WITH candidate_docs AS ( - SELECT "workspace_pages"."page_id" AS doc_id - FROM "workspace_pages" - WHERE "workspace_pages"."workspace_id" = ${workspace.id} - UNION - SELECT "snapshots"."guid" AS doc_id - FROM "snapshots" - WHERE "snapshots"."workspace_id" = ${workspace.id} - ) - SELECT candidate_docs.doc_id AS "docId" - FROM candidate_docs - WHERE ${predicate} - `; - return rows.map(row => row.docId); - } } diff --git a/packages/backend/server/src/plugins/indexer/result.ts b/packages/backend/server/src/plugins/indexer/result.ts new file mode 100644 index 0000000000..d05d6f0305 --- /dev/null +++ b/packages/backend/server/src/plugins/indexer/result.ts @@ -0,0 +1,199 @@ +import { camelCase, mapKeys } from 'lodash-es'; + +import { + AggregateInput, + SearchDoc, + SearchQueryOccur, + SearchQueryType, + SearchTable, +} from './types'; + +export interface SearchNode { + id: string; + score: number; + fields: Record; + highlights?: Record; + _source?: Record; +} + +export interface AggregateResult { + total: number; + hasMore: boolean; + buckets: Array<{ + key: string; + count: number; + hits: { nodes: SearchNode[] }; + }>; +} + +export interface SearchNodeWithMeta extends SearchNode { + _source: { + workspaceId: string; + docId: string; + }; +} + +export function formatSearchNodes(nodes: SearchNode[]) { + return nodes.map(node => ({ + ...node, + fields: mapKeys( + Object.fromEntries( + Object.entries(node.fields).map(([key, values]) => [ + key, + key === 'created_at' || key === 'updated_at' + ? values.map(value => new Date(value as string | number)) + : values, + ]) + ), + (_, key) => camelCase(key) + ), + highlights: node.highlights + ? mapKeys(node.highlights, (_, key) => camelCase(key)) + : undefined, + _source: { + workspaceId: (node._source?.workspace_id ?? + node.fields.workspace_id?.[0]) as string, + docId: (node._source?.doc_id ?? node.fields.doc_id?.[0]) as string, + }, + })) as SearchNodeWithMeta[]; +} + +export function buildSearchDocsInput( + workspaceId: string, + keyword: string, + options?: { limit?: number; docIds?: string[] } +): AggregateInput { + return { + table: SearchTable.block, + field: 'docId', + query: { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + ...(options?.docIds + ? [ + { + type: SearchQueryType.boolean as const, + occur: SearchQueryOccur.should, + queries: options.docIds.map(docId => ({ + type: SearchQueryType.match as const, + field: 'docId', + match: docId, + })), + }, + ] + : []), + { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'content', + match: keyword, + }, + { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.should, + queries: [ + { + type: SearchQueryType.match, + field: 'content', + match: keyword, + }, + { + type: SearchQueryType.boost, + boost: 1.5, + query: { + type: SearchQueryType.match, + field: 'flavour', + match: 'affine:page', + }, + }, + ], + }, + ], + }, + ], + }, + options: { + hits: { + fields: [ + 'blockId', + 'unitId', + 'projectionVersion', + 'sourceHash', + 'visibility', + 'elementId', + 'frameId', + 'sourceBlockId', + 'flavour', + 'content', + 'createdAt', + 'updatedAt', + 'createdByUserId', + 'updatedByUserId', + ], + highlights: [{ field: 'content', before: '', end: '' }], + pagination: { limit: 2 }, + }, + pagination: { limit: options?.limit ?? 20 }, + }, + }; +} + +export function collectSearchDocs( + result: AggregateResult, + workspaceId: string +) { + const docs: SearchDoc[] = []; + const missingTitles: { workspaceId: string; docId: string }[] = []; + const userIds: { userId: string }[] = []; + + for (const bucket of result.buckets) { + const node = bucket.hits.nodes[0]; + const docId = bucket.key; + const blockId = node.fields.blockId[0] as string; + const unitId = node.fields.unitId[0] as string; + const projectionVersion = node.fields.projectionVersion[0] as number; + const sourceHash = node.fields.sourceHash[0] as string; + const visibility = node.fields.visibility[0] as string; + const elementId = node.fields.elementId?.[0] as string | undefined; + const frameId = node.fields.frameId?.[0] as string | undefined; + const sourceBlockId = node.fields.sourceBlockId?.[0] as string | undefined; + const flavour = node.fields.flavour[0] as string; + const content = node.fields.content[0] as string; + const createdAt = node.fields.createdAt[0] as Date; + const updatedAt = node.fields.updatedAt[0] as Date; + const createdByUserId = node.fields.createdByUserId[0] as string; + const updatedByUserId = node.fields.updatedByUserId[0] as string; + const highlight = node.highlights?.content?.[0] as string; + const title = flavour === 'affine:page' ? content : ''; + if (!title) { + missingTitles.push({ workspaceId, docId }); + } + docs.push({ + docId, + blockId: sourceBlockId || blockId, + ...(unitId ? { unitId } : {}), + ...(projectionVersion ? { projectionVersion } : {}), + ...(sourceHash ? { sourceHash } : {}), + ...(visibility ? { visibility } : {}), + ...(elementId ? { elementId } : {}), + ...(frameId ? { frameId } : {}), + title, + highlight, + createdAt, + updatedAt, + createdByUserId, + updatedByUserId, + }); + userIds.push({ userId: createdByUserId }, { userId: updatedByUserId }); + } + return { docs, missingTitles, userIds }; +} diff --git a/packages/backend/server/src/plugins/indexer/scheduler.ts b/packages/backend/server/src/plugins/indexer/scheduler.ts new file mode 100644 index 0000000000..c7b97d70b4 --- /dev/null +++ b/packages/backend/server/src/plugins/indexer/scheduler.ts @@ -0,0 +1,22 @@ +import { Injectable } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; + +import { JobQueue } from '../../base'; + +@Injectable() +export class IndexerScheduler { + constructor(private readonly queue: JobQueue) {} + + @Cron(CronExpression.EVERY_30_SECONDS) + async autoIndexWorkspaces() { + await this.queue.add( + 'indexer.autoIndexWorkspaces', + {}, + { + // make sure only one job is running at a time + delay: 30 * 1000, + jobId: 'autoIndexWorkspaces', + } + ); + } +} diff --git a/packages/backend/server/src/plugins/indexer/service.ts b/packages/backend/server/src/plugins/indexer/service.ts index 725bf2749a..c6501d5260 100644 --- a/packages/backend/server/src/plugins/indexer/service.ts +++ b/packages/backend/server/src/plugins/indexer/service.ts @@ -1,1084 +1,156 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { camelCase, chunk, mapKeys, snakeCase } from 'lodash-es'; +import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; import { + InternalServerError, InvalidIndexerInput, - JobQueue, + OnEvent, SearchProviderNotFound, + SpaceAccessDenied, + WorkspacePermissionNotFound, } from '../../base'; -import { projectDocSearch } from '../../core/utils/blocksuite'; +import { BackendRuntimeProvider } from '../../core/backend-runtime'; +import { ServerFeature, ServerService } from '../../core/config'; import { Models } from '../../models'; -import { SearchProviderType } from './config'; -import { SearchProviderFactory } from './factory'; import { - AggregateQueryDSL, - BaseQueryDSL, - HighlightDSL, - ManticoresearchProvider, - OperationOptions, - SearchNode, - SearchProvider, - SearchQueryDSL, - TopHitsDSL, -} from './providers'; -import { - Block, - blockMapping, - BlockSchema, - blockSQL, - Doc, - docMapping, - DocSchema, - docSQL, - SearchTable, -} from './tables'; -import { - AggregateInput, - SearchDoc, - SearchHighlight, - SearchInput, - SearchQuery, - SearchQueryOccur, - SearchQueryType, -} from './types'; + type AggregateResult, + buildSearchDocsInput, + collectSearchDocs, + formatSearchNodes, + type SearchNode, +} from './result'; +import type { AggregateInput, SearchDoc, SearchInput } from './types'; -// always return these fields to check permission -const DefaultSourceFields = ['workspace_id', 'doc_id'] as const; - -export const SearchTableSorts = { - [SearchProviderType.Elasticsearch]: { - [SearchTable.block]: [ - '_score', - { updated_at: 'desc' }, - 'doc_id', - 'block_id', - ], - [SearchTable.doc]: ['_score', { updated_at: 'desc' }, 'doc_id'], - }, - // add id to sort and make sure scroll can work on manticoresearch - [SearchProviderType.Manticoresearch]: { - [SearchTable.block]: ['_score', { updated_at: 'desc' }, 'id'], - [SearchTable.doc]: ['_score', { updated_at: 'desc' }, 'id'], - }, -} as const; - -const SearchTableMappingStrings = { - [SearchProviderType.Elasticsearch]: { - [SearchTable.block]: JSON.stringify(blockMapping), - [SearchTable.doc]: JSON.stringify(docMapping), - }, - [SearchProviderType.Manticoresearch]: { - [SearchTable.block]: blockSQL, - [SearchTable.doc]: docSQL, - }, +type SearchResult = { + total: number; + nodes: SearchNode[]; + nextCursor?: string; }; -const SearchTableSchema = { - [SearchTable.block]: BlockSchema, - [SearchTable.doc]: DocSchema, +type SearchOperationOutput = { + ok: boolean; + value?: unknown; + errorCode?: string; }; -const SupportFullTextSearchFields = { - [SearchTable.block]: ['content'], - [SearchTable.doc]: ['title'], -}; - -const AllowAggregateFields = new Set(['docId', 'flavour']); - -type SnakeToCamelCase = - S extends `${infer Head}_${infer Tail}` - ? `${Head}${Capitalize>}` - : S; -type CamelizeKeys = { - [K in keyof T as SnakeToCamelCase]: T[K]; -}; -export type UpsertDoc = CamelizeKeys; -export type UpsertBlock = CamelizeKeys; -export type UpsertTypeByTable = - T extends SearchTable.block ? UpsertBlock : UpsertDoc; - -export interface SearchNodeWithMeta extends SearchNode { - _source: { - workspaceId: string; - docId: string; - }; -} - @Injectable() -export class IndexerService { - private readonly logger = new Logger(IndexerService.name); - +export class IndexerService implements OnApplicationBootstrap { constructor( + private readonly runtime: BackendRuntimeProvider, private readonly models: Models, - private readonly factory: SearchProviderFactory, - private readonly queue: JobQueue + private readonly server: ServerService ) {} - async createTables() { - let searchProvider: SearchProvider | undefined; - try { - searchProvider = this.factory.get(); - } catch (err) { - if (err instanceof SearchProviderNotFound) { - this.logger.debug('No search provider found, skip creating tables'); - return; - } - throw err; - } - const mappings = SearchTableMappingStrings[searchProvider.type]; - for (const table of Object.keys(mappings) as SearchTable[]) { - await searchProvider.createTable(table, mappings[table]); - } + async onApplicationBootstrap() { + await this.syncFeature(); } - async rebuildManticoreIndexes() { - let searchProvider: SearchProvider | undefined; - try { - searchProvider = this.factory.get(); - } catch (err) { - if (err instanceof SearchProviderNotFound) { - this.logger.debug('No search provider found, skip rebuilding tables'); - return; - } - throw err; - } - - if (!(searchProvider instanceof ManticoresearchProvider)) { - this.logger.debug( - `Search provider ${searchProvider.type} does not need manticore rebuild` - ); - return; - } - - const mappings = SearchTableMappingStrings[searchProvider.type]; - for (const table of Object.keys(mappings) as SearchTable[]) { - await searchProvider.recreateTable(table, mappings[table]); - } - - let lastWorkspaceSid = 0; - while (true) { - const workspaces = await this.models.workspace.list( - { sid: { gt: lastWorkspaceSid } }, - { id: true, sid: true }, - 100 - ); - if (!workspaces.length) { - break; - } - - for (const workspace of workspaces) { - await this.models.workspace.update( - workspace.id, - { indexed: false }, - false - ); - await this.queue.add( - 'indexer.indexWorkspace', - { - workspaceId: workspace.id, - }, - { - jobId: `indexWorkspace/${workspace.id}`, - priority: 100, - } - ); - } - - lastWorkspaceSid = workspaces[workspaces.length - 1].sid; - } + @OnEvent('config.changed.broadcast') + async onConfigChanged({ updates }: Events['config.changed.broadcast']) { + if (updates.indexer) await this.syncFeature(); } - async write( - table: T, - documents: UpsertTypeByTable[], - options?: OperationOptions + private async syncFeature() { + const status = (await this.runtime.searchStatus()) as { ready: boolean }; + if (status.ready) this.server.enableFeature(ServerFeature.Indexer); + else this.server.disableFeature(ServerFeature.Indexer); + } + + async search(actorUserId: string, workspaceId: string, input: SearchInput) { + const result = this.unwrap( + await this.runtime.searchAuthorized(actorUserId, workspaceId, input), + workspaceId + ); + return { ...result, nodes: formatSearchNodes(result.nodes) }; + } + + async aggregate( + actorUserId: string, + workspaceId: string, + input: AggregateInput ) { - const searchProvider = this.factory.get(); - const schema = SearchTableSchema[table]; - // slice documents to 1000 documents each time - const documentsChunks = chunk(documents, 1000); - for (const documentsChunk of documentsChunks) { - await searchProvider.write( - table, - documentsChunk.map(d => - schema.parse(mapKeys(d, (_, key) => snakeCase(key))) - ), - options - ); - } - } - - async search(input: SearchInput) { - const searchProvider = this.factory.get(); - const dsl = this.parseInput(input); - const result = await searchProvider.search(input.table, dsl); + const result = this.unwrap( + await this.runtime.aggregateAuthorized(actorUserId, workspaceId, input), + workspaceId + ); return { ...result, - nodes: this.#formatSearchNodes(result.nodes), + buckets: result.buckets.map(bucket => ({ + ...bucket, + hits: { + ...bucket.hits, + nodes: formatSearchNodes(bucket.hits.nodes), + }, + })), }; } - async aggregate(input: AggregateInput) { - const searchProvider = this.factory.get(); - const dsl = this.parseInput(input); - const result = await searchProvider.aggregate(input.table, dsl); - for (const bucket of result.buckets) { - bucket.hits = { - ...bucket.hits, - nodes: this.#formatSearchNodes(bucket.hits.nodes), - }; - } - return result; + async indexDoc(workspaceId: string, docId: string) { + await this.runtime.indexSearchDocument(workspaceId, docId); } - async listDocIds(workspaceId: string) { - const docIds: string[] = []; - let cursor: string | undefined; - do { - const result = await this.search({ - table: SearchTable.doc, - query: { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - options: { - fields: ['docId'], - pagination: { - limit: 10000, - cursor, - }, - }, - }); - if (result.nextCursor && result.nextCursor === cursor) { - // NOTE(@fengmk2): avoid infinite loop bug in manticoresearch - break; - } - docIds.push(...result.nodes.map(node => node.fields.docId[0] as string)); - cursor = result.nextCursor; - this.logger.debug( - `get ${result.nodes.length} new / ${docIds.length} total doc ids for workspace ${workspaceId}, nextCursor: ${cursor}` - ); - } while (cursor); - return docIds; + async deleteDoc(workspaceId: string, docId: string) { + await this.runtime.deleteSearchDocument(workspaceId, docId); } - async indexDoc( - workspaceId: string, - docId: string, - options?: OperationOptions - ) { - const docSnapshot = await this.models.doc.getSnapshot(workspaceId, docId); - if (!docSnapshot) { - this.logger.debug(`doc ${workspaceId}/${docId} not found`); - return; - } - if (docSnapshot.blob.length <= 2) { - this.logger.debug(`doc ${workspaceId}/${docId} is empty, skip indexing`); - return; - } - const metadata = { - workspaceId, - docId, - docSnapshotSize: docSnapshot.blob.length, - }; - - try { - const projection = projectDocSearch( - docSnapshot.blob, - docId, - docSnapshot.updatedAt.getTime().toString() - ); - await this.write( - SearchTable.doc, - [ - { - workspaceId, - docId, - title: projection.title, - summary: projection.units - .map(unit => unit.text) - .join('\n') - .slice(0, 1000), - // NOTE(@fengmk): journal is not supported yet - // journal: result.journal, - createdByUserId: docSnapshot.createdBy ?? '', - updatedByUserId: docSnapshot.updatedBy ?? '', - createdAt: docSnapshot.createdAt, - updatedAt: docSnapshot.updatedAt, - }, - ], - options - ); - await this.deleteBlocksByDocId(workspaceId, docId, options); - await this.write( - SearchTable.block, - projection.units.map(unit => ({ - workspaceId, - docId, - blockId: unit.blockId ?? unit.unitId, - unitId: unit.unitId, - projectionVersion: projection.version, - sourceHash: projection.sourceHash, - visibility: unit.visibility, - elementId: unit.elementId, - frameId: unit.frameId, - sourceBlockId: unit.blockId, - blob: unit.blobId, - refDocId: unit.refDocIds.length ? unit.refDocIds : undefined, - ref: unit.refs.length ? unit.refs : undefined, - content: unit.text, - flavour: `affine:${unit.type}`, - parentFlavour: unit.parentFlavour, - parentBlockId: unit.parentBlockId, - additional: unit.additional, - markdownPreview: undefined, - createdByUserId: docSnapshot.createdBy ?? '', - updatedByUserId: docSnapshot.updatedBy ?? '', - createdAt: docSnapshot.createdAt, - updatedAt: docSnapshot.updatedAt, - })), - options - ); - - this.logger.verbose( - `synced doc ${workspaceId}/${docId} with ${projection.units.length} search units` - ); - } catch (err) { - this.logger.warn( - `failed to parse ${workspaceId}/${docId}: ${err}`, - metadata - ); - } + async reconcileWorkspace(workspaceId: string) { + await this.runtime.reconcileSearchWorkspace(workspaceId); } - async deleteDoc( - workspaceId: string, - docId: string, - options?: OperationOptions - ) { - await this.deleteByQuery( - SearchTable.doc, - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId, - }, - ], - }, - options - ); - - await this.deleteBlocksByDocId(workspaceId, docId, options); - this.logger.log(`deleted doc ${workspaceId}/${docId}`); - } - - async deleteBlocksByDocId( - workspaceId: string, - docId: string, - options?: OperationOptions - ) { - await this.deleteByQuery( - SearchTable.block, - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'docId', - match: docId, - }, - ], - }, - options - ); - this.logger.debug(`deleted all blocks in doc ${workspaceId}/${docId}`); - } - - async deleteWorkspace(workspaceId: string, options?: OperationOptions) { - await this.deleteByQuery( - SearchTable.doc, - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - options - ); - this.logger.debug(`deleted all docs in workspace ${workspaceId}`); - await this.deleteByQuery( - SearchTable.block, - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - options - ); - this.logger.debug(`deleted all blocks in workspace ${workspaceId}`); - } - - async deleteByQuery( - table: T, - query: SearchQuery, - options?: OperationOptions - ) { - const searchProvider = this.factory.get(); - const dsl = this.#parseQuery(table, query); - await searchProvider.deleteByQuery(table, dsl, options); - } - - async searchBlobNames(workspaceId: string, blobIds: string[]) { - const result = await this.search({ - table: SearchTable.block, - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - { - type: SearchQueryType.match, - field: 'flavour', - match: 'affine:attachment', - }, - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.should, - queries: blobIds.map(blobId => ({ - type: SearchQueryType.match, - field: 'blob', - match: blobId, - })), - }, - ], - }, - options: { - fields: ['blob', 'content'], - pagination: { - limit: 10000, - }, - }, - }); - const blobNameMap = new Map(); - for (const node of result.nodes) { - const blobId = node.fields.blob[0] as string; - const content = node.fields.content[0] as string; - if (blobId && content) { - blobNameMap.set(blobId, content); - } - } - return blobNameMap; + async deleteWorkspace(workspaceId: string) { + await this.runtime.deleteSearchWorkspace(workspaceId); } async searchDocsByKeyword( + actorUserId: string, workspaceId: string, keyword: string, - options?: { - limit?: number; - docIds?: string[]; - } + options?: { limit?: number; docIds?: string[] } ): Promise { - if (options?.docIds?.length === 0) { - return []; - } - - const limit = options?.limit ?? 20; - const result = await this.aggregate({ - table: SearchTable.block, - field: 'docId', - query: { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'workspaceId', - match: workspaceId, - }, - ...(options?.docIds - ? [ - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.should, - queries: options.docIds.map(docId => ({ - type: SearchQueryType.match, - field: 'docId', - match: docId, - })), - }, - ] - : []), - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.must, - queries: [ - { - type: SearchQueryType.match, - field: 'content', - match: keyword, - }, - { - type: SearchQueryType.boolean, - occur: SearchQueryOccur.should, - queries: [ - { - type: SearchQueryType.match, - field: 'content', - match: keyword, - }, - { - type: SearchQueryType.boost, - boost: 1.5, - query: { - type: SearchQueryType.match, - field: 'flavour', - match: 'affine:page', - }, - }, - ], - }, - ], - }, - ], - }, - options: { - hits: { - fields: [ - 'blockId', - 'unitId', - 'projectionVersion', - 'sourceHash', - 'visibility', - 'elementId', - 'frameId', - 'sourceBlockId', - 'flavour', - 'content', - 'createdAt', - 'updatedAt', - 'createdByUserId', - 'updatedByUserId', - ], - highlights: [ - { - field: 'content', - before: '', - end: '', - }, - ], - pagination: { - limit: 2, - }, - }, - pagination: { - limit, - }, - }, - }); - - const docs: SearchDoc[] = []; - const missingTitles: { workspaceId: string; docId: string }[] = []; - const userIds: { userId: string }[] = []; - - for (const bucket of result.buckets) { - const docId = bucket.key; - const blockId = bucket.hits.nodes[0].fields.blockId[0] as string; - const unitId = bucket.hits.nodes[0].fields.unitId[0] as string; - const projectionVersion = bucket.hits.nodes[0].fields - .projectionVersion[0] as number; - const sourceHash = bucket.hits.nodes[0].fields.sourceHash[0] as string; - const visibility = bucket.hits.nodes[0].fields.visibility[0] as string; - const elementId = bucket.hits.nodes[0].fields.elementId?.[0] as - | string - | undefined; - const frameId = bucket.hits.nodes[0].fields.frameId?.[0] as - | string - | undefined; - const sourceBlockId = bucket.hits.nodes[0].fields.sourceBlockId?.[0] as - | string - | undefined; - const flavour = bucket.hits.nodes[0].fields.flavour[0] as string; - const content = bucket.hits.nodes[0].fields.content[0] as string; - const createdAt = bucket.hits.nodes[0].fields.createdAt[0] as Date; - const updatedAt = bucket.hits.nodes[0].fields.updatedAt[0] as Date; - const createdByUserId = bucket.hits.nodes[0].fields - .createdByUserId[0] as string; - const updatedByUserId = bucket.hits.nodes[0].fields - .updatedByUserId[0] as string; - const highlight = bucket.hits.nodes[0].highlights?.content?.[0] as string; - let title = ''; - - // hit title block - if (flavour === 'affine:page') { - title = content; - } else { - // hit content block, missing title - missingTitles.push({ workspaceId, docId }); - } - - docs.push({ - docId, - blockId: sourceBlockId || blockId, - ...(unitId ? { unitId } : {}), - ...(projectionVersion ? { projectionVersion } : {}), - ...(sourceHash ? { sourceHash } : {}), - ...(visibility ? { visibility } : {}), - ...(elementId ? { elementId } : {}), - ...(frameId ? { frameId } : {}), - title, - highlight, - createdAt, - updatedAt, - createdByUserId, - updatedByUserId, - }); - userIds.push({ userId: createdByUserId }, { userId: updatedByUserId }); - } - + if (options?.docIds?.length === 0) return []; + const result = await this.aggregate( + actorUserId, + workspaceId, + buildSearchDocsInput(workspaceId, keyword, options) + ); + const { docs, missingTitles, userIds } = collectSearchDocs( + result, + workspaceId + ); if (missingTitles.length > 0) { const metas = await this.models.doc.findMetas(missingTitles, { - select: { - title: true, - }, + select: { title: true }, }); - const titleMap = new Map(); - for (const meta of metas) { - if (meta?.title) { - titleMap.set(meta.docId, meta.title); - } - } + const titles = new Map( + metas.flatMap(meta => + meta?.title ? [[meta.docId, meta.title] as const] : [] + ) + ); for (const doc of docs) { - if (!doc.title) { - doc.title = titleMap.get(doc.docId) ?? ''; - } + if (!doc.title) doc.title = titles.get(doc.docId) ?? ''; } } - - const userMap = await this.models.user.getPublicUsersMap(userIds); - + const users = await this.models.user.getPublicUsersMap(userIds); for (const doc of docs) { - doc.createdByUser = userMap.get(doc.createdByUserId); - doc.updatedByUser = userMap.get(doc.updatedByUserId); + doc.createdByUser = users.get(doc.createdByUserId); + doc.updatedByUser = users.get(doc.updatedByUserId); } - return docs; } - #formatSearchNodes(nodes: SearchNode[]) { - return nodes.map(node => ({ - ...node, - fields: mapKeys(node.fields, (_, key) => camelCase(key)), - highlights: node.highlights - ? mapKeys(node.highlights, (_, key) => camelCase(key)) - : undefined, - _source: { - workspaceId: node._source.workspace_id, - docId: node._source.doc_id, - }, - })) as SearchNodeWithMeta[]; - } - - /** - * Parse input to ES query DSL - * @see https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html - */ - parseInput( - input: T - ): T extends SearchInput ? SearchQueryDSL : AggregateQueryDSL { - // common options - const query = this.#parseQuery(input.table, input.query); - const searchProvider = this.factory.get(); - const dsl: BaseQueryDSL = { - _source: [...DefaultSourceFields], - sort: [...SearchTableSorts[searchProvider.type][input.table]], - query, - }; - const pagination = input.options.pagination; - if (pagination?.limit) { - if (pagination.limit > 10000) { - throw new InvalidIndexerInput({ - reason: 'limit must be less than 10000', - }); - } - dsl.size = pagination.limit; + private unwrap(output: SearchOperationOutput, workspaceId: string): T { + if (output.ok) return output.value as T; + switch (output.errorCode) { + case 'workspace_denied': + throw new SpaceAccessDenied({ spaceId: workspaceId }); + case 'invalid_request': + case 'unsupported_query': + throw new InvalidIndexerInput({ reason: output.errorCode }); + case 'provider_unavailable': + throw new SearchProviderNotFound(); + case 'permission_unavailable': + throw new WorkspacePermissionNotFound({ spaceId: workspaceId }); + default: + throw new InternalServerError(); } - if (pagination?.skip) { - dsl.from = pagination.skip; - } - if (pagination?.cursor) { - dsl.cursor = pagination.cursor; - } - - if ('fields' in input.options) { - // for search input - const searchDsl: SearchQueryDSL = { - ...dsl, - fields: input.options.fields.map(snakeCase), - }; - if (input.options.highlights) { - searchDsl.highlight = this.#parseHighlights(input.options.highlights); - } - // @ts-expect-error should be SearchQueryDSL - return searchDsl; - } - - if ('field' in input) { - // for aggregate input - if (!AllowAggregateFields.has(input.field)) { - throw new InvalidIndexerInput({ - reason: `aggregate field "${input.field}" is not allowed`, - }); - } - - // input: { - // field: 'docId', - // options: { - // hits: { - // fields: [...], - // highlights: [...], - // pagination: { - // limit: 5, - // }, - // }, - // pagination: { - // limit: 100, - // }, - // }, - // } - // to - // "aggs": { - // "result": { - // "terms": { - // "field": "doc_id", - // "size": 100, - // "order": { - // "max_score": "desc" - // } - // }, - // "aggs": { - // "max_score": { - // "max": { - // "script": { - // "source": "_score" - // } - // } - // }, - // "result": { - // "top_hits": { - // "_source": false, - // "fields": [...], - // "highlights": [...], - // "size": 5 - // } - // } - // } - // } - // } - const topHits: TopHitsDSL = { - _source: [...DefaultSourceFields], - fields: input.options.hits.fields.map(snakeCase), - }; - if (input.options.hits.pagination?.limit) { - topHits.size = input.options.hits.pagination.limit; - } - if (input.options.hits.highlights) { - topHits.highlight = this.#parseHighlights( - input.options.hits.highlights - ); - } - const aggregateDsl: AggregateQueryDSL = { - ...dsl, - aggs: { - result: { - terms: { - field: snakeCase(input.field), - size: dsl.size, - order: { - max_score: 'desc', - }, - }, - aggs: { - max_score: { - max: { - script: { - source: '_score', - }, - }, - }, - result: { - // https://www.elastic.co/docs/reference/aggregations/search-aggregations-metrics-top-hits-aggregation - top_hits: topHits, - }, - }, - }, - }, - }; - // @ts-expect-error should be AggregateQueryDSL - return aggregateDsl; - } - - throw new InvalidIndexerInput({ - reason: '"field" or "fields" is required', - }); - } - - #parseQuery( - table: SearchTable, - query: SearchQuery, - parentNodes?: unknown[] - ): Record { - if (query.type === SearchQueryType.match) { - // required field and match - if (!query.field) { - throw new InvalidIndexerInput({ - reason: '"field" is required in match query', - }); - } - if (!query.match) { - throw new InvalidIndexerInput({ - reason: '"match" is required in match query', - }); - } - - // { - // type: 'match', - // field: 'content', - // match: keyword, - // } - // to - // { - // match: { - // content: { - // query: keyword - // }, - // }, - // } - // - // or - // { - // type: 'match', - // field: 'refDocId', - // match: docId, - // } - // to - // { - // term: { - // ref_doc_id: { - // value: docId - // }, - // }, - // } - const field = snakeCase(query.field); - const isFullTextField = SupportFullTextSearchFields[table].includes( - query.field - ); - const op = isFullTextField ? 'match' : 'term'; - const key = isFullTextField ? 'query' : 'value'; - const dsl = { - [op]: { - [field]: { - [key]: query.match, - ...(typeof query.boost === 'number' && { boost: query.boost }), - }, - }, - }; - if (parentNodes) { - parentNodes.push(dsl); - } - return dsl; - } - if (query.type === SearchQueryType.boolean) { - // required occur and queries - if (!query.occur) { - this.logger.debug(`query: ${JSON.stringify(query, null, 2)}`); - throw new InvalidIndexerInput({ - reason: '"occur" is required in boolean query', - }); - } - if (!query.queries) { - throw new InvalidIndexerInput({ - reason: '"queries" is required in boolean query', - }); - } - - // { - // type: 'boolean', - // occur: 'must_not', - // queries: [ - // { - // type: 'match', - // field: 'docId', - // match: 'docId1', - // }, - // ], - // } - // to - // { - // bool: { - // must_not: [ - // { - // match: { doc_id: { query: 'docId1' } } - // }, - // ], - // }, - // } - const nodes: unknown[] = []; - const dsl: Record = { - bool: { - [query.occur]: nodes, - ...(typeof query.boost === 'number' && { boost: query.boost }), - }, - }; - for (const subQuery of query.queries) { - this.#parseQuery(table, subQuery, nodes); - } - if (parentNodes) { - parentNodes.push(dsl); - } - return dsl; - } - if (query.type === SearchQueryType.exists) { - // required field - if (!query.field) { - throw new InvalidIndexerInput({ - reason: '"field" is required in exists query', - }); - } - - // { - // type: 'exists', - // field: 'refDocId', - // } - // to - // { - // exists: { - // field: 'ref_doc_id', - // }, - // } - const dsl = { - exists: { - field: snakeCase(query.field), - ...(typeof query.boost === 'number' && { boost: query.boost }), - }, - }; - if (parentNodes) { - parentNodes.push(dsl); - } - return dsl; - } - if (query.type === SearchQueryType.all) { - // { - // type: 'all' - // } - // to - // { - // match_all: {}, - // } - const dsl = { - match_all: { - ...(typeof query.boost === 'number' && { boost: query.boost }), - }, - }; - if (parentNodes) { - parentNodes.push(dsl); - } - return dsl; - } - if (query.type === SearchQueryType.boost) { - // required query and boost - if (!query.query) { - throw new InvalidIndexerInput({ - reason: '"query" is required in boost query', - }); - } - if (typeof query.boost !== 'number') { - throw new InvalidIndexerInput({ - reason: '"boost" is required in boost query', - }); - } - - // { - // type: 'boost', - // boost: 1.5, - // query: { - // type: 'match', - // field: 'flavour', - // match: 'affine:page', - // }, - // } - // to - // { - // "match": { - // "flavour": { - // "query": "affine:page", - // "boost": 1.5 - // } - // } - // } - return this.#parseQuery( - table, - { - ...query.query, - boost: query.boost, - }, - parentNodes - ); - } - throw new InvalidIndexerInput({ - reason: `unsupported query type: ${query.type}`, - }); - } - - /** - * Parse highlights to ES DSL - * @see https://www.elastic.co/docs/reference/elasticsearch/rest-apis/highlighting - */ - #parseHighlights(highlights: SearchHighlight[]) { - // [ - // { - // field: 'content', - // before: '', - // end: '', - // }, - // ] - // to - // { - // fields: { - // content: { - // pre_tags: [''], - // post_tags: [''], - // }, - // }, - // } - const fields = highlights.reduce( - (acc, highlight) => { - acc[snakeCase(highlight.field)] = { - pre_tags: [highlight.before], - post_tags: [highlight.end], - }; - return acc; - }, - {} as Record - ); - return { fields }; } } diff --git a/packages/backend/server/src/plugins/indexer/tables/block.ts b/packages/backend/server/src/plugins/indexer/tables/block.ts deleted file mode 100644 index f49ed08fa2..0000000000 --- a/packages/backend/server/src/plugins/indexer/tables/block.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { z } from 'zod'; - -export const BlockSchema = z.object({ - workspace_id: z.string(), - doc_id: z.string(), - block_id: z.string(), - unit_id: z.string().optional(), - projection_version: z.number().int().optional(), - source_hash: z.string().optional(), - visibility: z.string().optional(), - element_id: z.string().optional(), - frame_id: z.string().optional(), - source_block_id: z.string().optional(), - content: z.union([z.string(), z.string().array()]), - flavour: z.string(), - blob: z.union([z.string(), z.string().array()]).optional(), - ref_doc_id: z.union([z.string(), z.string().array()]).optional(), - ref: z.union([z.string(), z.string().array()]).optional(), - parent_flavour: z.string().optional(), - parent_block_id: z.string().optional(), - additional: z.string().optional(), - markdown_preview: z.string().optional(), - created_by_user_id: z.string(), - updated_by_user_id: z.string(), - created_at: z.date(), - updated_at: z.date(), -}); - -export type Block = z.input; - -export function getBlockUniqueId(block: Block) { - return `${block.workspace_id}/${block.doc_id}/${block.block_id}`; -} - -export const blockMapping = { - settings: { - analysis: { - analyzer: { - standard_with_cjk: { - tokenizer: 'standard', - filter: [ - 'lowercase', - 'cjk_bigram_and_unigrams', - // support `windows designer` => `windows`, `window`, `designer`, `design` - // @see https://www.elastic.co/docs/reference/text-analysis/analysis-remove-duplicates-tokenfilter - 'keyword_repeat', - 'stemmer', - 'remove_duplicates', - ], - }, - autocomplete: { - tokenizer: 'autocomplete_tokenizer', - filter: ['lowercase'], - }, - }, - tokenizer: { - autocomplete_tokenizer: { - type: 'edge_ngram', - min_gram: 1, - max_gram: 20, - token_chars: ['letter', 'digit', 'punctuation', 'symbol'], - }, - }, - filter: { - cjk_bigram_and_unigrams: { - type: 'cjk_bigram', - // output in unigram form, let `我是地球人` => `我`, `我是`, `是`, `是地`, `地`, `地球`, `球`, `球人`, `人` - // @see https://www.elastic.co/docs/reference/text-analysis/analysis-cjk-bigram-tokenfilter#analysis-cjk-bigram-tokenfilter-configure-parms - output_unigrams: true, - }, - }, - }, - }, - mappings: { - properties: { - workspace_id: { - type: 'keyword', - }, - doc_id: { - type: 'keyword', - }, - block_id: { - type: 'keyword', - }, - unit_id: { type: 'keyword' }, - projection_version: { type: 'integer' }, - source_hash: { type: 'keyword' }, - visibility: { type: 'keyword' }, - element_id: { type: 'keyword' }, - frame_id: { type: 'keyword' }, - source_block_id: { type: 'keyword' }, - content: { - type: 'text', - analyzer: 'standard_with_cjk', - search_analyzer: 'standard_with_cjk', - }, - flavour: { - type: 'keyword', - }, - blob: { - type: 'keyword', - }, - ref_doc_id: { - type: 'keyword', - }, - ref: { - type: 'text', - index: false, - }, - parent_flavour: { - type: 'keyword', - }, - parent_block_id: { - type: 'keyword', - }, - additional: { - type: 'text', - index: false, - }, - markdown_preview: { - type: 'text', - index: false, - }, - created_by_user_id: { - type: 'keyword', - }, - updated_by_user_id: { - type: 'keyword', - }, - created_at: { - type: 'date', - }, - updated_at: { - type: 'date', - }, - }, - }, -}; - -export const blockSQL = ` -CREATE TABLE IF NOT EXISTS block ( - workspace_id string attribute, - doc_id string attribute, - block_id string attribute, - unit_id string attribute, - projection_version int, - source_hash string attribute, - visibility string attribute, - element_id string attribute, - frame_id string attribute, - source_block_id string attribute, - content text, - flavour string attribute, - -- use flavour_indexed to match with boost - flavour_indexed string attribute indexed, - blob string attribute indexed, - -- ref_doc_id need match query - ref_doc_id string attribute indexed, - ref string stored, - parent_flavour string attribute, - -- use parent_flavour_indexed to match with boost - parent_flavour_indexed string attribute indexed, - parent_block_id string attribute, - -- use parent_block_id_indexed to match with boost, exists query - parent_block_id_indexed string attribute indexed, - additional string stored, - markdown_preview string stored, - created_by_user_id string attribute, - updated_by_user_id string attribute, - created_at timestamp, - updated_at timestamp -) -morphology = 'jieba_chinese, lemmatize_en_all, lemmatize_de_all, lemmatize_ru_all, libstemmer_ar, libstemmer_ca, stem_cz, libstemmer_da, libstemmer_nl, libstemmer_fi, libstemmer_fr, libstemmer_el, libstemmer_hi, libstemmer_hu, libstemmer_id, libstemmer_ga, libstemmer_it, libstemmer_lt, libstemmer_ne, libstemmer_no, libstemmer_pt, libstemmer_ro, libstemmer_es, libstemmer_sv, libstemmer_ta, libstemmer_tr' -charset_table = 'non_cjk, chinese' -ngram_len = '1' -ngram_chars = 'U+1100..U+11FF, U+3130..U+318F, U+A960..U+A97F, U+AC00..U+D7AF, U+D7B0..U+D7FF, U+3040..U+30FF, U+0E00..U+0E7F' -index_field_lengths = '1' -`; diff --git a/packages/backend/server/src/plugins/indexer/tables/doc.ts b/packages/backend/server/src/plugins/indexer/tables/doc.ts deleted file mode 100644 index ff5ce52e58..0000000000 --- a/packages/backend/server/src/plugins/indexer/tables/doc.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { z } from 'zod'; - -export const DocSchema = z.object({ - workspace_id: z.string(), - doc_id: z.string(), - title: z.string(), - summary: z.string(), - journal: z.string().optional(), - created_by_user_id: z.string(), - updated_by_user_id: z.string(), - created_at: z.date(), - updated_at: z.date(), -}); - -export type Doc = z.input; - -export function getDocUniqueId(doc: Doc) { - return `${doc.workspace_id}/${doc.doc_id}`; -} - -export const docMapping = { - settings: { - analysis: { - analyzer: { - standard_with_cjk: { - tokenizer: 'standard', - filter: [ - 'lowercase', - 'cjk_bigram_and_unigrams', - 'keyword_repeat', - 'stemmer', - 'remove_duplicates', - ], - }, - autocomplete: { - tokenizer: 'autocomplete_tokenizer', - filter: ['lowercase'], - }, - }, - tokenizer: { - autocomplete_tokenizer: { - type: 'edge_ngram', - min_gram: 1, - max_gram: 20, - token_chars: ['letter', 'digit', 'punctuation', 'symbol'], - }, - }, - filter: { - cjk_bigram_and_unigrams: { - type: 'cjk_bigram', - output_unigrams: true, - }, - }, - }, - }, - mappings: { - properties: { - workspace_id: { - type: 'keyword', - }, - doc_id: { - type: 'keyword', - }, - title: { - type: 'text', - analyzer: 'standard_with_cjk', - search_analyzer: 'standard_with_cjk', - fields: { - autocomplete: { - type: 'text', - analyzer: 'autocomplete', - search_analyzer: 'standard', - }, - }, - }, - summary: { - type: 'text', - index: false, - }, - journal: { - type: 'keyword', - }, - created_by_user_id: { - type: 'keyword', - }, - updated_by_user_id: { - type: 'keyword', - }, - created_at: { - type: 'date', - }, - updated_at: { - type: 'date', - }, - }, - }, -}; - -export const docSQL = ` -CREATE TABLE IF NOT EXISTS doc ( - workspace_id string attribute, - doc_id string attribute, - title text, - summary string stored, - journal string stored, - created_by_user_id string attribute, - updated_by_user_id string attribute, - created_at timestamp, - updated_at timestamp -) -morphology = 'jieba_chinese, lemmatize_en_all, lemmatize_de_all, lemmatize_ru_all, libstemmer_ar, libstemmer_ca, stem_cz, libstemmer_da, libstemmer_nl, libstemmer_fi, libstemmer_fr, libstemmer_el, libstemmer_hi, libstemmer_hu, libstemmer_id, libstemmer_ga, libstemmer_it, libstemmer_lt, libstemmer_ne, libstemmer_no, libstemmer_pt, libstemmer_ro, libstemmer_es, libstemmer_sv, libstemmer_ta, libstemmer_tr' -charset_table = 'non_cjk, chinese' -ngram_len = '1' -ngram_chars = 'U+1100..U+11FF, U+3130..U+318F, U+A960..U+A97F, U+AC00..U+D7AF, U+D7B0..U+D7FF, U+3040..U+30FF, U+0E00..U+0E7F' -index_field_lengths = '1' -`; diff --git a/packages/backend/server/src/plugins/indexer/tables/index.ts b/packages/backend/server/src/plugins/indexer/tables/index.ts deleted file mode 100644 index 729d11a651..0000000000 --- a/packages/backend/server/src/plugins/indexer/tables/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { getBlockUniqueId } from './block'; -import { getDocUniqueId } from './doc'; - -export enum SearchTable { - block = 'block', - doc = 'doc', -} - -export const SearchTableUniqueId = { - [SearchTable.block]: getBlockUniqueId, - [SearchTable.doc]: getDocUniqueId, -}; - -export const DateFieldNames = ['created_at', 'updated_at']; - -export * from './block'; -export * from './doc'; diff --git a/packages/backend/server/src/plugins/indexer/types.ts b/packages/backend/server/src/plugins/indexer/types.ts index 5bc41f9b5e..f97839be76 100644 --- a/packages/backend/server/src/plugins/indexer/types.ts +++ b/packages/backend/server/src/plugins/indexer/types.ts @@ -11,7 +11,11 @@ import { GraphQLJSONObject } from 'graphql-scalars'; import { PublicUserType } from '../../core/user'; import { PublicUser } from '../../models'; -import { SearchTable } from './tables'; + +export enum SearchTable { + block = 'block', + doc = 'doc', +} export enum SearchQueryType { match = 'match', diff --git a/packages/backend/server/src/plugins/oauth/controller.ts b/packages/backend/server/src/plugins/oauth/controller.ts index aca8ea98ac..6d09260521 100644 --- a/packages/backend/server/src/plugins/oauth/controller.ts +++ b/packages/backend/server/src/plugins/oauth/controller.ts @@ -14,6 +14,7 @@ import { ActionForbidden, getClientVersionFromRequest, MissingOauthQueryParameter, + Throttle, UnknownOauthProvider, URLHelper, UseNamedGuard, @@ -24,6 +25,7 @@ import { OAuthProviderFactory } from './factory'; import { OAuthCallbackBodySchema, OAuthPreflightBodySchema } from './input'; import { OAuthService } from './service'; +@Throttle('strict') @Controller('/api/oauth') export class OAuthController { constructor( diff --git a/packages/backend/server/src/server.ts b/packages/backend/server/src/server.ts index 5734516bd9..d574059f47 100644 --- a/packages/backend/server/src/server.ts +++ b/packages/backend/server/src/server.ts @@ -19,6 +19,7 @@ import { import { SocketIoAdapter } from './base/websocket'; import { AuthGuard } from './core/auth'; import { TelemetryService } from './core/telemetry/service'; +import { ServerRole } from './env'; import { serverTimingAndCache } from './middleware/timing'; const OneMB = 1024 * 1024; @@ -93,7 +94,11 @@ export async function run() { }) ); - app.useGlobalGuards(app.get(AuthGuard), app.get(CloudThrottlerGuard)); + if (env.role === ServerRole.Worker) { + app.useGlobalGuards(app.get(CloudThrottlerGuard)); + } else { + app.useGlobalGuards(app.get(AuthGuard), app.get(CloudThrottlerGuard)); + } app.useGlobalInterceptors(app.get(CacheInterceptor)); app.useGlobalFilters(new GlobalExceptionFilter(app.getHttpAdapter())); app.use(cookieParser()); diff --git a/packages/common/nbstore/src/__tests__/cloud-doc-updates.spec.ts b/packages/common/nbstore/src/__tests__/cloud-doc-updates.spec.ts index 71816d2d84..22aa0512ab 100644 --- a/packages/common/nbstore/src/__tests__/cloud-doc-updates.spec.ts +++ b/packages/common/nbstore/src/__tests__/cloud-doc-updates.spec.ts @@ -1,16 +1,51 @@ -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; +import { CloudAwarenessStorage } from '../impls/cloud/awareness'; import { CloudDocStorage } from '../impls/cloud/doc'; const base64UpdateA = 'AQID'; const base64UpdateB = 'BAUG'; +class FakeSocket { + connected = true; + readonly emitted: Array<{ event: string; payload: unknown }> = []; + readonly handlers = new Map void>(); + + on(event: string, handler: (...args: unknown[]) => void) { + this.handlers.set(event, handler); + return this; + } + + once(event: string, handler: (...args: unknown[]) => void) { + this.handlers.set(event, handler); + return this; + } + + off(event: string, handler?: (...args: unknown[]) => void) { + if (!handler || this.handlers.get(event) === handler) { + this.handlers.delete(event); + } + return this; + } + + emit(event: string, payload?: unknown) { + this.emitted.push({ event, payload }); + return true; + } + + async emitWithAck(event: string, payload: unknown) { + this.emitted.push({ event, payload }); + return { data: { clientId: 'client-1', success: true } }; + } +} + describe('CloudDocStorage broadcast updates', () => { test('emits updates from batch payload', () => { const storage = new CloudDocStorage({ id: 'space-1', serverBaseUrl: 'http://localhost', isSelfHosted: true, + syncProtocol: 'legacy', type: 'workspace', readonlyMode: true, }); @@ -38,4 +73,164 @@ describe('CloudDocStorage broadcast updates', () => { new Uint8Array([4, 5, 6]), ]); }); + + test('repairs strict invalidation through readable timestamps', async () => { + const storage = new CloudDocStorage({ + id: 'space-1', + serverBaseUrl: 'http://localhost', + isSelfHosted: true, + syncProtocol: 'batch', + type: 'workspace', + readonlyMode: true, + }); + + (storage as any).connection.idConverter = { + oldIdToNewId: (id: string) => id, + newIdToOldId: (id: string) => id, + }; + + const getDocTimestamps = vi + .spyOn(storage, 'getDocTimestamps') + .mockResolvedValue({ 'doc-a': new Date(1_000) }); + const received: Array<{ docId: string; bin: Uint8Array }> = []; + storage.subscribeDocUpdate(update => { + received.push({ docId: update.docId, bin: update.bin }); + }); + + storage.onServerInvalidation({ + spaceType: 'workspace', + spaceId: 'space-1', + timestamp: 1_000, + }); + storage.onServerInvalidation({ + spaceType: 'workspace', + spaceId: 'space-1', + timestamp: 1_001, + }); + + await vi.waitFor(() => expect(received).toHaveLength(1)); + expect(getDocTimestamps).toHaveBeenCalledOnce(); + expect(received[0]).toMatchObject({ docId: 'doc-a' }); + expect(received[0]?.bin).toEqual(new Uint8Array()); + }); + + test.each([ + ['legacy', 'space:join'], + ['batch', 'space:join-batch'], + ] as const)( + '%s route sends its own join event', + async (syncProtocol, event) => { + vi.stubGlobal('BUILD_CONFIG', { appVersion: '0.27.5' }); + const fakeSocket = new FakeSocket(); + const disconnect = vi.fn(); + const storage = new CloudDocStorage({ + id: 'space-1', + serverBaseUrl: 'http://localhost', + isSelfHosted: true, + syncProtocol, + type: 'workspace', + readonlyMode: true, + }); + const connection = storage.connection as any; + + Object.defineProperty(connection, 'manager', { + configurable: true, + value: { + connect: () => ({ socket: fakeSocket, disconnect }), + }, + }); + vi.spyOn(connection, 'getIdConverter').mockResolvedValue({ + oldIdToNewId: (id: string) => id, + newIdToOldId: (id: string) => id, + }); + + const inner = await connection.doConnect(); + expect(fakeSocket.emitted[0]?.event).toBe(event); + + inner.disconnect(); + vi.unstubAllGlobals(); + } + ); + + test.each([ + ['legacy', 'space:join-awareness'], + ['batch', 'space:join-batch'], + ] as const)( + '%s awareness joins for active documents', + async (syncProtocol, event) => { + vi.stubGlobal('BUILD_CONFIG', { appVersion: '0.27.5' }); + const fakeSocket = new FakeSocket(); + const storage = new CloudAwarenessStorage({ + id: 'space-1', + serverBaseUrl: 'http://localhost', + isSelfHosted: true, + syncProtocol, + type: 'workspace', + }); + + Object.defineProperty(storage, 'connection', { + configurable: true, + value: { + status: 'connected', + inner: { socket: fakeSocket }, + onStatusChanged: () => () => {}, + }, + }); + + const unsubscribeA = storage.subscribeUpdate( + 'doc-a', + () => {}, + async () => null + ); + const unsubscribeB = storage.subscribeUpdate( + 'doc-b', + () => {}, + async () => null + ); + + await vi.waitFor(() => { + expect( + fakeSocket.emitted.filter( + ({ event: emittedEvent }) => emittedEvent === event + ) + ).toHaveLength(syncProtocol === 'batch' ? 1 : 2); + }); + + if (syncProtocol === 'batch') { + expect(fakeSocket.emitted).toContainEqual({ + event, + payload: { + spaces: [ + { spaceType: 'workspace', spaceId: 'space-1', docId: 'doc-a' }, + { spaceType: 'workspace', spaceId: 'space-1', docId: 'doc-b' }, + ], + clientVersion: '0.27.5', + }, + }); + } else { + expect(fakeSocket.emitted).toContainEqual({ + event, + payload: { + spaceType: 'workspace', + spaceId: 'space-1', + docId: 'doc-a', + clientVersion: '0.27.5', + }, + }); + expect(fakeSocket.emitted).toContainEqual({ + event, + payload: { + spaceType: 'workspace', + spaceId: 'space-1', + docId: 'doc-b', + clientVersion: '0.27.5', + }, + }); + } + + unsubscribeA(); + unsubscribeB(); + vi.unstubAllGlobals(); + } + ); }); diff --git a/packages/common/nbstore/src/__tests__/sync.spec.ts b/packages/common/nbstore/src/__tests__/sync.spec.ts index 29235e5234..12b7c9d45f 100644 --- a/packages/common/nbstore/src/__tests__/sync.spec.ts +++ b/packages/common/nbstore/src/__tests__/sync.spec.ts @@ -694,6 +694,7 @@ test('indexer defers indexed clock persistence until a refresh happens on delaye }) ); const indexer = new TrackingIndexerStorage(calls, 30_000); + const update = vi.spyOn(indexer, 'update'); const indexerSyncStorage = new TrackingIndexerSyncStorage(calls); const sync = new IndexerSyncImpl( docStorage, @@ -712,6 +713,15 @@ test('indexer defers indexed clock persistence until a refresh happens on delaye sync.start(); await sync.waitForCompleted(); + const docUpdate = update.mock.calls.find(([table]) => table === 'doc'); + expect(docUpdate).toBeDefined(); + expect([...(docUpdate?.[1].fields ?? [])]).toEqual( + expect.arrayContaining([ + ['docId', ['doc1']], + ['title', ['Doc 1']], + ['summary', ['summary']], + ]) + ); expect(calls).not.toContain('setClock:doc1'); sync.stop(); diff --git a/packages/common/nbstore/src/impls/cloud/awareness.ts b/packages/common/nbstore/src/impls/cloud/awareness.ts index 17f9280d48..1571b580cd 100644 --- a/packages/common/nbstore/src/impls/cloud/awareness.ts +++ b/packages/common/nbstore/src/impls/cloud/awareness.ts @@ -6,12 +6,15 @@ import type { SpaceType } from '../../utils/universal-id'; import { base64ToUint8Array, SocketConnection, + SPACE_JOIN_BATCH_LIMIT, + type SyncProtocol, uint8ArrayToBase64, } from './socket'; interface CloudAwarenessStorageOptions { isSelfHosted: boolean; serverBaseUrl: string; + syncProtocol: SyncProtocol; type: SpaceType; id: string; } @@ -32,6 +35,97 @@ export class CloudAwarenessStorage extends AwarenessStorageBase { return this.connection.inner.socket; } + private readonly activeAwarenessIds = new Set(); + private readonly joinedAwarenessIds = new Set(); + private joinPromise: Promise | undefined; + + private joinActiveAwareness(): Promise { + if ( + this.connection.status !== 'connected' || + this.activeAwarenessIds.size === 0 + ) { + return Promise.resolve(); + } + + if (this.joinPromise) { + return this.joinPromise; + } + + const batchPromise = (async () => { + while (this.connection.status === 'connected') { + await Promise.resolve(); + const pendingIds = [...this.activeAwarenessIds].filter( + docId => !this.joinedAwarenessIds.has(docId) + ); + if (pendingIds.length === 0) { + return; + } + + if (this.options.syncProtocol === 'batch') { + for ( + let index = 0; + index < pendingIds.length; + index += SPACE_JOIN_BATCH_LIMIT + ) { + const spaces = pendingIds + .slice(index, index + SPACE_JOIN_BATCH_LIMIT) + .map(docId => ({ + spaceType: this.options.type, + spaceId: this.options.id, + docId, + })); + const response = await this.socket.emitWithAck('space:join-batch', { + spaces, + clientVersion: BUILD_CONFIG.appVersion, + }); + + if ('error' in response) { + throw new Error( + `Awareness join failed: ${response.error.name}: ${response.error.message}` + ); + } + if (!response.data.success) { + throw new Error('Awareness join was rejected'); + } + } + } else { + for (const docId of pendingIds) { + const response = await this.socket.emitWithAck( + 'space:join-awareness', + { + spaceType: this.options.type, + spaceId: this.options.id, + docId, + clientVersion: BUILD_CONFIG.appVersion, + } + ); + + if ('error' in response) { + throw new Error( + `Awareness join failed: ${response.error.name}: ${response.error.message}` + ); + } + if (!response.data.success) { + throw new Error('Awareness join was rejected'); + } + } + } + + for (const docId of pendingIds) { + if (this.activeAwarenessIds.has(docId)) { + this.joinedAwarenessIds.add(docId); + } + } + } + })(); + + const sharedPromise = batchPromise.finally(() => { + this.joinPromise = undefined; + }); + this.joinPromise = sharedPromise; + return sharedPromise; + } + override async update(record: AwarenessRecord): Promise { const encodedUpdate = await uint8ArrayToBase64(record.bin); this.socket.emit('space:update-awareness', { @@ -47,19 +141,31 @@ export class CloudAwarenessStorage extends AwarenessStorageBase { onUpdate: (update: AwarenessRecord, origin?: string) => void, onCollect: () => Promise ): () => void { + this.activeAwarenessIds.add(id); + // leave awareness const leave = () => { - if (this.connection.status !== 'connected') return; + this.activeAwarenessIds.delete(id); + this.joinedAwarenessIds.delete(id); this.socket.off('space:collect-awareness', handleCollectAwareness); this.socket.off( 'space:broadcast-awareness-update', handleBroadcastAwarenessUpdate ); - this.socket.emit('space:leave-awareness', { - spaceType: this.options.type, - spaceId: this.options.id, - docId: id, - }); + if (this.connection.status !== 'connected') return; + if (this.options.syncProtocol === 'batch') { + this.socket.emit('space:leave-batch', { + spaceType: this.options.type, + spaceId: this.options.id, + docIds: [id], + }); + } else { + this.socket.emit('space:leave-awareness', { + spaceType: this.options.type, + spaceId: this.options.id, + docId: id, + }); + } }; // join awareness, and collect awareness from others @@ -69,12 +175,8 @@ export class CloudAwarenessStorage extends AwarenessStorageBase { 'space:broadcast-awareness-update', handleBroadcastAwarenessUpdate ); - await this.socket.emitWithAck('space:join-awareness', { - spaceType: this.options.type, - spaceId: this.options.id, - docId: id, - clientVersion: BUILD_CONFIG.appVersion, - }); + await this.joinActiveAwareness(); + if (this.connection.status !== 'connected') return; this.socket.emit('space:load-awarenesses', { spaceType: this.options.type, spaceId: this.options.id, @@ -142,6 +244,9 @@ export class CloudAwarenessStorage extends AwarenessStorageBase { const unsubscribeConnectionStatusChanged = this.connection.onStatusChanged( status => { + if (status !== 'connected') { + this.joinedAwarenessIds.clear(); + } if (status === 'connected') { joinAndCollect().catch(err => console.error('awareness join failed', err) diff --git a/packages/common/nbstore/src/impls/cloud/doc.ts b/packages/common/nbstore/src/impls/cloud/doc.ts index 84c20cb36f..2e5dd4e913 100644 --- a/packages/common/nbstore/src/impls/cloud/doc.ts +++ b/packages/common/nbstore/src/impls/cloud/doc.ts @@ -13,12 +13,14 @@ import { base64ToUint8Array, type ServerEventsMap, SocketConnection, + type SyncProtocol, uint8ArrayToBase64, } from './socket'; interface CloudDocStorageOptions extends DocStorageOptions { serverBaseUrl: string; isSelfHosted: boolean; + syncProtocol: SyncProtocol; type: SpaceType; } @@ -42,22 +44,6 @@ export class CloudDocStorage extends DocStorageBase { } readonly spaceType = this.options.type; - onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => { - if ( - this.spaceType !== message.spaceType || - this.spaceId !== message.spaceId - ) { - return; - } - - this.emit('update', { - docId: this.idConverter.oldIdToNewId(message.docId), - bin: base64ToUint8Array(message.update), - timestamp: new Date(message.timestamp), - editor: message.editor, - }); - }; - onServerUpdates: ServerEventsMap['space:broadcast-doc-updates'] = message => { if ( this.spaceType !== message.spaceType || @@ -66,9 +52,11 @@ export class CloudDocStorage extends DocStorageBase { return; } + const docId = this.idConverter.oldIdToNewId(message.docId); + this.serverDocTimestamps.set(docId, message.timestamp); for (const update of message.updates) { this.emit('update', { - docId: this.idConverter.oldIdToNewId(message.docId), + docId, bin: base64ToUint8Array(update), timestamp: new Date(message.timestamp), editor: message.editor, @@ -76,10 +64,70 @@ export class CloudDocStorage extends DocStorageBase { } }; + onServerInvalidation: ServerEventsMap['space:broadcast-doc-invalidation'] = + message => { + if ( + this.options.syncProtocol !== 'batch' || + this.spaceType !== message.spaceType || + this.spaceId !== message.spaceId + ) { + return; + } + + this.invalidationPending = true; + this.scheduleInvalidationRepair(); + }; + + private readonly serverDocTimestamps = new Map(); + private invalidationTimer?: ReturnType; + private invalidationRepair?: Promise; + private invalidationPending = false; + + private scheduleInvalidationRepair() { + if (this.invalidationTimer) { + clearTimeout(this.invalidationTimer); + } + this.invalidationTimer = setTimeout(() => { + this.invalidationTimer = undefined; + if (this.invalidationRepair) { + return; + } + this.invalidationPending = false; + this.invalidationRepair = this.repairInvalidatedDocs() + .catch(error => { + console.error('failed to repair invalidated docs', error); + }) + .finally(() => { + this.invalidationRepair = undefined; + if (this.invalidationPending) { + this.scheduleInvalidationRepair(); + } + }); + }, 50); + } + + private async repairInvalidatedDocs() { + const timestamps = await this.getDocTimestamps(); + for (const [docId, timestamp] of Object.entries(timestamps)) { + const newDocId = this.idConverter.oldIdToNewId(docId); + const timestampValue = timestamp.getTime(); + const previous = this.serverDocTimestamps.get(newDocId); + if (previous !== undefined && previous >= timestampValue) { + continue; + } + this.serverDocTimestamps.set(newDocId, timestampValue); + this.emit('update', { + docId: newDocId, + bin: new Uint8Array(), + timestamp, + }); + } + } + readonly connection = new CloudDocStorageConnection( this.options, - this.onServerUpdate, - this.onServerUpdates + this.onServerUpdates, + this.onServerInvalidation ); override async getDocSnapshot(docId: string) { @@ -216,8 +264,8 @@ export class CloudDocStorage extends DocStorageBase { class CloudDocStorageConnection extends SocketConnection { constructor( private readonly options: CloudDocStorageOptions, - private readonly onServerUpdate: ServerEventsMap['space:broadcast-doc-update'], - private readonly onServerUpdates: ServerEventsMap['space:broadcast-doc-updates'] + private readonly onServerUpdates: ServerEventsMap['space:broadcast-doc-updates'], + private readonly onServerInvalidation: ServerEventsMap['space:broadcast-doc-invalidation'] ) { super(options.serverBaseUrl, options.isSelfHosted); } @@ -228,22 +276,36 @@ class CloudDocStorageConnection extends SocketConnection { const { socket, disconnect } = await super.doConnect(signal); try { - const res = await socket.emitWithAck('space:join', { - spaceType: this.options.type, - spaceId: this.options.id, - clientVersion: BUILD_CONFIG.appVersion, - }); + const res = + this.options.syncProtocol === 'batch' + ? await socket.emitWithAck('space:join-batch', { + spaces: [ + { + spaceType: this.options.type, + spaceId: this.options.id, + }, + ], + clientVersion: BUILD_CONFIG.appVersion, + }) + : await socket.emitWithAck('space:join', { + spaceType: this.options.type, + spaceId: this.options.id, + clientVersion: BUILD_CONFIG.appVersion, + }); if ('error' in res) { throw createWebsocketError(res.error); } + if (!res.data.success) { + throw new Error('Space join was rejected'); + } if (!this.idConverter) { this.idConverter = await this.getIdConverter(socket); } - socket.on('space:broadcast-doc-update', this.onServerUpdate); socket.on('space:broadcast-doc-updates', this.onServerUpdates); + socket.on('space:broadcast-doc-invalidation', this.onServerInvalidation); return { socket, disconnect }; } catch (e) { @@ -259,12 +321,20 @@ class CloudDocStorageConnection extends SocketConnection { socket: Socket; disconnect: () => void; }) { - socket.emit('space:leave', { - spaceType: this.options.type, - spaceId: this.options.id, - }); - socket.off('space:broadcast-doc-update', this.onServerUpdate); + if (this.options.syncProtocol === 'batch') { + socket.emit('space:leave-batch', { + spaceType: this.options.type, + spaceId: this.options.id, + docIds: [], + }); + } else { + socket.emit('space:leave', { + spaceType: this.options.type, + spaceId: this.options.id, + }); + } socket.off('space:broadcast-doc-updates', this.onServerUpdates); + socket.off('space:broadcast-doc-invalidation', this.onServerInvalidation); super.doDisconnect({ socket, disconnect }); } diff --git a/packages/common/nbstore/src/impls/cloud/socket.ts b/packages/common/nbstore/src/impls/cloud/socket.ts index 0da5776d97..5e61024e89 100644 --- a/packages/common/nbstore/src/impls/cloud/socket.ts +++ b/packages/common/nbstore/src/impls/cloud/socket.ts @@ -28,14 +28,6 @@ type WebsocketResponse = }; interface ServerEvents { - 'space:broadcast-doc-update': { - spaceType: string; - spaceId: string; - docId: string; - update: string; - timestamp: number; - editor: string; - }; 'space:broadcast-doc-updates': { spaceType: string; spaceId: string; @@ -45,6 +37,11 @@ interface ServerEvents { editor?: string; compressed?: boolean; }; + 'space:broadcast-doc-invalidation': { + spaceType: string; + spaceId: string; + timestamp: number; + }; 'space:collect-awareness': { spaceType: string; @@ -62,11 +59,29 @@ interface ServerEvents { 'realtime:event': RealtimeEvent; } +export type SyncProtocol = 'legacy' | 'batch'; + interface ClientEvents { 'space:join': [ { spaceType: string; spaceId: string; clientVersion: string }, - { clientId: string }, + { clientId: string; success: boolean }, ]; + 'space:join-batch': [ + { + spaces: Array<{ + spaceType: string; + spaceId: string; + docId?: string; + }>; + clientVersion: string; + }, + { clientId: string; success: boolean }, + ]; + 'space:leave-batch': { + spaceType: string; + spaceId: string; + docIds: string[]; + }; 'space:leave': { spaceType: string; spaceId: string }; 'space:join-awareness': [ { @@ -75,7 +90,7 @@ interface ClientEvents { docId: string; clientVersion: string; }, - { clientId: string }, + { clientId: string; success: boolean }, ]; 'space:leave-awareness': { spaceType: string; @@ -133,6 +148,8 @@ interface ClientEvents { 'realtime:unsubscribe': [RealtimeUnsubscribeEnvelope, { ok: true }]; } +export const SPACE_JOIN_BATCH_LIMIT = 100; + export type ServerEventsMap = { [Key in keyof ServerEvents]: (data: ServerEvents[Key]) => void; }; diff --git a/packages/common/nbstore/src/impls/sqlite/db.ts b/packages/common/nbstore/src/impls/sqlite/db.ts index bace458c2b..c49b7add30 100644 --- a/packages/common/nbstore/src/impls/sqlite/db.ts +++ b/packages/common/nbstore/src/impls/sqlite/db.ts @@ -15,6 +15,42 @@ export interface SqliteNativeDBOptions { readonly id: string; } +export interface NativeIndexField { + field: string; + values: string[]; +} + +export interface NativeIndexQuery { + kind: 'match' | 'exists' | 'all' | 'boolean' | 'boost'; + field?: string; + value?: string; + occur?: 'must' | 'should' | 'must_not'; + clauses?: NativeIndexQuery[]; + boost?: number; +} + +export interface NativeIndexSearchOptions { + limit: number; + offset: number; + fields: string[]; + highlights: string[]; +} + +export interface NativeIndexHit { + id: string; + score: number; + fields: NativeIndexField[]; + highlights: { + field: string; + values: { valueIndex: number; spans: { start: number; end: number }[] }[]; + }[]; +} + +export interface NativeIndexSearchResult { + total: number; + hits: NativeIndexHit[]; +} + export interface NativeDBApis { connect: (id: string) => Promise; disconnect: (id: string) => Promise; @@ -40,6 +76,7 @@ export interface NativeDBApis { indexedClock: Date, indexerVersion: number ) => Promise; + setDocIndexedClocks: (id: string, clocks: DocIndexedClock[]) => Promise; clearDocIndexedClock: (id: string, docId: string) => Promise; getBlob: (id: string, key: string) => Promise; setBlob: (id: string, blob: BlobRecord) => Promise; @@ -95,36 +132,42 @@ export interface NativeDBApis { blobId: string ) => Promise; crawlDocData: (id: string, docId: string) => Promise; - ftsAddDocument: ( + indexUpsert: ( id: string, - indexName: string, - docId: string, - text: string, - index: boolean + table: string, + document: { id: string; fields: NativeIndexField[] } ) => Promise; - ftsDeleteDocument: ( + indexDelete: (id: string, table: string, docId: string) => Promise; + indexSearch: ( id: string, - indexName: string, - docId: string - ) => Promise; - ftsSearch: ( + table: string, + query: NativeIndexQuery, + options: NativeIndexSearchOptions + ) => Promise; + indexAggregate: ( id: string, - indexName: string, - query: string - ) => Promise<{ id: string; score: number; terms: Array }[]>; - ftsGetDocument: ( + table: string, + query: NativeIndexQuery, + field: string, + limit: number, + offset: number, + hits?: NativeIndexSearchOptions + ) => Promise<{ + total: number; + buckets: { + key: string; + count: number; + score: number; + hits: NativeIndexHit[]; + }[]; + }>; + indexDeleteByQuery: ( id: string, - indexName: string, - docId: string - ) => Promise; - ftsGetMatches: ( - id: string, - indexName: string, - docId: string, - query: string - ) => Promise<{ start: number; end: number }[]>; - ftsFlushIndex: (id: string) => Promise; - ftsIndexVersion: () => Promise; + table: string, + query: NativeIndexQuery + ) => Promise; + indexFlush: (id: string) => Promise; + indexVersion: () => Promise; } type NativeDBApisWrapper = NativeDBApis extends infer APIs diff --git a/packages/common/nbstore/src/impls/sqlite/index.ts b/packages/common/nbstore/src/impls/sqlite/index.ts index 4c09365dce..7d125fc57f 100644 --- a/packages/common/nbstore/src/impls/sqlite/index.ts +++ b/packages/common/nbstore/src/impls/sqlite/index.ts @@ -8,7 +8,15 @@ import { SqliteIndexerSyncStorage } from './indexer-sync'; export * from './blob'; export * from './blob-sync'; -export { bindNativeDBApis, type NativeDBApis } from './db'; +export { + bindNativeDBApis, + type NativeDBApis, + type NativeIndexField, + type NativeIndexHit, + type NativeIndexQuery, + type NativeIndexSearchOptions, + type NativeIndexSearchResult, +} from './db'; export * from './doc'; export * from './doc-sync'; export * from './indexer'; diff --git a/packages/common/nbstore/src/impls/sqlite/indexer-sync.ts b/packages/common/nbstore/src/impls/sqlite/indexer-sync.ts index d57daec763..7e6e5873b8 100644 --- a/packages/common/nbstore/src/impls/sqlite/indexer-sync.ts +++ b/packages/common/nbstore/src/impls/sqlite/indexer-sync.ts @@ -7,6 +7,7 @@ import { NativeDBConnection, type SqliteNativeDBOptions } from './db'; export class SqliteIndexerSyncStorage extends IndexerSyncStorageBase { static readonly identifier = 'SqliteIndexerSyncStorage'; + override readonly commitsIndexAtomically = true; override connection = share(new NativeDBConnection(this.options)); @@ -32,6 +33,10 @@ export class SqliteIndexerSyncStorage extends IndexerSyncStorageBase { ); } + override async setDocIndexedClocks(clocks: DocIndexedClock[]): Promise { + await this.db.setDocIndexedClocks(clocks); + } + override async clearDocIndexedClock(docId: string): Promise { await this.db.clearDocIndexedClock(docId); } diff --git a/packages/common/nbstore/src/impls/sqlite/indexer/index.ts b/packages/common/nbstore/src/impls/sqlite/indexer/index.ts index 089dcfc331..4c6af67132 100644 --- a/packages/common/nbstore/src/impls/sqlite/indexer/index.ts +++ b/packages/common/nbstore/src/impls/sqlite/indexer/index.ts @@ -7,18 +7,21 @@ import type { AggregateOptions, AggregateResult, IndexerDocument, + IndexerSchema, Query, SearchOptions, SearchResult, } from '../../../storage'; import { IndexerStorageBase } from '../../../storage'; -import { IndexerSchema } from '../../../storage/indexer/schema'; import { fromPromise } from '../../../utils/from-promise'; import { backoffRetry, exhaustMapWithTrailing } from '../../idb/indexer/utils'; -import { NativeDBConnection, type SqliteNativeDBOptions } from '../db'; +import { + NativeDBConnection, + type NativeIndexQuery, + type NativeIndexSearchOptions, + type SqliteNativeDBOptions, +} from '../db'; import { createNode } from './node-builder'; -import { queryRaw } from './query'; -import { getText, tryParseArrayField } from './utils'; const SQLITE_INDEXER_VERSION_OFFSET = 1; @@ -43,33 +46,21 @@ export class SqliteIndexerStorage extends IndexerStorageBase { query: Query, options?: O ): Promise> { - const match = await queryRaw(this.connection, table, query); - - // Pagination const limit = options?.pagination?.limit ?? 10; const skip = options?.pagination?.skip ?? 0; - const ids = match.toArray(); - const pagedIds = ids.slice(skip, skip + limit); - - const nodes = []; - for (const id of pagedIds) { - const node = await createNode( - this.connection, - table, - id, - match.getScore(id), - options ?? {}, - query - ); - nodes.push(node); - } + const result = await this.connection.apis.indexSearch( + String(table), + toNativeQuery(query), + toNativeOptions(options, limit, skip) + ); + const nodes = result.hits.map(hit => createNode(hit, options ?? {})); return { pagination: { - count: ids.length, + count: result.total, limit, skip, - hasMore: ids.length > skip + limit, + hasMore: result.total > skip + limit, }, nodes, }; @@ -84,72 +75,49 @@ export class SqliteIndexerStorage extends IndexerStorageBase { field: keyof IndexerSchema[T], options?: O ): Promise> { - const match = await queryRaw(this.connection, table, query); - const ids = match.toArray(); - - const buckets: any[] = []; - - for (const id of ids) { - const text = await this.connection.apis.ftsGetDocument( - `${table}:${field as string}`, - id - ); - if (typeof text === 'string' && text.length > 0) { - let values: string[] = [text]; - const parsed = tryParseArrayField(text); - if (parsed) { - values = parsed; - } - - for (const val of values) { - let bucket = buckets.find(b => b.key === val); - if (!bucket) { - bucket = { key: val, count: 0, score: 0 }; - if (options?.hits) { - bucket.hits = { - pagination: { count: 0, limit: 0, skip: 0, hasMore: false }, - nodes: [], - }; - } - buckets.push(bucket); + const limit = options?.pagination?.limit ?? 10; + const skip = options?.pagination?.skip ?? 0; + const hitLimit = options?.hits?.pagination?.limit ?? 3; + const hitSkip = options?.hits?.pagination?.skip ?? 0; + const result = await this.connection.apis.indexAggregate( + String(table), + toNativeQuery(query), + String(field), + limit, + skip, + options?.hits + ? toNativeOptions(options.hits, hitLimit, hitSkip) + : undefined + ); + const hitsOptions = options?.hits; + const buckets = result.buckets.map(bucket => ({ + key: bucket.key, + count: bucket.count, + score: bucket.score, + ...(hitsOptions + ? { + hits: { + pagination: { + count: bucket.count, + limit: hitLimit, + skip: hitSkip, + hasMore: bucket.count > hitSkip + hitLimit, + }, + nodes: bucket.hits.map(hit => createNode(hit, hitsOptions)), + }, } - bucket.count++; - - if (options?.hits) { - const hitLimit = options.hits.pagination?.limit ?? 3; - if (bucket.hits.nodes.length < hitLimit) { - const node = await createNode( - this.connection, - table, - id, - match.getScore(id), - options.hits, - query - ); - bucket.hits.nodes.push(node); - bucket.hits.pagination.count++; - } - } - } - } else if (text != null && typeof text !== 'string') { - console.warn('[nbstore] invalid indexed aggregate type', { - table, - field: field as string, - id, - type: typeof text, - }); - } - } + : {}), + })); return { pagination: { - count: buckets.length, - limit: 0, - skip: 0, - hasMore: false, + count: result.total, + limit, + skip, + hasMore: result.total > skip + limit, }, buckets, - }; + } as AggregateResult; } search$>( @@ -190,38 +158,23 @@ export class SqliteIndexerStorage extends IndexerStorageBase { table: T, query: Query ): Promise { - const match = await queryRaw(this.connection, table, query); - const ids = match.toArray(); - for (const id of ids) { - await this.delete(table, id); - } + await this.connection.apis.indexDeleteByQuery( + String(table), + toNativeQuery(query) + ); } async insert( table: T, document: IndexerDocument ): Promise { - const schema = IndexerSchema[table]; - for (const [field, values] of document.fields) { - const fieldSchema = schema[field]; - // @ts-expect-error -- IndexerSchema uses runtime-keyed fields from each table schema. - const shouldIndex = fieldSchema.index !== false; - // @ts-expect-error -- IndexerSchema uses runtime-keyed fields from each table schema. - const shouldStore = fieldSchema.store !== false; - - if (!shouldStore && !shouldIndex) continue; - - const text = getText(values); - - if (typeof text === 'string') { - await this.connection.apis.ftsAddDocument( - `${table}:${field as string}`, - document.id, - text, - shouldIndex - ); - } - } + await this.connection.apis.indexUpsert(String(table), { + id: document.id, + fields: [...document.fields].map(([field, values]) => ({ + field: String(field), + values, + })), + }); this.tableUpdate$.next(table); } @@ -229,10 +182,7 @@ export class SqliteIndexerStorage extends IndexerStorageBase { table: T, id: string ): Promise { - const schema = IndexerSchema[table]; - for (const field of Object.keys(schema)) { - await this.connection.apis.ftsDeleteDocument(`${table}:${field}`, id); - } + await this.connection.apis.indexDelete(String(table), id); this.tableUpdate$.next(table); } @@ -249,13 +199,52 @@ export class SqliteIndexerStorage extends IndexerStorageBase { } async refreshIfNeed(): Promise { - await this.connection.apis.ftsFlushIndex(); + await this.connection.apis.indexFlush(); } async indexVersion(): Promise { return ( - (await this.connection.apis.ftsIndexVersion()) + + (await this.connection.apis.indexVersion()) + SQLITE_INDEXER_VERSION_OFFSET ); } } + +function toNativeQuery(query: Query): NativeIndexQuery { + switch (query.type) { + case 'match': + return { kind: 'match', field: String(query.field), value: query.match }; + case 'exists': + return { kind: 'exists', field: String(query.field) }; + case 'all': + return { kind: 'all' }; + case 'boolean': + return { + kind: 'boolean', + occur: query.occur, + clauses: query.queries.map(toNativeQuery), + }; + case 'boost': + return { + kind: 'boost', + boost: query.boost, + clauses: [toNativeQuery(query.query)], + }; + } +} + +function toNativeOptions( + options: SearchOptions | undefined, + limit: number, + offset: number +): NativeIndexSearchOptions { + const highlights = options?.highlights?.map(item => String(item.field)) ?? []; + return { + limit, + offset, + fields: [ + ...new Set([...(options?.fields?.map(String) ?? []), ...highlights]), + ], + highlights, + }; +} diff --git a/packages/common/nbstore/src/impls/sqlite/indexer/match.ts b/packages/common/nbstore/src/impls/sqlite/indexer/match.ts deleted file mode 100644 index 51fead106b..0000000000 --- a/packages/common/nbstore/src/impls/sqlite/indexer/match.ts +++ /dev/null @@ -1,105 +0,0 @@ -export class Match { - scores = new Map(); - /** - * id -> field -> index(multi value field) -> [start, end][] - */ - highlighters = new Map< - string, - Map> - >(); - - constructor() {} - - size() { - return this.scores.size; - } - - getScore(id: string) { - return this.scores.get(id) ?? 0; - } - - addScore(id: string, score: number) { - const currentScore = this.scores.get(id) || 0; - this.scores.set(id, currentScore + score); - } - - getHighlighters(id: string, field: string) { - return this.highlighters.get(id)?.get(field); - } - - addHighlighter( - id: string, - field: string, - index: number, - newRanges: [number, number][] - ) { - const fields = - this.highlighters.get(id) || - new Map>(); - const values = fields.get(field) || new Map(); - const ranges = values.get(index) || []; - ranges.push(...newRanges); - values.set(index, ranges); - fields.set(field, values); - this.highlighters.set(id, fields); - } - - and(other: Match) { - const newMatch = new Match(); - for (const [id, score] of this.scores) { - if (other.scores.has(id)) { - newMatch.addScore(id, score + (other.scores.get(id) ?? 0)); - newMatch.copyExtData(this, id); - newMatch.copyExtData(other, id); - } - } - return newMatch; - } - - or(other: Match) { - const newMatch = new Match(); - for (const [id, score] of this.scores) { - newMatch.addScore(id, score); - newMatch.copyExtData(this, id); - } - for (const [id, score] of other.scores) { - newMatch.addScore(id, score); - newMatch.copyExtData(other, id); - } - return newMatch; - } - - exclude(other: Match) { - const newMatch = new Match(); - for (const [id, score] of this.scores) { - if (!other.scores.has(id)) { - newMatch.addScore(id, score); - newMatch.copyExtData(this, id); - } - } - return newMatch; - } - - boost(boost: number) { - const newMatch = new Match(); - for (const [id, score] of this.scores) { - newMatch.addScore(id, score * boost); - newMatch.copyExtData(this, id); - } - return newMatch; - } - - toArray() { - return Array.from(this.scores.entries()) - .sort((a, b) => b[1] - a[1]) - .map(e => e[0]); - } - - private copyExtData(from: Match, id: string) { - for (const [field, values] of from.highlighters.get(id) ?? []) { - for (const [index, ranges] of values) { - this.addHighlighter(id, field, index, ranges); - } - } - } -} diff --git a/packages/common/nbstore/src/impls/sqlite/indexer/node-builder.spec.ts b/packages/common/nbstore/src/impls/sqlite/indexer/node-builder.spec.ts index 1f9dacee1f..9984b7e3f4 100644 --- a/packages/common/nbstore/src/impls/sqlite/indexer/node-builder.spec.ts +++ b/packages/common/nbstore/src/impls/sqlite/indexer/node-builder.spec.ts @@ -1,70 +1,35 @@ -import { describe, expect, it, vi } from 'vitest'; +import { describe, expect, it } from 'vitest'; -import type { NativeDBConnection } from '../db'; -import { SqliteIndexerStorage } from '.'; +import type { NativeIndexHit } from '../db'; import { createNode } from './node-builder'; -import { getText } from './utils'; -const query = { type: 'match', field: 'title', match: 'query' } as const; +function hit(fields: NativeIndexHit['fields']): NativeIndexHit { + return { id: 'doc-id', score: 1, fields, highlights: [] }; +} -const connectionWith = (value: unknown) => - ({ - apis: { - ftsGetDocument: vi.fn().mockResolvedValue(value), - ftsGetMatches: vi.fn().mockResolvedValue([]), - }, - }) as unknown as NativeDBConnection; - -describe('sqlite indexer node fields', () => { +describe('sqlite indexer native result mapping', () => { it.each([ - ['string', 'summary', 'summary'], - ['singleton array', ['one'], 'one'], + ['string', ['summary'], 'summary'], ['array', ['one', 'two'], ['one', 'two']], - ['serialized array', '["one","two"]', ['one', 'two']], - ['malformed array', '[not-json]', '[not-json]'], - ['null', null, ''], - ['missing', undefined, ''], - ['wrong type', 42, ''], - ])('isolates %s values', async (_, value, expected) => { - const node = await createNode( - connectionWith(Array.isArray(value) ? getText(value) : value), - 'doc', - 'doc-id', - 1, - { fields: ['title'] }, - query - ); - + ['missing', [], ''], + ])('maps %s stored values', (_, values, expected) => { + const node = createNode(hit([{ field: 'title', values }]), { + fields: ['title'], + }); expect(node.fields.title).toEqual(expected); }); - it('does not highlight non-string values', async () => { - const connection = connectionWith({ invalid: true }); - const node = await createNode( - connection, - 'doc', - 'doc-id', - 1, - { highlights: [{ field: 'title', before: '', end: '' }] }, - query - ); - - expect(node.highlights.title).toEqual([]); - expect(connection.apis.ftsGetMatches).not.toHaveBeenCalled(); - }); - - it('isolates wrong aggregate field types', async () => { - const connection = connectionWith(42); - connection.apis.ftsSearch = vi - .fn() - .mockResolvedValue([{ id: 'doc-id', score: 1, terms: [] }]); - const storage = Object.create( - SqliteIndexerStorage.prototype - ) as SqliteIndexerStorage; - Object.defineProperty(storage, 'connection', { value: connection }); - - await expect( - storage.aggregate('doc', query, 'title') - ).resolves.toMatchObject({ buckets: [] }); + it('formats native highlight spans', () => { + const native = hit([{ field: 'title', values: ['hello search'] }]); + native.highlights = [ + { + field: 'title', + values: [{ valueIndex: 0, spans: [{ start: 6, end: 12 }] }], + }, + ]; + const node = createNode(native, { + highlights: [{ field: 'title', before: '', end: '' }], + }); + expect(node.highlights.title).toEqual(['hello search']); }); }); diff --git a/packages/common/nbstore/src/impls/sqlite/indexer/node-builder.ts b/packages/common/nbstore/src/impls/sqlite/indexer/node-builder.ts index f01adb0e4a..4c1be45dd8 100644 --- a/packages/common/nbstore/src/impls/sqlite/indexer/node-builder.ts +++ b/packages/common/nbstore/src/impls/sqlite/indexer/node-builder.ts @@ -1,113 +1,44 @@ -import { type Query, type SearchOptions } from '../../../storage'; +import { type SearchOptions } from '../../../storage'; import { highlighter } from '../../idb/indexer/highlighter'; -import { type NativeDBConnection } from '../db'; -import { tryParseArrayField } from './utils'; +import type { NativeIndexHit } from '../db'; -export async function createNode( - connection: NativeDBConnection, - table: string, - id: string, - score: number, - options: SearchOptions, - query: Query -) { - const node: any = { id, score }; +export function createNode(hit: NativeIndexHit, options: SearchOptions) { + const node: any = { id: hit.id, score: hit.score }; + const fields = new Map(hit.fields.map(field => [field.field, field.values])); if (options.fields) { - const fields: Record = {}; - for (const field of options.fields) { - const text = await connection.apis.ftsGetDocument( - `${table}:${field as string}`, - id - ); - if (typeof text === 'string') { - const parsed = tryParseArrayField(text); - if (parsed) { - fields[field as string] = parsed; - } else { - fields[field as string] = text; - } - } else if (text == null) { - fields[field as string] = ''; - } else { - console.warn('[nbstore] invalid indexed field type', { - table, - field: field as string, - id, - type: typeof text, - }); - fields[field as string] = ''; - } - } - node.fields = fields; + node.fields = Object.fromEntries( + options.fields.map(field => { + const values = fields.get(String(field)) ?? []; + return [String(field), values.length > 1 ? values : (values[0] ?? '')]; + }) + ); } if (options.highlights) { - const highlights: Record = {}; - const queryStrings = extractQueryStrings(query); - - for (const h of options.highlights) { - const text = await connection.apis.ftsGetDocument( - `${table}:${h.field as string}`, - id - ); - if (typeof text === 'string' && text.length > 0) { - const queryString = Array.from(queryStrings).join(' '); - const matches = await connection.apis.ftsGetMatches( - `${table}:${h.field as string}`, - id, - queryString - ); - - if (matches.length > 0) { - const highlighted = highlighter( + const highlights = new Map( + hit.highlights.map(item => [item.field, item.values]) + ); + node.highlights = Object.fromEntries( + options.highlights.map(option => { + const field = String(option.field); + const source = fields.get(field) ?? []; + const fragments = (highlights.get(field) ?? []).flatMap(value => { + const text = source[value.valueIndex]; + if (!text) return []; + const fragment = highlighter( text, - h.before, - h.end, - matches.map(m => [m.start, m.end]), - { - maxPrefix: 20, - maxLength: 50, - } + option.before, + option.end, + value.spans.map(span => [span.start, span.end]), + { maxPrefix: 20, maxLength: 50 } ); - highlights[h.field as string] = highlighted ? [highlighted] : []; - } else { - highlights[h.field as string] = []; - } - } else { - if (text != null && typeof text !== 'string') { - console.warn('[nbstore] invalid indexed highlight type', { - table, - field: h.field as string, - id, - type: typeof text, - }); - } - highlights[h.field as string] = []; - } - } - node.highlights = highlights; + return fragment ? [fragment] : []; + }); + return [field, fragments]; + }) + ); } return node; } - -function extractQueryStrings(query: Query): Set { - const terms = new Set(); - if (query.type === 'match') { - terms.add(query.match); - } else if (query.type === 'boolean') { - for (const q of query.queries) { - const subTerms = extractQueryStrings(q); - for (const term of subTerms) { - terms.add(term); - } - } - } else if (query.type === 'boost') { - const subTerms = extractQueryStrings(query.query); - for (const term of subTerms) { - terms.add(term); - } - } - return terms; -} diff --git a/packages/common/nbstore/src/impls/sqlite/indexer/query.ts b/packages/common/nbstore/src/impls/sqlite/indexer/query.ts deleted file mode 100644 index d2f04e9e25..0000000000 --- a/packages/common/nbstore/src/impls/sqlite/indexer/query.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { IndexerSchema, type Query } from '../../../storage'; -import { type NativeDBConnection } from '../db'; -import { Match } from './match'; - -export async function queryRaw( - connection: NativeDBConnection, - table: string, - query: Query -): Promise { - if (query.type === 'match') { - const indexName = `${table}:${String(query.field)}`; - const hits = await connection.apis.ftsSearch(indexName, query.match); - const match = new Match(); - for (const hit of hits ?? []) { - match.addScore(hit.id, hit.score); - } - return match; - } else if (query.type === 'boolean') { - const matches: Match[] = []; - for (const q of query.queries) { - matches.push(await queryRaw(connection, table, q)); - } - - if (query.occur === 'must') { - if (matches.length === 0) return new Match(); - return matches.reduce((acc, m) => acc.and(m)); - } else if (query.occur === 'should') { - if (matches.length === 0) return new Match(); - return matches.reduce((acc, m) => acc.or(m)); - } else if (query.occur === 'must_not') { - const union = matches.reduce((acc, m) => acc.or(m), new Match()); - const all = await matchAll(connection, table); - return all.exclude(union); - } - } else if (query.type === 'all') { - return matchAll(connection, table); - } else if (query.type === 'boost') { - const match = await queryRaw(connection, table, query.query); - return match.boost(query.boost); - } else if (query.type === 'exists') { - const indexName = `${table}:${String(query.field)}`; - const hits = await connection.apis.ftsSearch(indexName, '*'); - const match = new Match(); - for (const hit of hits ?? []) { - match.addScore(hit.id, 1); - } - return match; - } - - return new Match(); -} - -export async function matchAll( - connection: NativeDBConnection, - table: string -): Promise { - const schema = IndexerSchema[table as keyof IndexerSchema]; - if (!schema) return new Match(); - - const match = new Match(); - for (const field of Object.keys(schema)) { - const indexName = `${table}:${field}`; - let hits = await connection.apis.ftsSearch(indexName, ''); - if (!hits || hits.length === 0) { - hits = await connection.apis.ftsSearch(indexName, '*'); - } - for (const hit of hits ?? []) { - match.addScore(hit.id, 1); - } - } - return match; -} diff --git a/packages/common/nbstore/src/impls/sqlite/indexer/utils.ts b/packages/common/nbstore/src/impls/sqlite/indexer/utils.ts deleted file mode 100644 index 2945f59ba1..0000000000 --- a/packages/common/nbstore/src/impls/sqlite/indexer/utils.ts +++ /dev/null @@ -1,23 +0,0 @@ -export function getText( - val: string | string[] | undefined -): string | undefined { - if (Array.isArray(val)) { - if (val.length === 1) { - return val[0]; - } - return JSON.stringify(val); - } - return val; -} - -export function tryParseArrayField(text: string): any[] | null { - if (text.startsWith('[') && text.endsWith(']')) { - try { - const parsed = JSON.parse(text); - if (Array.isArray(parsed)) { - return parsed; - } - } catch {} - } - return null; -} diff --git a/packages/common/nbstore/src/storage/indexer-sync.ts b/packages/common/nbstore/src/storage/indexer-sync.ts index 3cb8168294..e0a0dc6908 100644 --- a/packages/common/nbstore/src/storage/indexer-sync.ts +++ b/packages/common/nbstore/src/storage/indexer-sync.ts @@ -8,18 +8,24 @@ export interface DocIndexedClock extends DocClock { export interface IndexerSyncStorage extends Storage { readonly storageType: 'indexerSync'; + readonly commitsIndexAtomically: boolean; getDocIndexedClock(docId: string): Promise; setDocIndexedClock(docClock: DocIndexedClock): Promise; + setDocIndexedClocks(docClocks: DocIndexedClock[]): Promise; clearDocIndexedClock(docId: string): Promise; } export abstract class IndexerSyncStorageBase implements IndexerSyncStorage { readonly storageType = 'indexerSync'; + readonly commitsIndexAtomically: boolean = false; abstract connection: Connection; abstract getDocIndexedClock(docId: string): Promise; abstract setDocIndexedClock(docClock: DocIndexedClock): Promise; + async setDocIndexedClocks(docClocks: DocIndexedClock[]): Promise { + for (const clock of docClocks) await this.setDocIndexedClock(clock); + } abstract clearDocIndexedClock(docId: string): Promise; } diff --git a/packages/common/nbstore/src/sync/doc/peer.ts b/packages/common/nbstore/src/sync/doc/peer.ts index c44a67aede..687f8777c8 100644 --- a/packages/common/nbstore/src/sync/doc/peer.ts +++ b/packages/common/nbstore/src/sync/doc/peer.ts @@ -553,13 +553,16 @@ export class DocSyncPeer { this.actions.addDoc(docId); this.actions.updateRemoteClock(docId, remoteClock); - // schedule push job - this.schedule({ - type: 'save', - docId, - remoteClock: remoteClock, - update, - }); + if (isEmptyUpdate(update)) { + this.schedule({ type: 'pull', docId }); + } else { + this.schedule({ + type: 'save', + docId, + remoteClock: remoteClock, + update, + }); + } }, }; diff --git a/packages/common/nbstore/src/sync/indexer/index.ts b/packages/common/nbstore/src/sync/indexer/index.ts index 3c35baed1d..2a2858a8ee 100644 --- a/packages/common/nbstore/src/sync/indexer/index.ts +++ b/packages/common/nbstore/src/sync/indexer/index.ts @@ -349,9 +349,13 @@ export class IndexerSyncImpl implements IndexerSync { IndexerDocument.from(docId, { docId, title, + summary: existingDoc.summary, }) ); - this.status.docsInIndexer.set(docId, { title }); + this.status.docsInIndexer.set(docId, { + title, + summary: existingDoc.summary, + }); this.status.statusUpdatedSubject$.next(docId); } } else { @@ -461,9 +465,15 @@ export class IndexerSyncImpl implements IndexerSync { await this.indexer.update( 'doc', IndexerDocument.from(docId, { + docId, + title: existingDoc.title, summary: preview, }) ); + this.status.docsInIndexer.set(docId, { + title: existingDoc.title, + summary: preview, + }); } this.pendingIndexedClocks.set(docId, { @@ -496,18 +506,21 @@ export class IndexerSyncImpl implements IndexerSync { this.lastRefreshed + recommendRefreshInterval < Date.now(); const forceRefresh = recommendRefreshInterval <= 0; if (force || needRefresh || forceRefresh) { - await this.indexer.refreshIfNeed(); - await this.flushPendingIndexedClocks(); + if (this.indexerSync.commitsIndexAtomically) { + await this.flushPendingIndexedClocks(); + } else { + await this.indexer.refreshIfNeed(); + await this.flushPendingIndexedClocks(); + } this.lastRefreshed = Date.now(); } } private async flushPendingIndexedClocks() { if (this.pendingIndexedClocks.size === 0) return; - for (const [docId, clock] of this.pendingIndexedClocks) { - await this.indexerSync.setDocIndexedClock(clock); - this.pendingIndexedClocks.delete(docId); - } + const clocks = [...this.pendingIndexedClocks.values()]; + await this.indexerSync.setDocIndexedClocks(clocks); + for (const clock of clocks) this.pendingIndexedClocks.delete(clock.docId); } /** @@ -559,16 +572,20 @@ export class IndexerSyncImpl implements IndexerSync { pagination: { limit: Infinity, }, - fields: ['docId', 'title'], + fields: ['docId', 'title', 'summary'], } ); return new Map( docs.nodes.map(node => { const title = node.fields.title; + const summary = node.fields.summary; return [ node.id, - { title: typeof title === 'string' ? title : undefined }, + { + title: typeof title === 'string' ? title : undefined, + summary: typeof summary === 'string' ? summary : undefined, + }, ]; }) ); @@ -691,7 +708,10 @@ class IndexerSyncStatus { jobs = new AsyncPriorityQueue(); rootDoc = new YDoc({ guid: this.rootDocId }); rootDocReady = false; - docsInIndexer = new Map(); + docsInIndexer = new Map< + string, + { title: string | undefined; summary?: string } + >(); docsInRootDoc = new Map(); currentJob: string | null = null; errorMessage: string | null = null; diff --git a/packages/frontend/admin/src/config.json b/packages/frontend/admin/src/config.json index 18ca2f580a..f1f7c6cc96 100644 --- a/packages/frontend/admin/src/config.json +++ b/packages/frontend/admin/src/config.json @@ -95,6 +95,10 @@ "type": "Boolean", "desc": "Whether request abuse source facts should trust Cloudflare headers from the origin edge." }, + "signInRateLimit": { + "type": "Object", + "desc": "Limits for sign-in attempts shared through Redis by source IP and email. ttl is measured in milliseconds." + }, "inviteQuotaShadowMode": { "type": "Boolean", "desc": "Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions." @@ -298,13 +302,6 @@ "desc": "Whether allow guest users to create demo workspaces." } }, - "docService": { - "endpoint": { - "type": "String", - "desc": "The endpoint of the doc service.", - "env": "DOC_SERVICE_ENDPOINT" - } - }, "telemetry": { "allowedOrigin": { "type": "Array", @@ -406,12 +403,12 @@ }, "provider.type": { "type": "String", - "desc": "Indexer search service provider name", + "desc": "Indexer search provider. Self-hosted uses the embedded provider by default; remote providers require an endpoint.", "env": "AFFINE_INDEXER_SEARCH_PROVIDER" }, "provider.endpoint": { "type": "String", - "desc": "Indexer search service endpoint", + "desc": "Remote indexer endpoint. Not used by the embedded provider.", "env": "AFFINE_INDEXER_SEARCH_ENDPOINT" }, "provider.apiKey": { diff --git a/packages/frontend/admin/src/modules/settings/config.ts b/packages/frontend/admin/src/modules/settings/config.ts index 52de850934..b1197c15e4 100644 --- a/packages/frontend/admin/src/modules/settings/config.ts +++ b/packages/frontend/admin/src/modules/settings/config.ts @@ -163,6 +163,23 @@ export const KNOWN_CONFIG_GROUPS = [ }, ], } as ConfigGroup<'copilot'>, + { + name: 'Indexer', + module: 'indexer', + fields: [ + { + key: 'provider.type', + type: 'Enum', + options: ['embedded', 'manticoresearch', 'elasticsearch'], + desc: 'Search provider. Embedded keeps external credentials for later reuse.', + }, + 'provider.endpoint', + 'provider.apiKey', + 'provider.username', + 'provider.password', + 'autoIndex.batchSize', + ], + } as ConfigGroup<'indexer'>, ]; export const UNKNOWN_CONFIG_GROUPS = ALL_CONFIGURABLE_MODULES.filter( diff --git a/packages/frontend/admin/src/modules/settings/index.spec.tsx b/packages/frontend/admin/src/modules/settings/index.spec.tsx index 89e3119995..f1652ea270 100644 --- a/packages/frontend/admin/src/modules/settings/index.spec.tsx +++ b/packages/frontend/admin/src/modules/settings/index.spec.tsx @@ -24,13 +24,26 @@ vi.mock('../header', () => ({ vi.mock('./config-input-row', () => ({ ConfigRow: ({ field, + defaultValue, + onChange, onErrorChange, }: { field: string; + defaultValue?: unknown; + onChange?: (field: string, value: unknown) => void; onErrorChange?: (field: string, error?: string) => void; }) => (
-
{field}
+
{`${field}:${defaultValue}`}
+ +