From 03dea53b30e50dbecabf49406685a64b9419ff13 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Thu, 29 Jun 2023 22:02:46 +0800 Subject: [PATCH] build: affine Node.js server charts (#2895) --- .github/deployment/front/affine.nginx.conf | 2 +- .github/helm/affine/.helmignore | 23 +++ .github/helm/affine/Chart.yaml | 6 + .github/helm/affine/charts/graphql/Chart.yaml | 6 + .../affine/charts/graphql/templates/NOTES.txt | 16 ++ .../charts/graphql/templates/_helpers.tpl | 132 ++++++++++++ .../charts/graphql/templates/deployment.yaml | 126 ++++++++++++ .../charts/graphql/templates/jwt-secret.yaml | 7 + .../charts/graphql/templates/migration.yaml | 34 +++ .../templates/oauth-github-secret.yaml | 10 + .../templates/oauth-google-secret.yaml | 10 + .../charts/graphql/templates/r2-secret.yaml | 9 + .../charts/graphql/templates/service.yaml | 15 ++ .../graphql/templates/serviceaccount.yaml | 12 ++ .../templates/tests/test-connection.yaml | 15 ++ .../helm/affine/charts/graphql/values.yaml | 69 +++++++ .github/helm/affine/charts/web/.helmignore | 23 +++ .github/helm/affine/charts/web/Chart.yaml | 6 + .../affine/charts/web/templates/NOTES.txt | 16 ++ .../affine/charts/web/templates/_helpers.tpl | 62 ++++++ .../charts/web/templates/deployment.yaml | 57 ++++++ .../affine/charts/web/templates/service.yaml | 15 ++ .../charts/web/templates/serviceaccount.yaml | 12 ++ .../web/templates/tests/test-connection.yaml | 15 ++ .github/helm/affine/charts/web/values.yaml | 37 ++++ .github/helm/affine/templates/_helpers.tpl | 62 ++++++ .github/helm/affine/templates/ingress.yaml | 64 ++++++ .github/helm/affine/values.yaml | 17 ++ .github/workflows/build.yml | 37 +++- apps/server/package.json | 2 +- apps/server/schema.prisma | 3 +- apps/server/src/app.controller.ts | 13 ++ apps/server/src/app.ts | 2 + apps/server/src/config/def.ts | 27 +-- apps/server/src/config/default.ts | 193 +++++++++++------- apps/server/src/config/env.ts | 10 +- apps/server/src/index.ts | 8 +- apps/server/src/modules/storage/s3.ts | 9 +- .../src/modules/storage/storage.service.ts | 4 +- .../src/modules/workspaces/controller.ts | 15 +- .../server/src/modules/workspaces/resolver.ts | 7 +- apps/server/src/prelude.ts | 8 +- apps/server/src/storage/index.ts | 20 +- 43 files changed, 1112 insertions(+), 124 deletions(-) create mode 100644 .github/helm/affine/.helmignore create mode 100644 .github/helm/affine/Chart.yaml create mode 100644 .github/helm/affine/charts/graphql/Chart.yaml create mode 100644 .github/helm/affine/charts/graphql/templates/NOTES.txt create mode 100644 .github/helm/affine/charts/graphql/templates/_helpers.tpl create mode 100644 .github/helm/affine/charts/graphql/templates/deployment.yaml create mode 100644 .github/helm/affine/charts/graphql/templates/jwt-secret.yaml create mode 100644 .github/helm/affine/charts/graphql/templates/migration.yaml create mode 100644 .github/helm/affine/charts/graphql/templates/oauth-github-secret.yaml create mode 100644 .github/helm/affine/charts/graphql/templates/oauth-google-secret.yaml create mode 100644 .github/helm/affine/charts/graphql/templates/r2-secret.yaml create mode 100644 .github/helm/affine/charts/graphql/templates/service.yaml create mode 100644 .github/helm/affine/charts/graphql/templates/serviceaccount.yaml create mode 100644 .github/helm/affine/charts/graphql/templates/tests/test-connection.yaml create mode 100644 .github/helm/affine/charts/graphql/values.yaml create mode 100644 .github/helm/affine/charts/web/.helmignore create mode 100644 .github/helm/affine/charts/web/Chart.yaml create mode 100644 .github/helm/affine/charts/web/templates/NOTES.txt create mode 100644 .github/helm/affine/charts/web/templates/_helpers.tpl create mode 100644 .github/helm/affine/charts/web/templates/deployment.yaml create mode 100644 .github/helm/affine/charts/web/templates/service.yaml create mode 100644 .github/helm/affine/charts/web/templates/serviceaccount.yaml create mode 100644 .github/helm/affine/charts/web/templates/tests/test-connection.yaml create mode 100644 .github/helm/affine/charts/web/values.yaml create mode 100644 .github/helm/affine/templates/_helpers.tpl create mode 100644 .github/helm/affine/templates/ingress.yaml create mode 100644 .github/helm/affine/values.yaml create mode 100644 apps/server/src/app.controller.ts diff --git a/.github/deployment/front/affine.nginx.conf b/.github/deployment/front/affine.nginx.conf index 3a5f6e6217..f5a2ec0c15 100644 --- a/.github/deployment/front/affine.nginx.conf +++ b/.github/deployment/front/affine.nginx.conf @@ -3,7 +3,7 @@ server { root /app/dist; location / { - try_files $uri $uri/index.html $uri.html =404;; + try_files $uri $uri/index.html $uri.html =404; } error_page 404 /404.html; diff --git a/.github/helm/affine/.helmignore b/.github/helm/affine/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/.github/helm/affine/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/.github/helm/affine/Chart.yaml b/.github/helm/affine/Chart.yaml new file mode 100644 index 0000000000..d2e490ed86 --- /dev/null +++ b/.github/helm/affine/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: affine +description: AFFiNE cloud chart +type: application +version: 0.0.0 +appVersion: '0.7.0-canary.18' diff --git a/.github/helm/affine/charts/graphql/Chart.yaml b/.github/helm/affine/charts/graphql/Chart.yaml new file mode 100644 index 0000000000..acb3e90b7b --- /dev/null +++ b/.github/helm/affine/charts/graphql/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: graphql +description: AFFiNE GraphQL server +type: application +version: 0.0.0 +appVersion: '0.7.0-canary.18' diff --git a/.github/helm/affine/charts/graphql/templates/NOTES.txt b/.github/helm/affine/charts/graphql/templates/NOTES.txt new file mode 100644 index 0000000000..2cec931c10 --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/NOTES.txt @@ -0,0 +1,16 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "graphql.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "graphql.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "graphql.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "graphql.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/_helpers.tpl b/.github/helm/affine/charts/graphql/templates/_helpers.tpl new file mode 100644 index 0000000000..fc5851ede6 --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/_helpers.tpl @@ -0,0 +1,132 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "graphql.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "graphql.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "graphql.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "graphql.labels" -}} +helm.sh/chart: {{ include "graphql.chart" . }} +{{ include "graphql.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "graphql.selectorLabels" -}} +app.kubernetes.io/name: {{ include "graphql.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "graphql.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "graphql.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "jwt.key" -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.app.jwt.secretName -}} +{{- if and $secret $secret.data.private -}} +{{/* + Reusing existing secret data +*/}} +key: {{ $secret.data.private }} +{{- else -}} +{{/* + Generate new data +*/}} +key: {{ genPrivateKey "ecdsa" | b64enc }} +{{- end -}} +{{- end -}} + +{{- define "objectStorage.r2" -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.app.objectStorage.r2.secretName -}} +{{- if $secret -}} +{{/* + Reusing existing secret data +*/}} +accountId: {{ $secret.data.accountId }} +accessKeyId: {{ $secret.data.accessKeyId }} +secretAccessKey: {{ $secret.data.secretAccessKey }} +bucket: {{ $secret.data.bucket }} +{{- else -}} +{{/* + Generate new data +*/}} +accountId: {{ .Values.app.objectStorage.r2.accountId | b64enc }} +accessKeyId: {{ .Values.app.objectStorage.r2.accessKeyId | b64enc }} +secretAccessKey: {{ .Values.app.objectStorage.r2.secretAccessKey | b64enc }} +bucket: {{ .Values.app.objectStorage.r2.bucket | b64enc }} +{{- end -}} +{{- end -}} + +{{- define "objectStorage.oauth.google" -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.app.oauth.google.secretName -}} +{{- if $secret -}} +{{/* + Reusing existing secret data +*/}} +clientId: {{ $secret.data.clientId }} +clientSecret: {{ $secret.data.clientSecret }} +{{- else -}} +{{/* + Generate new data +*/}} +clientId: "{{ .Values.app.oauth.google.clientId | b64enc }}" +clientSecret: "{{ .Values.app.oauth.google.clientSecret | b64enc }}" +{{- end -}} +{{- end -}} + +{{- define "objectStorage.oauth.github" -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.app.oauth.github.secretName -}} +{{- if $secret -}} +{{/* + Reusing existing secret data +*/}} +clientId: {{ $secret.data.clientId }} +clientSecret: {{ $secret.data.clientSecret }} +{{- else -}} +{{/* + Generate new data +*/}} +clientId: "{{ .Values.app.oauth.github.clientId | b64enc }}" +clientSecret: "{{ .Values.app.oauth.github.clientSecret | b64enc }}" +{{- end -}} +{{- end -}} diff --git a/.github/helm/affine/charts/graphql/templates/deployment.yaml b/.github/helm/affine/charts/graphql/templates/deployment.yaml new file mode 100644 index 0000000000..5c8762ee54 --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/deployment.yaml @@ -0,0 +1,126 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "graphql.fullname" . }} + labels: + {{- include "graphql.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "graphql.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "graphql.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "graphql.serviceAccountName" . }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: AUTH_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: "{{ .Values.app.jwt.secretName }}" + key: key + - name: NODE_ENV + value: "{{ .Values.env }}" + - name: DATABSE_PASSWORD + valueFrom: + secretKeyRef: + name: pg-postgresql + key: postgres-password + - name: DATABASE_URL + value: postgres://{{ .Values.database.user }}:$(DATABSE_PASSWORD)@{{ .Values.database.url }}:{{ .Values.database.port }}/{{ .Values.database.name }} + - name: AFFINE_SERVER_PORT + value: "{{ .Values.service.port }}" + - name: AFFINE_SERVER_SUB_PATH + value: "{{ .Values.app.path }}" + - name: AFFINE_SERVER_HOST + value: "{{ .Values.app.host }}" + - name: ENABLE_R2_OBJECT_STORAGE + value: "{{ .Values.app.objectStorage.r2.enabled }}" + {{ if .Values.app.objectStorage.r2.enabled }} + - name: R2_OBJECT_STORAGE_ACCOUNT_ID + valueFrom: + secretKeyRef: + name: "{{ .Values.app.objectStorage.r2.secretName }}" + key: accountId + - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: "{{ .Values.app.objectStorage.r2.secretName }}" + key: accessKeyId + - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: "{{ .Values.app.objectStorage.r2.secretName }}" + key: secretAccessKey + - name: R2_OBJECT_STORAGE_BUCKET + valueFrom: + secretKeyRef: + name: "{{ .Values.app.objectStorage.r2.secretName }}" + key: bucket + {{ end }} + {{ if .Values.app.oauth.google.enabled }} + - name: OAUTH_GOOGLE_CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{ .Values.app.oauth.google.secretName }}" + key: clientId + - name: OAUTH_GOOGLE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{ .Values.app.oauth.google.secretName }}" + key: clientSecret + {{ end }} + {{ if .Values.app.oauth.github.enabled }} + - name: OAUTH_GITHUB_CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{ .Values.app.oauth.github.secretName }}" + key: clientId + - name: OAUTH_GITHUB_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{ .Values.app.oauth.github.secretName }}" + key: clientSecret + {{ end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/jwt-secret.yaml b/.github/helm/affine/charts/graphql/templates/jwt-secret.yaml new file mode 100644 index 0000000000..74d11faecb --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/jwt-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "{{ .Values.app.jwt.secretName }}" +type: Opaque +data: +{{- ( include "jwt.key" . ) | indent 2 -}} diff --git a/.github/helm/affine/charts/graphql/templates/migration.yaml b/.github/helm/affine/charts/graphql/templates/migration.yaml new file mode 100644 index 0000000000..4aa885f8eb --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/migration.yaml @@ -0,0 +1,34 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "graphql.fullname" . }}-database-migration + labels: + {{- include "graphql.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-1" + "helm.sh/hook-delete-policy": before-hook-creation + +spec: + template: + spec: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + command: ["yarn", "prisma", "migrate", "deploy"] + env: + - name: NODE_ENV + value: "{{ .Values.env }}" + - name: DATABSE_PASSWORD + valueFrom: + secretKeyRef: + name: pg-postgresql + key: postgres-password + - name: DATABASE_URL + value: postgres://{{ .Values.database.user }}:$(DATABSE_PASSWORD)@{{ .Values.database.url }}:{{ .Values.database.port }}/{{ .Values.database.name }} + resources: + requests: + cpu: '100m' + memory: '200Mi' + restartPolicy: Never + backoffLimit: 1 diff --git a/.github/helm/affine/charts/graphql/templates/oauth-github-secret.yaml b/.github/helm/affine/charts/graphql/templates/oauth-github-secret.yaml new file mode 100644 index 0000000000..3c9f6f7e7a --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/oauth-github-secret.yaml @@ -0,0 +1,10 @@ +{{- if .Values.app.oauth.github.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ .Values.app.oauth.github.secretName }}" +type: Opaque +data: +{{- ( include "objectStorage.oauth.github" . ) | indent 2 -}} + +{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/oauth-google-secret.yaml b/.github/helm/affine/charts/graphql/templates/oauth-google-secret.yaml new file mode 100644 index 0000000000..fd939fa63a --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/oauth-google-secret.yaml @@ -0,0 +1,10 @@ +{{- if .Values.app.oauth.google.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ .Values.app.oauth.google.secretName }}" +type: Opaque +data: +{{- ( include "objectStorage.oauth.google" . ) | indent 2 -}} + +{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/r2-secret.yaml b/.github/helm/affine/charts/graphql/templates/r2-secret.yaml new file mode 100644 index 0000000000..50b7336dc7 --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/r2-secret.yaml @@ -0,0 +1,9 @@ +{{- if .Values.app.objectStorage.r2.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ .Values.app.objectStorage.r2.secretName }}" +type: Opaque +data: + {{- ( include "objectStorage.r2" . ) | indent 2 -}} +{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/service.yaml b/.github/helm/affine/charts/graphql/templates/service.yaml new file mode 100644 index 0000000000..729d7246b2 --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "graphql.fullname" . }} + labels: + {{- include "graphql.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "graphql.selectorLabels" . | nindent 4 }} diff --git a/.github/helm/affine/charts/graphql/templates/serviceaccount.yaml b/.github/helm/affine/charts/graphql/templates/serviceaccount.yaml new file mode 100644 index 0000000000..14dac586bf --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "graphql.serviceAccountName" . }} + labels: + {{- include "graphql.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/tests/test-connection.yaml b/.github/helm/affine/charts/graphql/templates/tests/test-connection.yaml new file mode 100644 index 0000000000..6e46e00d71 --- /dev/null +++ b/.github/helm/affine/charts/graphql/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "graphql.fullname" . }}-test-connection" + labels: + {{- include "graphql.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "graphql.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/.github/helm/affine/charts/graphql/values.yaml b/.github/helm/affine/charts/graphql/values.yaml new file mode 100644 index 0000000000..c99969e6ea --- /dev/null +++ b/.github/helm/affine/charts/graphql/values.yaml @@ -0,0 +1,69 @@ +replicaCount: 1 +image: + repository: ghcr.io/toeverything/affine-graphql + pullPolicy: IfNotPresent + tag: '' + +imagePullSecrets: [] +nameOverride: '' +fullnameOverride: '' +# map to NODE_ENV environment variable +env: 'production' +database: + user: 'postgres' + url: 'pg-postgresql' + port: '5432' + name: 'affine' +app: + # AFFINE_SERVER_SUB_PATH + path: '' + # AFFINE_SERVER_HOST + host: '0.0.0.0' + jwt: + secretName: jwt-private-key + # base64 encoded ecdsa private key + privateKey: '' + objectStorage: + r2: + enabled: false + secretName: r2 + accountId: '' + accessKeyId: '' + secretAccessKey: '' + bucket: '' + oauth: + google: + enabled: false + secretName: oauth-google + clientId: '' + clientSecret: '' + github: + enabled: false + secretName: oauth-github + clientId: '' + clientSecret: '' + +serviceAccount: + create: true + annotations: {} + name: 'affine-graphql' + +podAnnotations: {} + +podSecurityContext: + fsGroup: 2000 + +resources: + limits: + cpu: '2000m' + memory: 4Gi + requests: + cpu: '1000m' + memory: 2Gi + +probe: + initialDelaySeconds: 20 + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/.github/helm/affine/charts/web/.helmignore b/.github/helm/affine/charts/web/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/.github/helm/affine/charts/web/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/.github/helm/affine/charts/web/Chart.yaml b/.github/helm/affine/charts/web/Chart.yaml new file mode 100644 index 0000000000..8e41632a80 --- /dev/null +++ b/.github/helm/affine/charts/web/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: web +description: A Helm chart for Kubernetes +type: application +version: 0.0.0 +appVersion: "0.7.0-canary.18" diff --git a/.github/helm/affine/charts/web/templates/NOTES.txt b/.github/helm/affine/charts/web/templates/NOTES.txt new file mode 100644 index 0000000000..98c2294816 --- /dev/null +++ b/.github/helm/affine/charts/web/templates/NOTES.txt @@ -0,0 +1,16 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "web.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "web.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "web.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "web.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/.github/helm/affine/charts/web/templates/_helpers.tpl b/.github/helm/affine/charts/web/templates/_helpers.tpl new file mode 100644 index 0000000000..dff203a79d --- /dev/null +++ b/.github/helm/affine/charts/web/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "web.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "web.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "web.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "web.labels" -}} +helm.sh/chart: {{ include "web.chart" . }} +{{ include "web.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "web.selectorLabels" -}} +app.kubernetes.io/name: {{ include "web.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "web.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "web.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/web/templates/deployment.yaml b/.github/helm/affine/charts/web/templates/deployment.yaml new file mode 100644 index 0000000000..5c43bda2c2 --- /dev/null +++ b/.github/helm/affine/charts/web/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "web.fullname" . }} + labels: + {{- include "web.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "web.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "web.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "web.serviceAccountName" . }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/.github/helm/affine/charts/web/templates/service.yaml b/.github/helm/affine/charts/web/templates/service.yaml new file mode 100644 index 0000000000..589411f9a5 --- /dev/null +++ b/.github/helm/affine/charts/web/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "web.fullname" . }} + labels: + {{- include "web.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "web.selectorLabels" . | nindent 4 }} diff --git a/.github/helm/affine/charts/web/templates/serviceaccount.yaml b/.github/helm/affine/charts/web/templates/serviceaccount.yaml new file mode 100644 index 0000000000..f9f19b3454 --- /dev/null +++ b/.github/helm/affine/charts/web/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "web.serviceAccountName" . }} + labels: + {{- include "web.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/web/templates/tests/test-connection.yaml b/.github/helm/affine/charts/web/templates/tests/test-connection.yaml new file mode 100644 index 0000000000..f36083416f --- /dev/null +++ b/.github/helm/affine/charts/web/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "web.fullname" . }}-test-connection" + labels: + {{- include "web.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "web.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/.github/helm/affine/charts/web/values.yaml b/.github/helm/affine/charts/web/values.yaml new file mode 100644 index 0000000000..40bbffdfa6 --- /dev/null +++ b/.github/helm/affine/charts/web/values.yaml @@ -0,0 +1,37 @@ +replicaCount: 1 + +image: + repository: ghcr.io/toeverything/affine-front + pullPolicy: IfNotPresent + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "affine-web" + +podAnnotations: {} + +podSecurityContext: + fsGroup: 2000 + +resources: + limits: + cpu: '500m' + memory: 2Gi + requests: + cpu: '500m' + memory: 2Gi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +probe: + initialDelaySeconds: 1 diff --git a/.github/helm/affine/templates/_helpers.tpl b/.github/helm/affine/templates/_helpers.tpl new file mode 100644 index 0000000000..005cc9bf56 --- /dev/null +++ b/.github/helm/affine/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "affine.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "affine.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "affine.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "affine.labels" -}} +helm.sh/chart: {{ include "affine.chart" . }} +{{ include "affine.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "affine.selectorLabels" -}} +app.kubernetes.io/name: {{ include "affine.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "affine.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "affine.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/.github/helm/affine/templates/ingress.yaml b/.github/helm/affine/templates/ingress.yaml new file mode 100644 index 0000000000..9df2d3254a --- /dev/null +++ b/.github/helm/affine/templates/ingress.yaml @@ -0,0 +1,64 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "affine.fullname" . -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "affine.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + - host: "{{ .Values.ingress.host }}" + http: + paths: + - path: /graphql + pathType: Prefix + backend: + service: + name: affine-graphql + port: + number: {{ .Values.graphql.service.port }} + - path: /api + pathType: Prefix + backend: + service: + name: affine-graphql + port: + number: {{ .Values.graphql.service.port }} + - path: / + pathType: Prefix + backend: + service: + name: affine-web + port: + number: {{ .Values.web.service.port }} + +{{- end }} diff --git a/.github/helm/affine/values.yaml b/.github/helm/affine/values.yaml new file mode 100644 index 0000000000..0bf6aaa9b5 --- /dev/null +++ b/.github/helm/affine/values.yaml @@ -0,0 +1,17 @@ +ingress: + enabled: false + className: '' + annotations: + kubernetes.io/ingress.class: nginx + host: affine.pro + tls: [] + +graphql: + service: + type: ClusterIP + port: 3000 + +web: + service: + type: ClusterIP + port: 8080 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 538b55c6c9..24b9bce14a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,6 +53,12 @@ jobs: run: yarn prettier . --ignore-unknown --cache --check - name: Run circular run: yarn circular + - name: Upload server dist + uses: actions/upload-artifact@v3 + with: + name: server-dist + path: ./apps/server/dist + if-no-files-found: error build-docs: name: Build Docs @@ -172,6 +178,12 @@ jobs: env: CARGO_TARGET_DIR: '${{ github.workspace }}/target' DATABASE_URL: postgresql://affine:affine@localhost:5432/affine + - name: Upload storage.node + uses: actions/upload-artifact@v3 + with: + name: storage.node + path: ./packages/storage/storage.node + if-no-files-found: error - name: Upload server test coverage results uses: codecov/codecov-action@v3 with: @@ -439,7 +451,9 @@ jobs: if: github.ref == 'refs/heads/master' name: Build Docker needs: - - build-web-desktop + - lint + - desktop-test + - server-test runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -448,6 +462,16 @@ jobs: with: name: next-js-static path: ./apps/web/out + - name: Download server dist + uses: actions/download-artifact@v3 + with: + name: server-dist + path: ./apps/server/dist + - name: Download storage.node + uses: actions/download-artifact@v3 + with: + name: storage.node + path: ./apps/server - name: Setup Git short hash run: | echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" @@ -473,14 +497,21 @@ jobs: file: .github/deployment/front/Dockerfile tags: ghcr.io/toeverything/affine-front:${{ env.GIT_SHORT_HASH }},ghcr.io/toeverything/affine-front:latest + # setup node without cache configuration + # Prisma cache is not compatible with docker build cache - name: Setup Node.js - uses: ./.github/actions/setup-node + uses: actions/setup-node@v3 with: - package-install: false + node-version-file: '.nvmrc' + registry-url: https://npm.pkg.github.com + scope: '@toeverything' - name: Install Node.js dependencies run: yarn workspaces focus @affine/server --production + - name: Generate Prisma client + run: yarn workspace @affine/server prisma generate + - name: Build graphql Dockerfile uses: docker/build-push-action@v4 with: diff --git a/apps/server/package.json b/apps/server/package.json index 198c9da07a..8fead7baed 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -15,7 +15,6 @@ "postinstall": "prisma generate" }, "dependencies": { - "@affine/storage": "workspace:*", "@apollo/server": "^4.7.4", "@auth/prisma-adapter": "^1.0.0", "@aws-sdk/client-s3": "^3.359.0", @@ -41,6 +40,7 @@ "rxjs": "^7.8.1" }, "devDependencies": { + "@affine/storage": "workspace:*", "@napi-rs/image": "^1.6.1", "@nestjs/testing": "^10.0.3", "@types/express": "^4.17.17", diff --git a/apps/server/schema.prisma b/apps/server/schema.prisma index 285bc01094..3d8a6557fc 100644 --- a/apps/server/schema.prisma +++ b/apps/server/schema.prisma @@ -1,5 +1,6 @@ generator client { - provider = "prisma-client-js" + provider = "prisma-client-js" + binaryTargets = ["native", "debian-openssl-3.0.x"] } datasource db { diff --git a/apps/server/src/app.controller.ts b/apps/server/src/app.controller.ts new file mode 100644 index 0000000000..613ee59f48 --- /dev/null +++ b/apps/server/src/app.controller.ts @@ -0,0 +1,13 @@ +import { Controller, Get } from '@nestjs/common'; + +import pkg from '../package.json' assert { type: 'json' }; + +@Controller('/') +export class AppController { + @Get() + hello() { + return { + message: `AFFiNE GraphQL server: ${pkg.version}`, + }; + } +} diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 011b51dcaf..224bbe40f7 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; +import { AppController } from './app.controller'; import { ConfigModule } from './config'; import { GqlModule } from './graphql.module'; import { BusinessModules } from './modules'; @@ -14,5 +15,6 @@ import { StorageModule } from './storage'; StorageModule.forRoot(), ...BusinessModules, ], + controllers: [AppController], }) export class AppModule {} diff --git a/apps/server/src/config/def.ts b/apps/server/src/config/def.ts index a85e813083..fbb6785468 100644 --- a/apps/server/src/config/def.ts +++ b/apps/server/src/config/def.ts @@ -106,7 +106,7 @@ export interface AFFiNEConfig { /** * which port the server will listen on * - * @default 3000 + * @default 3010 * @env AFFINE_SERVER_PORT */ port: number; @@ -153,23 +153,13 @@ export interface AFFiNEConfig { /** * whether use remote object storage */ - enable: boolean; - /** - * used to store all uploaded builds and analysis reports - * - * the concrete type definition is not given here because different storage providers introduce - * significant differences in configuration - * - * @example - * { - * provider: 'aws', - * region: 'eu-west-1', - * aws_access_key_id: '', - * aws_secret_access_key: '', - * // other aws storage config... - * } - */ - config: Record; + r2: { + enabled: boolean; + accountId: string; + bucket: string; + accessKeyId: string; + secretAccessKey: string; + }; /** * Only used when `enable` is `false` */ @@ -224,6 +214,7 @@ export interface AFFiNEConfig { Record< ExternalAccount, { + enabled: boolean; clientId: string; clientSecret: string; /** diff --git a/apps/server/src/config/default.ts b/apps/server/src/config/default.ts index 8c2ebdc943..0c65dc4adf 100644 --- a/apps/server/src/config/default.ts +++ b/apps/server/src/config/default.ts @@ -1,5 +1,6 @@ /// +import { createPrivateKey, createPublicKey } from 'node:crypto'; import { homedir } from 'node:os'; import { join } from 'node:path'; @@ -7,82 +8,130 @@ import parse from 'parse-duration'; import pkg from '../../package.json' assert { type: 'json' }; import type { AFFiNEConfig } from './def'; +import { applyEnvToConfig } from './env'; // Don't use this in production -export const examplePublicKey = `-----BEGIN PUBLIC KEY----- -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEnxM+GhB6eNKPmTP6uH5Gpr+bmQ87 -hHGeOiCsay0w/aPwMqzAOKkZGqX+HZ9BNGy/yiXmnscey5b2vOTzxtRvxA== ------END PUBLIC KEY-----`; +export const examplePrivateKey = `-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIEtyAJLIULkphVhqXqxk4Nr8Ggty3XLwUJWBxzAWCWTMoAoGCCqGSM49 +AwEHoUQDQgAEF3U/0wIeJ3jRKXeFKqQyBKlr9F7xaAUScRrAuSP33rajm3cdfihI +3JvMxVNsS2lE8PSGQrvDrJZaDo0L+Lq9Gg== +-----END EC PRIVATE KEY-----`; -// Don't use this in production -export const examplePrivateKey = `-----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgWOog5SFXs1Vjh/WP -QCYPQKgf/jsNmWsvD+jYSn6mi3yhRANCAASfEz4aEHp40o+ZM/q4fkamv5uZDzuE -cZ46IKxrLTD9o/AyrMA4qRkapf4dn0E0bL/KJeaexx7Llva85PPG1G/E ------END PRIVATE KEY-----`; +const jwtKeyPair = (function () { + const AUTH_PRIVATE_KEY = process.env.AUTH_PRIVATE_KEY ?? examplePrivateKey; + const privateKey = createPrivateKey({ + key: Buffer.from(AUTH_PRIVATE_KEY), + format: 'pem', + type: 'sec1', + }) + .export({ + format: 'pem', + type: 'pkcs8', + }) + .toString('utf8'); + const publicKey = createPublicKey({ + key: Buffer.from(AUTH_PRIVATE_KEY), + format: 'pem', + type: 'spki', + }) + .export({ + format: 'pem', + type: 'spki', + }) + .toString('utf8'); -export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => ({ - serverId: 'affine-nestjs-server', - version: pkg.version, - ENV_MAP: {}, - env: process.env.NODE_ENV ?? 'development', - get prod() { - return this.env === 'production'; - }, - get dev() { - return this.env === 'development'; - }, - get test() { - return this.env === 'test'; - }, - get deploy() { - return !this.dev && !this.test; - }, - https: false, - host: 'localhost', - port: 3010, - path: '', - get origin() { - return this.dev - ? 'http://localhost:8080' - : `${this.https ? 'https' : 'http'}://${this.host}${ - this.host === 'localhost' ? `:${this.port}` : '' - }`; - }, - get baseUrl() { - return `${this.origin}${this.path}`; - }, - db: { - url: '', - }, - graphql: { - buildSchemaOptions: { - numberScalarMode: 'integer', + return { + publicKey, + privateKey, + }; +})(); + +export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => { + const defaultConfig = { + serverId: 'affine-nestjs-server', + version: pkg.version, + ENV_MAP: { + AFFINE_SERVER_PORT: 'port', + AFFINE_SERVER_HOST: 'host', + AFFINE_SERVER_SUB_PATH: 'path', + DATABASE_URL: 'db.url', + AUTH_PRIVATE_KEY: 'auth.privateKey', + ENABLE_R2_OBJECT_STORAGE: 'objectStorage.r2.enabled', + R2_OBJECT_STORAGE_ACCOUNT_ID: 'objectStorage.r2.accountId', + R2_OBJECT_STORAGE_ACCESS_KEY_ID: 'objectStorage.r2.accessKeyId', + R2_OBJECT_STORAGE_SECRET_ACCESS_KEY: 'objectStorage.r2.secretAccessKey', + R2_OBJECT_STORAGE_BUCKET: 'objectStorage.r2.bucket', + OAUTH_GOOGLE_CLIENT_ID: 'auth.oauthProviders.google.clientId', + OAUTH_GOOGLE_CLIENT_SECRET: 'auth.oauthProviders.google.clientSecret', + OAUTH_GITHUB_CLIENT_ID: 'auth.oauthProviders.github.clientId', + OAUTH_GITHUB_CLIENT_SECRET: 'auth.oauthProviders.github.clientSecret', + } satisfies AFFiNEConfig['ENV_MAP'], + env: process.env.NODE_ENV ?? 'development', + get prod() { + return this.env === 'production'; }, - introspection: true, - playground: true, - debug: true, - }, - auth: { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - accessTokenExpiresIn: parse('1h')! / 1000, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - refreshTokenExpiresIn: parse('7d')! / 1000, - leeway: 60, - publicKey: examplePublicKey, - privateKey: examplePrivateKey, - enableSignup: true, - enableOauth: false, - nextAuthSecret: '', - oauthProviders: {}, - }, - objectStorage: { - enable: false, - config: {}, - fs: { - path: join(homedir(), '.affine-storage'), + get dev() { + return this.env === 'development'; }, - }, -}); + get test() { + return this.env === 'test'; + }, + get deploy() { + return !this.dev && !this.test; + }, + https: false, + host: 'localhost', + port: 3010, + path: '', + db: { + url: '', + }, + get origin() { + return this.dev + ? 'http://localhost:8080' + : `${this.https ? 'https' : 'http'}://${this.host}${ + this.host === 'localhost' ? `:${this.port}` : '' + }`; + }, + get baseUrl() { + return `${this.origin}${this.path}`; + }, + graphql: { + buildSchemaOptions: { + numberScalarMode: 'integer', + }, + introspection: true, + playground: true, + debug: true, + }, + auth: { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + accessTokenExpiresIn: parse('1h')! / 1000, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + refreshTokenExpiresIn: parse('7d')! / 1000, + leeway: 60, + privateKey: jwtKeyPair.privateKey, + publicKey: jwtKeyPair.publicKey, + enableSignup: true, + enableOauth: false, + nextAuthSecret: '', + oauthProviders: {}, + }, + objectStorage: { + r2: { + enabled: false, + bucket: '', + accountId: '', + accessKeyId: '', + secretAccessKey: '', + }, + fs: { + path: join(homedir(), '.affine-storage'), + }, + }, + } as const; -export { registerEnvs } from './env'; + applyEnvToConfig(defaultConfig); + + return defaultConfig; +}; diff --git a/apps/server/src/config/env.ts b/apps/server/src/config/env.ts index bfcd2c7d5c..f68dfedd5f 100644 --- a/apps/server/src/config/env.ts +++ b/apps/server/src/config/env.ts @@ -1,17 +1,17 @@ import { set } from 'lodash-es'; -import { parseEnvValue } from './def'; +import { type AFFiNEConfig, parseEnvValue } from './def'; -export function registerEnvs() { - for (const env in globalThis.AFFiNE.ENV_MAP) { - const config = globalThis.AFFiNE.ENV_MAP[env]; +export function applyEnvToConfig(rawConfig: AFFiNEConfig) { + for (const env in rawConfig.ENV_MAP) { + const config = rawConfig.ENV_MAP[env]; const [path, value] = typeof config === 'string' ? [config, process.env[env]] : [config[0], parseEnvValue(process.env[env], config[1])]; if (typeof value !== 'undefined') { - set(globalThis.AFFiNE, path, process.env[env]); + set(rawConfig, path, value); } } } diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index 8ebb9ef5c1..22846e24b1 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -27,12 +27,12 @@ app.use( }) ); -const host = process.env.HOST ?? 'localhost'; -const port = process.env.PORT ?? 3010; - const config = app.get(Config); -if (!config.objectStorage.enable) { +const host = config.host ?? 'localhost'; +const port = config.port ?? 3010; + +if (!config.objectStorage.r2.enabled) { app.use('/assets', staticMiddleware(config.objectStorage.fs.path)); } diff --git a/apps/server/src/modules/storage/s3.ts b/apps/server/src/modules/storage/s3.ts index 9f4a5cec54..c849c5aa26 100644 --- a/apps/server/src/modules/storage/s3.ts +++ b/apps/server/src/modules/storage/s3.ts @@ -8,7 +8,14 @@ export const S3_SERVICE = Symbol('S3_SERVICE'); export const S3: FactoryProvider = { provide: S3_SERVICE, useFactory: (config: Config) => { - const s3 = new S3Client(config.objectStorage.config); + const s3 = new S3Client({ + region: 'auto', + endpoint: `https://${config.objectStorage.r2.accountId}.r2.cloudflarestorage.com`, + credentials: { + accessKeyId: config.objectStorage.r2.accessKeyId, + secretAccessKey: config.objectStorage.r2.secretAccessKey, + }, + }); return s3; }, inject: [Config], diff --git a/apps/server/src/modules/storage/storage.service.ts b/apps/server/src/modules/storage/storage.service.ts index 8bda04b6e1..f923ca8352 100644 --- a/apps/server/src/modules/storage/storage.service.ts +++ b/apps/server/src/modules/storage/storage.service.ts @@ -15,11 +15,11 @@ export class StorageService { ) {} async uploadFile(key: string, file: FileUpload) { - if (this.config.objectStorage.enable) { + if (this.config.objectStorage.r2.enabled) { await this.s3.send( new PutObjectCommand({ Body: file.createReadStream(), - Bucket: this.config.objectStorage.config.bucket, + Bucket: this.config.objectStorage.r2.bucket, Key: key, }) ); diff --git a/apps/server/src/modules/workspaces/controller.ts b/apps/server/src/modules/workspaces/controller.ts index e98f75795f..b29a4d412f 100644 --- a/apps/server/src/modules/workspaces/controller.ts +++ b/apps/server/src/modules/workspaces/controller.ts @@ -1,10 +1,19 @@ -import { Storage } from '@affine/storage'; -import { Controller, Get, NotFoundException, Param, Res } from '@nestjs/common'; +import type { Storage } from '@affine/storage'; +import { + Controller, + Get, + Inject, + NotFoundException, + Param, + Res, +} from '@nestjs/common'; import type { Response } from 'express'; +import { StorageProvide } from '../../storage'; + @Controller('/api/workspaces') export class WorkspacesController { - constructor(private readonly storage: Storage) {} + constructor(@Inject(StorageProvide) private readonly storage: Storage) {} @Get('/:id/blobs/:name') async blob( diff --git a/apps/server/src/modules/workspaces/resolver.ts b/apps/server/src/modules/workspaces/resolver.ts index bbd4465739..98d1e1bacf 100644 --- a/apps/server/src/modules/workspaces/resolver.ts +++ b/apps/server/src/modules/workspaces/resolver.ts @@ -1,5 +1,5 @@ -import { Storage } from '@affine/storage'; -import { ForbiddenException, NotFoundException } from '@nestjs/common'; +import type { Storage } from '@affine/storage'; +import { ForbiddenException, Inject, NotFoundException } from '@nestjs/common'; import { Args, Field, @@ -21,6 +21,7 @@ import type { User, Workspace } from '@prisma/client'; import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs'; import { PrismaService } from '../../prisma'; +import { StorageProvide } from '../../storage'; import type { FileUpload } from '../../types'; import { Auth, CurrentUser } from '../auth'; import { UserType } from '../users/resolver'; @@ -60,7 +61,7 @@ export class WorkspaceResolver { constructor( private readonly prisma: PrismaService, private readonly permissionProvider: PermissionService, - private readonly storage: Storage + @Inject(StorageProvide) private readonly storage: Storage ) {} @ResolveField(() => Permission, { diff --git a/apps/server/src/prelude.ts b/apps/server/src/prelude.ts index edafc03e54..10c53c1bed 100644 --- a/apps/server/src/prelude.ts +++ b/apps/server/src/prelude.ts @@ -1,12 +1,6 @@ import 'reflect-metadata'; import 'dotenv/config'; -import { getDefaultAFFiNEConfig, registerEnvs } from './config/default'; +import { getDefaultAFFiNEConfig } from './config/default'; globalThis.AFFiNE = getDefaultAFFiNEConfig(); - -globalThis.AFFiNE.ENV_MAP = { - DATABASE_URL: 'db.url', -}; - -registerEnvs(); diff --git a/apps/server/src/storage/index.ts b/apps/server/src/storage/index.ts index e62e5fcfd1..22525bf0cd 100644 --- a/apps/server/src/storage/index.ts +++ b/apps/server/src/storage/index.ts @@ -1,14 +1,28 @@ -import { Storage } from '@affine/storage'; +import { createRequire } from 'node:module'; + +import type { Storage } from '@affine/storage'; import { type DynamicModule, type FactoryProvider } from '@nestjs/common'; import { Config } from '../config'; +export const StorageProvide = Symbol('Storage'); + +const require = createRequire(import.meta.url); + export class StorageModule { static forRoot(): DynamicModule { const storageProvider: FactoryProvider = { - provide: Storage, + provide: StorageProvide, useFactory: async (config: Config) => { - return Storage.connect(config.db.url); + let StorageFactory: typeof Storage; + try { + // dev mode + StorageFactory = (await import('@affine/storage')).Storage; + } catch { + // In docker + StorageFactory = require('../../storage.node').Storage; + } + return StorageFactory.connect(config.db.url); }, inject: [Config], };