refactor(server): better selfhost deployment (#9036)

packages/backend/server/.dockerignore

@@ -0,0 +1,2 @@
+# required DATABASE_URL affects app start
+schema.prisma

packages/backend/server/package.json

@@ -165,6 +165,7 @@
       "*.gen.*"
     ],
     "env": {
+      "NODE_ENV": "development",
       "AFFINE_SERVER_EXTERNAL_URL": "http://localhost:8080",
       "TS_NODE_TRANSPILE_ONLY": true,
       "TS_NODE_PROJECT": "./tsconfig.json",

packages/backend/server/scripts/self-host-predeploy.js

@@ -3,61 +3,47 @@ import { generateKeyPairSync } from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 
-import { parse } from 'dotenv';
-
 const SELF_HOST_CONFIG_DIR = '/root/.affine/config';
-/**
- * @type {Array<{ from: string; to?: string, modifier?: (content: string): string }>}
- */
-const configFiles = [
-  { from: './.env.example', to: '.env' },
-  { from: './dist/config/affine.js', modifier: configCleaner },
-  { from: './dist/config/affine.env.js', modifier: configCleaner },
-];
 
-function configCleaner(content) {
+function generateConfigFile() {
+  const content = fs.readFileSync('./dist/config/affine.js', 'utf-8');
   return content.replace(
     /(^\/\/#.*$)|(^\/\/\s+TODO.*$)|("use\sstrict";?)|(^.*eslint-disable.*$)/gm,
     ''
   );
 }
 
+function generatePrivateKey() {
+  const key = generateKeyPairSync('ec', {
+    namedCurve: 'prime256v1',
+  }).privateKey.export({
+    type: 'sec1',
+    format: 'pem',
+  });
+
+  if (key instanceof Buffer) {
+    return key.toString('utf-8');
+  }
+
+  return key;
+}
+
+/**
+ * @type {Array<{ to: string; generator: () => string }>}
+ */
+const configFiles = [
+  { to: 'affine.js', generator: generateConfigFile },
+  { to: 'private.key', generator: generatePrivateKey },
+];
+
 function prepare() {
   fs.mkdirSync(SELF_HOST_CONFIG_DIR, { recursive: true });
 
-  for (const { from, to, modifier } of configFiles) {
-    const targetFileName = to ?? path.parse(from).base;
-    const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, targetFileName);
+  for (const { to, generator } of configFiles) {
+    const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, to);
     if (!fs.existsSync(targetFilePath)) {
       console.log(`creating config file [${targetFilePath}].`);
-      if (modifier) {
-        const content = fs.readFileSync(from, 'utf-8');
-        fs.writeFileSync(targetFilePath, modifier(content), 'utf-8');
-      } else {
-        fs.cpSync(from, targetFilePath, {
-          force: false,
-        });
-      }
-    }
-
-    // make the default .env
-    if (to === '.env') {
-      const dotenvFile = fs.readFileSync(targetFilePath, 'utf-8');
-      const envs = parse(dotenvFile);
-      // generate a new private key
-      if (!envs.AFFINE_PRIVATE_KEY) {
-        const privateKey = generateKeyPairSync('ec', {
-          namedCurve: 'prime256v1',
-        }).privateKey.export({
-          type: 'sec1',
-          format: 'pem',
-        });
-
-        fs.writeFileSync(
-          targetFilePath,
-          `AFFINE_PRIVATE_KEY=${privateKey}\n` + dotenvFile
-        );
-      }
+      fs.writeFileSync(targetFilePath, generator(), 'utf-8');
     }
   }
 }

packages/backend/server/src/config/affine.env.ts

@@ -12,7 +12,7 @@ AFFiNE.ENV_MAP = {
   MAILER_PASSWORD: 'mailer.auth.pass',
   MAILER_SENDER: 'mailer.from.address',
   MAILER_SECURE: ['mailer.secure', 'boolean'],
-  DATABASE_URL: 'database.datasourceUrl',
+  DATABASE_URL: 'prisma.datasourceUrl',
   OAUTH_GOOGLE_CLIENT_ID: 'plugins.oauth.providers.google.clientId',
   OAUTH_GOOGLE_CLIENT_SECRET: 'plugins.oauth.providers.google.clientSecret',
   OAUTH_GITHUB_CLIENT_ID: 'plugins.oauth.providers.github.clientId',

packages/backend/server/src/config/affine.self.ts

@@ -17,6 +17,11 @@
 // ====================================================================================
 const env = process.env;
 
+AFFiNE.serverName = AFFiNE.affine.canary
+  ? 'AFFiNE Canary Cloud'
+  : AFFiNE.affine.beta
+    ? 'AFFiNE Beta Cloud'
+    : 'AFFiNE Cloud';
 AFFiNE.metrics.enabled = !AFFiNE.node.test;
 
 if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {

packages/backend/server/src/config/affine.ts

@@ -8,22 +8,12 @@
 // Any changes in this file won't take effect before server restarted.
 //
 //
-// > Configurations merge order
-//   1. load environment variables (`.env` if provided, and from system)
-//   2. load `src/fundamentals/config/default.ts` for all default settings
-//   3. apply `./affine.ts` patches (this file)
-//   4. apply `./affine.env.ts` patches
-//
-//
 // ###############################################################
 // ##                       General settings                    ##
 // ###############################################################
 //
-// /* The unique identity of the server */
-// AFFiNE.serverId = 'some-randome-uuid';
-//
 // /* The name of AFFiNE Server, may show on the UI */
-// AFFiNE.serverName = 'Your Cool AFFiNE Selfhosted Cloud';
+AFFiNE.serverName = 'My Selfhosted AFFiNE Cloud';
 //
 // /* Whether the server is deployed behind a HTTPS proxied environment */
 AFFiNE.server.https = false;

packages/backend/server/src/core/config/resolver.ts

@@ -261,7 +261,7 @@ export class ServerServiceConfigResolver {
   }
 
   database(): ServerDatabaseConfig {
-    const url = new URL(this.config.database.datasourceUrl);
+    const url = new URL(this.config.prisma.datasourceUrl);
 
     return {
       host: url.hostname,

packages/backend/server/src/fundamentals/config/default.ts

@@ -14,7 +14,7 @@ import { readEnv } from './env';
 import { defaultStartupConfig } from './register';
 
 function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig {
-  const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'development', [
+  const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'production', [
     'development',
     'test',
     'production',

packages/backend/server/src/fundamentals/index.ts

@@ -29,7 +29,7 @@ export {
   mapSseError,
   OptionalModule,
 } from './nestjs';
-export type { PrismaTransaction } from './prisma';
+export { type PrismaTransaction } from './prisma';
 export * from './storage';
 export { type StorageProvider, StorageProviderFactory } from './storage';
 export { CloudThrottlerGuard, SkipThrottle, Throttle } from './throttler';

packages/backend/server/src/fundamentals/prisma/config.ts

@@ -8,10 +8,10 @@ interface PrismaStartupConfiguration extends Prisma.PrismaClientOptions {
 
 declare module '../config' {
   interface AppConfig {
-    database: ModuleConfig<PrismaStartupConfiguration>;
+    prisma: ModuleConfig<PrismaStartupConfiguration>;
   }
 }
 
-defineStartupConfig('database', {
+defineStartupConfig('prisma', {
   datasourceUrl: '',
 });

packages/backend/server/src/fundamentals/prisma/index.ts

@@ -14,7 +14,7 @@ const clientProvider: Provider = {
       return PrismaService.INSTANCE;
     }
 
-    return new PrismaService(config.database);
+    return new PrismaService(config.prisma);
   },
   inject: [Config],
 };

packages/backend/server/src/prelude.ts

@@ -1,6 +1,7 @@
 import 'reflect-metadata';
 
-import { cpSync } from 'node:fs';
+import { cpSync, existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
 import { join, parse } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 
@@ -10,42 +11,57 @@ import {
   applyEnvToConfig,
   getAFFiNEConfigModifier,
 } from './fundamentals/config';
-import { enablePlugin } from './plugins';
 
 const PROJECT_CONFIG_PATH = join(fileURLToPath(import.meta.url), '../config');
-async function loadRemote(remoteDir: string, file: string) {
-  let fileToLoad = join(PROJECT_CONFIG_PATH, file);
+const CUSTOM_CONFIG_PATH = `${homedir()}/.affine/config`;
 
-  if (PROJECT_CONFIG_PATH !== remoteDir) {
-    const remoteFile = join(remoteDir, file);
+async function loadConfig(configDir: string, file: string) {
+  let fileToLoad: string | undefined;
+
+  if (PROJECT_CONFIG_PATH !== configDir) {
+    const remoteFile = join(configDir, file);
     const remoteFileAtLocal = join(
       PROJECT_CONFIG_PATH,
       parse(file).name + '.remote.js'
     );
-    cpSync(remoteFile, remoteFileAtLocal, {
-      force: true,
-    });
-    fileToLoad = remoteFileAtLocal;
+    if (existsSync(remoteFile)) {
+      cpSync(remoteFile, remoteFileAtLocal, {
+        force: true,
+      });
+      fileToLoad = remoteFileAtLocal;
+    }
+  } else {
+    fileToLoad = join(PROJECT_CONFIG_PATH, file);
   }
 
-  await import(pathToFileURL(fileToLoad).href);
+  if (fileToLoad) {
+    await import(pathToFileURL(fileToLoad).href);
+  }
+}
+
+function loadPrivateKey() {
+  const file = join(CUSTOM_CONFIG_PATH, 'private.key');
+  if (!process.env.AFFINE_PRIVATE_KEY && existsSync(file)) {
+    const privateKey = readFileSync(file, 'utf-8');
+    process.env.AFFINE_PRIVATE_KEY = privateKey;
+  }
 }
 
 async function load() {
-  const AFFiNE_CONFIG_PATH =
-    process.env.AFFINE_CONFIG_PATH ?? PROJECT_CONFIG_PATH;
   // Initializing AFFiNE config
   //
   // 1. load dotenv file to `process.env`
   // load `.env` under pwd
   config();
+  // @deprecated removed
   // load `.env` under user config folder
   config({
-    path: join(AFFiNE_CONFIG_PATH, '.env'),
+    path: join(CUSTOM_CONFIG_PATH, '.env'),
   });
 
   // 2. generate AFFiNE default config and assign to `globalThis.AFFiNE`
   globalThis.AFFiNE = getAFFiNEConfigModifier();
+  const { enablePlugin } = await import('./plugins/registry');
   globalThis.AFFiNE.use = enablePlugin;
   globalThis.AFFiNE.plugins.use = enablePlugin;
 
@@ -53,24 +69,23 @@ async function load() {
   //   Modules may contribute to ENV_MAP, figure out a good way to involve them instead of hardcoding in `./config/affine.env`
   // 3. load env => config map to `globalThis.AFFiNE.ENV_MAP
   // load local env map as well in case there are new env added
-  await loadRemote(PROJECT_CONFIG_PATH, 'affine.env.js');
-  const projectEnvMap = AFFiNE.ENV_MAP;
-  await loadRemote(AFFiNE_CONFIG_PATH, 'affine.env.js');
-  const customEnvMap = AFFiNE.ENV_MAP;
-  AFFiNE.ENV_MAP = { ...projectEnvMap, ...customEnvMap };
+  await loadConfig(PROJECT_CONFIG_PATH, 'affine.env');
 
   // 4. load `config/affine` to patch custom configs
   // load local config as well in case there are new default configurations added
-  await loadRemote(PROJECT_CONFIG_PATH, 'affine.js');
-  await loadRemote(AFFiNE_CONFIG_PATH, 'affine.js');
+  await loadConfig(PROJECT_CONFIG_PATH, 'affine');
+  await loadConfig(CUSTOM_CONFIG_PATH, 'affine');
 
   // 5. load `config/affine.self` to patch custom configs
   // This is the file only take effect in [AFFiNE Cloud]
   if (!AFFiNE.isSelfhosted) {
-    await loadRemote(PROJECT_CONFIG_PATH, 'affine.self.js');
+    await loadConfig(PROJECT_CONFIG_PATH, 'affine.self');
   }
 
-  // 6. apply `process.env` map overriding to `globalThis.AFFiNE`
+  // 6. load `config/private.key` to patch app configs
+  loadPrivateKey();
+
+  // 7. apply `process.env` map overriding to `globalThis.AFFiNE`
   applyEnvToConfig(globalThis.AFFiNE);
 }