build: allow node package depends on workspace packages (#11963)

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Added a unified CLI entry point for server operations and introduced a new CLI executable alias.
  - Centralized and simplified server startup, allowing selection between CLI and server modes.
  - Added static migration module aggregation for easier migration management.

- **Bug Fixes**
  - Improved platform-specific native module loading for better compatibility and reliability.

- **Refactor**
  - Streamlined server build, startup, and artifact management processes.
  - Reorganized and simplified workflow and configuration files for backend services.
  - Transitioned export styles and import mechanisms for native modules to enhance maintainability.

- **Chores**
  - Removed unused dependencies and configuration files.
  - Updated test cases to reflect refined server flavor logic.
  - Adjusted package and build configurations for consistency and clarity.

- **Revert**
  - Removed legacy scripts and loaders no longer needed after refactor.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/native/index.js

@@ -1,15 +1,14 @@
-import { createRequire } from 'node:module';
-
-const require = createRequire(import.meta.url);
-
 /** @type {import('.')} */
-const binding = require('./server-native.node');
+let binding;
+try {
+  binding = require('./server-native.node');
+} catch {
+  binding =
+    process.arch === 'arm64'
+      ? require('./server-native.arm64.node')
+      : process.arch === 'arm'
+        ? require('./server-native.armv7.node')
+        : require('./server-native.x64.node');
+}
 
-export const mergeUpdatesInApplyWay = binding.mergeUpdatesInApplyWay;
-export const verifyChallengeResponse = binding.verifyChallengeResponse;
-export const mintChallengeResponse = binding.mintChallengeResponse;
-export const getMime = binding.getMime;
-export const Tokenizer = binding.Tokenizer;
-export const fromModelName = binding.fromModelName;
-export const htmlSanitize = binding.htmlSanitize;
-export const parseDoc = binding.parseDoc;
+module.exports = binding;

packages/backend/native/package.json

@@ -4,7 +4,6 @@
   "engines": {
     "node": ">= 10.16.0 < 11 || >= 11.8.0"
   },
-  "type": "module",
   "main": "./index.js",
   "module": "./index.js",
   "types": "index.d.ts",

packages/backend/server/package.json

@@ -8,7 +8,7 @@
     "run-test": "./scripts/run-test.ts"
   },
   "scripts": {
-    "build": "tsc -b",
+    "build": "affine bundle -p @affine/server",
     "dev": "nodemon ./src/index.ts",
     "dev:mail": "email dev -d src/mails",
     "test": "ava --concurrency 1 --serial",
@@ -17,14 +17,16 @@
     "test:copilot:coverage": "c8 ava --timeout=5m \"src/__tests__/copilot-*.spec.ts\"",
     "e2e": "cross-env TEST_MODE=e2e ava --serial",
     "e2e:coverage": "cross-env TEST_MODE=e2e c8 ava --serial",
-    "data-migration": "cross-env NODE_ENV=development r ./src/data/index.ts",
+    "data-migration": "cross-env NODE_ENV=development SERVER_FLAVOR=script r ./src/index.ts",
     "init": "yarn prisma migrate dev && yarn data-migration run",
     "seed": "r ./src/seed/index.ts",
     "genconfig": "r ./scripts/genconfig.ts",
-    "predeploy": "yarn prisma migrate deploy && node --import ./scripts/register.js ./dist/data/index.js run",
+    "cli": "cross-env SERVER_FLAVOR=script node ./dist/main.js",
+    "predeploy": "yarn prisma migrate deploy && yarn cli run",
     "postinstall": "prisma generate"
   },
   "dependencies": {
+    "@affine/server-native": "workspace:*",
     "@ai-sdk/google": "^1.2.10",
     "@ai-sdk/openai": "^1.3.18",
     "@ai-sdk/perplexity": "^1.1.6",
@@ -72,6 +74,7 @@
     "ai": "^4.3.4",
     "bullmq": "^5.40.2",
     "cookie-parser": "^1.4.7",
+    "cross-env": "^7.0.3",
     "date-fns": "^4.0.0",
     "dotenv": "^16.4.7",
     "eventemitter2": "^6.4.9",
@@ -117,7 +120,6 @@
     "@affine-tools/cli": "workspace:*",
     "@affine-tools/utils": "workspace:*",
     "@affine/graphql": "workspace:*",
-    "@affine/server-native": "workspace:*",
     "@faker-js/faker": "^9.6.0",
     "@nestjs/testing": "patch:@nestjs/testing@npm%3A10.4.15#~/.yarn/patches/@nestjs-testing-npm-10.4.15-d591a1705a.patch",
     "@types/cookie-parser": "^1.4.8",
@@ -138,7 +140,6 @@
     "@types/supertest": "^6.0.2",
     "ava": "^6.2.0",
     "c8": "^10.1.3",
-    "cross-env": "^7.0.3",
     "nodemon": "^3.1.7",
     "react-email": "4.0.7",
     "sinon": "^20.0.0",

packages/backend/server/scripts/loader.js

@@ -1,11 +0,0 @@
-import { create, createEsmHooks } from 'ts-node';
-
-const service = create({
-  experimentalSpecifierResolution: 'node',
-  transpileOnly: true,
-  logError: true,
-  skipProject: true,
-});
-const hooks = createEsmHooks(service);
-
-export const resolve = hooks.resolve;

packages/backend/server/scripts/register.js

@@ -1,4 +0,0 @@
-import { register } from 'node:module';
-import { pathToFileURL } from 'node:url';
-
-register('./scripts/loader.js', pathToFileURL('./'));

packages/backend/server/src/__tests__/env.spec.ts

@@ -111,7 +111,7 @@ test('should tell flavors correctly', t => {
     sync: true,
     renderer: true,
     doc: true,
-    script: true,
+    script: false,
   });
 
   process.env.SERVER_FLAVOR = 'graphql';
@@ -122,6 +122,15 @@ test('should tell flavors correctly', t => {
     doc: false,
     script: false,
   });
+
+  process.env.SERVER_FLAVOR = 'script';
+  t.deepEqual(new Env().flavors, {
+    graphql: false,
+    sync: false,
+    renderer: false,
+    doc: false,
+    script: true,
+  });
 });
 
 test('should tell selfhosted correctly', t => {

packages/backend/server/src/cli.ts

@@ -1,16 +1,12 @@
 import { Logger } from '@nestjs/common';
 import { CommandFactory } from 'nest-commander';
 
-async function bootstrap() {
-  process.env.SERVER_FLAVOR = 'script';
+import { CliAppModule } from './data/app';
 
-  await import('../prelude');
-  const { CliAppModule } = await import('./app');
+export async function run() {
   await CommandFactory.run(CliAppModule, new Logger()).catch(e => {
     console.error(e);
     process.exit(1);
   });
   process.exit(0);
 }
-
-await bootstrap();

packages/backend/server/src/data/commands/create.ts

@@ -1,5 +1,5 @@
-import { writeFileSync } from 'node:fs';
-import { join } from 'node:path';
+import { appendFileSync, writeFileSync } from 'node:fs';
+import { join, parse } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { Logger } from '@nestjs/common';
@@ -45,15 +45,21 @@ export class CreateCommand extends CommandRunner {
 
     const timestamp = Date.now();
     const content = this.createScript(upperFirst(camelCase(name)) + timestamp);
-    const fileName = `${timestamp}-${kebabCase(name)}.ts`;
-    const filePath = join(
+    const migrationDir = join(
       fileURLToPath(import.meta.url),
-      '../../migrations',
-      fileName
+      '../../migrations'
     );
+    const fileName = `${timestamp}-${kebabCase(name)}.ts`;
+    const filePath = join(migrationDir, fileName);
 
     this.logger.log(`Creating ${fileName}...`);
     writeFileSync(filePath, content);
+    const indexFile = join(migrationDir, 'index.ts');
+    appendFileSync(
+      indexFile,
+      `export * from './${parse(fileName).name}';`,
+      'utf-8'
+    );
     this.logger.log(`Migration file created at ${filePath}`);
     this.logger.log('Done');
   }

packages/backend/server/src/data/commands/run.ts

@@ -1,49 +1,28 @@
-import { readdirSync } from 'node:fs';
-import { join } from 'node:path';
-import { fileURLToPath, pathToFileURL } from 'node:url';
-
 import { Logger } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
 import { PrismaClient } from '@prisma/client';
 import { once } from 'lodash-es';
 import { Command, CommandRunner } from 'nest-commander';
 
+import * as migrations from '../migrations';
+
 interface Migration {
-  file: string;
   name: string;
   always?: boolean;
   up: (db: PrismaClient, injector: ModuleRef) => Promise<void>;
   down: (db: PrismaClient, injector: ModuleRef) => Promise<void>;
 }
 
-export const collectMigrations = once(async () => {
-  const folder = join(fileURLToPath(import.meta.url), '../../migrations');
-
-  const migrationFiles = readdirSync(folder)
-    .filter(desc =>
-      desc.endsWith(import.meta.url.endsWith('.ts') ? '.ts' : '.js')
-    )
-    .map(desc => join(folder, desc));
-
-  migrationFiles.sort((a, b) => a.localeCompare(b));
-
-  const migrations: Migration[] = await Promise.all(
-    migrationFiles.map(async file => {
-      return import(pathToFileURL(file).href).then(mod => {
-        const migration = mod[Object.keys(mod)[0]];
-
-        return {
-          file,
-          name: migration.name,
-          always: migration.always,
-          up: migration.up,
-          down: migration.down,
-        };
-      });
-    })
-  );
-
-  return migrations;
+export const collectMigrations = once(() => {
+  return Object.values(migrations).map(migration => {
+    return {
+      name: migration.name,
+      // @ts-expect-error optional
+      always: migration.always,
+      up: migration.up,
+      down: migration.down,
+    };
+  }) as Migration[];
 });
 
 @Command({
@@ -60,7 +39,7 @@ export class RunCommand extends CommandRunner {
   }
 
   override async run(): Promise<void> {
-    const migrations = await collectMigrations();
+    const migrations = collectMigrations();
     const done: Migration[] = [];
     for (const migration of migrations) {
       const exists = await this.db.dataMigration.count({
@@ -85,7 +64,7 @@ export class RunCommand extends CommandRunner {
   }
 
   async runOne(name: string) {
-    const migrations = await collectMigrations();
+    const migrations = collectMigrations();
     const migration = migrations.find(m => m.name === name);
 
     if (!migration) {
@@ -162,7 +141,7 @@ export class RevertCommand extends CommandRunner {
       throw new Error('A migration name is required');
     }
 
-    const migrations = await collectMigrations();
+    const migrations = collectMigrations();
 
     const migration = migrations.find(m => m.name === name);
 

packages/backend/server/src/data/migrations/index.ts

@@ -0,0 +1,7 @@
+export * from './0001-refresh-features';
+export * from './1698398506533-guid';
+export * from './1703756315970-unamed-account';
+export * from './1721299086340-refresh-unnamed-user';
+export * from './1732861452428-migrate-invite-status';
+export * from './1733125339942-universal-subscription';
+export * from './1738590347632-feature-redundant';

packages/backend/server/src/env.ts

@@ -102,7 +102,8 @@ export class Env implements AppEnv {
       sync: this.isFlavor(Flavor.Sync),
       renderer: this.isFlavor(Flavor.Renderer),
       doc: this.isFlavor(Flavor.Doc),
-      script: this.isFlavor(Flavor.Script),
+      // Script in a special flavor, return true only when it is set explicitly
+      script: this.FLAVOR === Flavor.Script,
     };
   }
 

packages/backend/server/src/index.ts

@@ -1,20 +1,11 @@
 /// <reference types="./global.d.ts" />
 import './prelude';
 
-import { Logger } from '@nestjs/common';
+import { run as runCli } from './cli';
+import { run as runServer } from './server';
 
-import { createApp } from './app';
-import { Config, URLHelper } from './base';
-
-const app = await createApp();
-const config = app.get(Config);
-const url = app.get(URLHelper);
-const listeningHost = '0.0.0.0';
-
-await app.listen(config.server.port, listeningHost);
-
-const logger = new Logger('App');
-
-logger.log(`AFFiNE Server is running in [${env.DEPLOYMENT_TYPE}] mode`);
-logger.log(`Listening on http://${listeningHost}:${config.server.port}`);
-logger.log(`And the public server should be recognized as ${url.home}`);
+if (env.flavors.script) {
+  await runCli();
+} else {
+  await runServer();
+}

packages/backend/server/src/native.ts

@@ -1,17 +1,4 @@
-import { createRequire } from 'node:module';
-
-let serverNativeModule: typeof import('@affine/server-native');
-try {
-  serverNativeModule = await import('@affine/server-native');
-} catch {
-  const require = createRequire(import.meta.url);
-  serverNativeModule =
-    process.arch === 'arm64'
-      ? require('../server-native.arm64.node')
-      : process.arch === 'arm'
-        ? require('../server-native.armv7.node')
-        : require('../server-native.node');
-}
+import serverNativeModule from '@affine/server-native';
 
 export const mergeUpdatesInApplyWay = serverNativeModule.mergeUpdatesInApplyWay;
 

packages/backend/server/src/server.ts

@@ -9,6 +9,7 @@ import {
   CloudThrottlerGuard,
   Config,
   GlobalExceptionFilter,
+  URLHelper,
 } from './base';
 import { SocketIoAdapter } from './base/websocket';
 import { AuthGuard } from './core/auth';
@@ -16,7 +17,7 @@ import { serverTimingAndCache } from './middleware/timing';
 
 const OneMB = 1024 * 1024;
 
-export async function createApp() {
+export async function run() {
   const { AppModule } = await import('./app.module');
 
   const app = await NestFactory.create<NestExpressApplication>(AppModule, {
@@ -28,7 +29,8 @@ export async function createApp() {
 
   app.useBodyParser('raw', { limit: 100 * OneMB });
 
-  app.useLogger(app.get(AFFiNELogger));
+  const logger = app.get(AFFiNELogger);
+  app.useLogger(logger);
   const config = app.get(Config);
 
   if (config.server.path) {
@@ -57,5 +59,12 @@ export async function createApp() {
   const adapter = new SocketIoAdapter(app);
   app.useWebSocketAdapter(adapter);
 
-  return app;
+  const url = app.get(URLHelper);
+  const listeningHost = '0.0.0.0';
+
+  await app.listen(config.server.port, listeningHost);
+
+  logger.log(`AFFiNE Server is running in [${env.DEPLOYMENT_TYPE}] mode`);
+  logger.log(`Listening on http://${listeningHost}:${config.server.port}`);
+  logger.log(`And the public server should be recognized as ${url.home}`);
 }

packages/backend/server/tsconfig.json

@@ -12,9 +12,9 @@
   },
   "include": ["./src"],
   "references": [
+    { "path": "../native" },
     { "path": "../../../tools/cli" },
     { "path": "../../../tools/utils" },
-    { "path": "../../common/graphql" },
-    { "path": "../native" }
+    { "path": "../../common/graphql" }
   ]
 }