refactor(server): indexer & worker & sync perf (#15504)

packages/backend/server/src/base/job/queue/executor.ts

@@ -2,7 +2,7 @@ import { getQueueToken, getSharedConfigToken } from '@nestjs/bullmq';
 import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
 import { Job, Queue as Bullmq, Worker, WorkerOptions } from 'bullmq';
-import { difference, merge } from 'lodash-es';
+import { merge } from 'lodash-es';
 import { CLS_ID, ClsServiceManager } from 'nestjs-cls';
 
 import { Config } from '../../config';
@@ -10,7 +10,8 @@ import { OnEvent } from '../../event';
 import { metrics, wrapCallMetric } from '../../metrics';
 import { QueueRedis } from '../../redis';
 import { genRequestId } from '../../utils';
-import { JOB_SIGNAL, namespace, Queue, QUEUES } from './def';
+import { JOB_SIGNAL, namespace, Queue } from './def';
+import { queuesForRole } from './owner';
 import { JobHandlerScanner } from './scanner';
 
 @Injectable()
@@ -27,18 +28,7 @@ export class JobExecutor implements OnModuleDestroy {
 
   @OnEvent('config.init')
   async onConfigInit() {
-    const queues = env.flavors.graphql
-      ? difference(QUEUES, [Queue.DOC, Queue.INDEXER])
-      : [];
-
-    // Enable doc/indexer queues in both doc and front service.
-    if (env.flavors.doc || env.flavors.front) {
-      queues.push(Queue.DOC);
-      // NOTE(@fengmk2): Once the index task cannot be processed in time, it needs to be separated from the doc service and deployed independently.
-      queues.push(Queue.INDEXER);
-    }
-
-    await this.startWorkers(queues);
+    await this.startWorkers(queuesForRole(env.role));
   }
 
   @OnEvent('config.changed')

packages/backend/server/src/base/job/queue/index.ts

@@ -55,3 +55,4 @@ export class JobModule {
 
 export { JobQueue };
 export { JOB_SIGNAL, OnJob } from './def';
+export { queuesForRole, WORKER_QUEUES } from './owner';

packages/backend/server/src/base/job/queue/owner.ts

@@ -0,0 +1,24 @@
+import { ServerRole } from '../../../env';
+import { Queue, QUEUES } from './def';
+
+export const WORKER_QUEUES = [
+  Queue.DOC,
+  Queue.INDEXER,
+  Queue.BACKENDRUNTIME,
+] as const;
+
+export function queuesForRole(role: ServerRole | undefined): Queue[] {
+  switch (role) {
+    case ServerRole.AllInOne:
+      return [...QUEUES];
+    case ServerRole.Api:
+      return QUEUES.filter(
+        queue => !(WORKER_QUEUES as readonly Queue[]).includes(queue)
+      );
+    case ServerRole.Worker:
+      return [...WORKER_QUEUES];
+    case ServerRole.Frontend:
+    case undefined:
+      return [];
+  }
+}