Files
AFFiNE-Mirror/packages/backend/server/src/app.module.ts
fengmk2 a1bcf77447 feat(server): add cloud indexer with Elasticsearch and Manticoresearch providers (#11835)
close CLOUD-137

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Introduced advanced workspace-scoped search and aggregation capabilities with support for complex queries, highlights, and pagination.
  - Added pluggable search providers: Elasticsearch and Manticoresearch.
  - New GraphQL queries, schema types, and resolver support for search and aggregation.
  - Enhanced configuration options for search providers in self-hosted and cloud deployments.
  - Added Docker Compose services and environment variables for Elasticsearch and Manticoresearch.
  - Integrated indexer service into deployment and CI workflows.

- **Bug Fixes**
  - Improved error handling with new user-friendly error messages for search provider and indexer issues.

- **Documentation**
  - Updated configuration examples and environment variable references for indexer and search providers.

- **Tests**
  - Added extensive end-to-end and provider-specific tests covering indexing, searching, aggregation, deletion, and error cases.
  - Included snapshot tests and test fixtures for search providers.

- **Chores**
  - Updated deployment scripts, Helm charts, and Kubernetes manifests to include indexer-related environment variables and secrets.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2025-05-14 14:52:41 +00:00

197 lines
5.7 KiB
TypeScript

import { DynamicModule, ExecutionContext } from '@nestjs/common';
import { ScheduleModule } from '@nestjs/schedule';
import { ClsPluginTransactional } from '@nestjs-cls/transactional';
import { TransactionalAdapterPrisma } from '@nestjs-cls/transactional-adapter-prisma';
import { PrismaClient } from '@prisma/client';
import { Request, Response } from 'express';
import { ClsModule } from 'nestjs-cls';
import { AppController } from './app.controller';
import {
getRequestIdFromHost,
getRequestIdFromRequest,
ScannerModule,
} from './base';
import { CacheModule } from './base/cache';
import { ConfigModule } from './base/config';
import { ErrorModule } from './base/error';
import { EventModule } from './base/event';
import { GqlModule } from './base/graphql';
import { HelpersModule } from './base/helpers';
import { JobModule } from './base/job';
import { LoggerModule } from './base/logger';
import { MetricsModule } from './base/metrics';
import { MutexModule } from './base/mutex';
import { PrismaModule } from './base/prisma';
import { RedisModule } from './base/redis';
import { StorageProviderModule } from './base/storage';
import { RateLimiterModule } from './base/throttler';
import { WebSocketModule } from './base/websocket';
import { AuthModule } from './core/auth';
import { ServerConfigModule, ServerConfigResolverModule } from './core/config';
import { DocStorageModule } from './core/doc';
import { DocRendererModule } from './core/doc-renderer';
import { DocServiceModule } from './core/doc-service';
import { FeatureModule } from './core/features';
import { MailModule } from './core/mail';
import { NotificationModule } from './core/notification';
import { PermissionModule } from './core/permission';
import { QuotaModule } from './core/quota';
import { SelfhostModule } from './core/selfhost';
import { StorageModule } from './core/storage';
import { SyncModule } from './core/sync';
import { UserModule } from './core/user';
import { VersionModule } from './core/version';
import { WorkspaceModule } from './core/workspaces';
import { Env } from './env';
import { ModelsModule } from './models';
import { CaptchaModule } from './plugins/captcha';
import { CopilotModule } from './plugins/copilot';
import { CustomerIoModule } from './plugins/customerio';
import { GCloudModule } from './plugins/gcloud';
import { IndexerModule } from './plugins/indexer';
import { LicenseModule } from './plugins/license';
import { OAuthModule } from './plugins/oauth';
import { PaymentModule } from './plugins/payment';
import { WorkerModule } from './plugins/worker';
export const FunctionalityModules = [
ClsModule.forRoot({
global: true,
// for http / graphql request
middleware: {
mount: true,
generateId: true,
idGenerator(req: Request) {
// make every request has a unique id to tracing
return getRequestIdFromRequest(req, 'http');
},
setup(cls, _req, res: Response) {
res.setHeader('X-Request-Id', cls.getId());
},
},
// for websocket connection
// https://papooch.github.io/nestjs-cls/considerations/compatibility#websockets
interceptor: {
mount: true,
generateId: true,
idGenerator(context: ExecutionContext) {
// make every request has a unique id to tracing
return getRequestIdFromHost(context);
},
},
plugins: [
// https://papooch.github.io/nestjs-cls/plugins/available-plugins/transactional/prisma-adapter
new ClsPluginTransactional({
adapter: new TransactionalAdapterPrisma({
prismaInjectionToken: PrismaClient,
}),
}),
],
}),
LoggerModule,
ScannerModule,
PrismaModule,
EventModule,
ConfigModule,
RedisModule,
CacheModule,
MutexModule,
MetricsModule,
RateLimiterModule,
StorageProviderModule,
HelpersModule,
ErrorModule,
WebSocketModule,
JobModule.forRoot(),
ModelsModule,
];
export class AppModuleBuilder {
private readonly modules: AFFiNEModule[] = [];
use(...modules: AFFiNEModule[]): this {
modules.forEach(m => {
this.modules.push(m);
});
return this;
}
useIf(predicator: () => boolean, ...modules: AFFiNEModule[]): this {
if (predicator()) {
this.use(...modules);
}
return this;
}
compile(): DynamicModule {
class AppModule {}
return {
module: AppModule,
imports: this.modules,
controllers: [AppController],
};
}
}
export function buildAppModule(env: Env) {
const factor = new AppModuleBuilder();
factor
// basic
.use(...FunctionalityModules)
// enable schedule module on graphql server and doc service
.useIf(
() => env.flavors.graphql || env.flavors.doc,
ScheduleModule.forRoot(),
IndexerModule
)
// auth
.use(UserModule, AuthModule, PermissionModule)
// business modules
.use(
ServerConfigModule,
FeatureModule,
QuotaModule,
DocStorageModule,
NotificationModule,
MailModule
)
// renderer server only
.useIf(() => env.flavors.renderer, DocRendererModule)
// sync server only
.useIf(() => env.flavors.sync, SyncModule)
// graphql server only
.useIf(
() => env.flavors.graphql,
GqlModule,
VersionModule,
StorageModule,
ServerConfigResolverModule,
WorkspaceModule,
LicenseModule,
PaymentModule,
CopilotModule,
CaptchaModule,
OAuthModule,
CustomerIoModule
)
// doc service only
.useIf(() => env.flavors.doc, DocServiceModule)
// self hosted server only
.useIf(() => env.dev || env.selfhosted, WorkerModule, SelfhostModule)
// gcloud
.useIf(() => env.gcp, GCloudModule);
return factor.compile();
}
export const AppModule = buildAppModule(env);