fix(server): wrong usage of optl (#6714)

packages/backend/server/package.json

@@ -8,15 +8,16 @@
     "run-test": "./scripts/run-test.ts"
   },
   "scripts": {
+    "run:script": "node --import ./scripts/register.js",
     "build": "tsc",
-    "start": "node --loader ts-node/esm/transpile-only.mjs ./src/index.ts",
+    "start": "yarn run:script ./src/index.ts",
     "dev": "nodemon ./src/index.ts",
     "test": "ava --concurrency 1 --serial",
     "test:coverage": "c8 ava --concurrency 1 --serial",
     "postinstall": "prisma generate",
-    "data-migration": "node --loader ts-node/esm/transpile-only.mjs ./src/data/index.ts",
-    "predeploy": "yarn prisma migrate deploy && node --import ./scripts/register.js ./dist/data/index.js run",
-    "predeploy:ts": "yarn prisma migrate deploy && node --loader ts-node/esm/transpile-only.mjs ./src/data/index.ts run"
+    "data-migration": "yarn run:script ./src/data/index.ts",
+    "predeploy": "yarn prisma migrate deploy && yarn run:script ./dist/data/index.js run",
+    "db:upgrade": "yarn prisma migrate deploy && yarn data-migration run"
   },
   "dependencies": {
     "@apollo/server": "^4.10.2",
@@ -42,8 +43,8 @@
     "@node-rs/jsonwebtoken": "^0.5.2",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/core": "^1.25.0",
-    "@opentelemetry/exporter-prometheus": "^0.52.0",
-    "@opentelemetry/exporter-zipkin": "^1.25.0",
+    "@opentelemetry/exporter-metrics-otlp-proto": "^0.52.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.52.0",
     "@opentelemetry/host-metrics": "^0.35.2",
     "@opentelemetry/instrumentation": "^0.52.0",
     "@opentelemetry/instrumentation-graphql": "^0.42.0",
@@ -165,9 +166,8 @@
     "exec": "node",
     "script": "./src/index.ts",
     "nodeArgs": [
-      "--loader",
-      "ts-node/esm.mjs",
-      "--es-module-specifier-resolution=node"
+      "--import",
+      "./scripts/register.js"
     ],
     "ignore": [
       "**/__tests__/**",

packages/backend/server/scripts/loader.js

@@ -1,11 +1,32 @@
-import { create, createEsmHooks } from 'ts-node';
+import * as otel from '@opentelemetry/instrumentation/hook.mjs';
+import { createEsmHooks, register } from 'ts-node';
 
-const service = create({
+const service = register({
   experimentalSpecifierResolution: 'node',
   transpileOnly: true,
   logError: true,
-  skipProject: true,
 });
-const hooks = createEsmHooks(service);
 
-export const resolve = hooks.resolve;
+/**
+ * @type {import('ts-node').NodeLoaderHooksAPI2}
+ 
+ */
+const ts = createEsmHooks(service);
+
+/**
+ * @type {import('ts-node').NodeLoaderHooksAPI2.ResolveHook}
+ */
+export const resolve = (specifier, context, defaultResolver) => {
+  return ts.resolve(specifier, context, (s, c) => {
+    return otel.resolve(s, c, defaultResolver);
+  });
+};
+
+/**
+ * @type {import('ts-node').NodeLoaderHooksAPI2.LoadHook}
+ */
+export const load = async (url, context, defaultLoader) => {
+  return await otel.load(url, context, (u, c) => {
+    return ts.load(u, c, defaultLoader);
+  });
+};

packages/backend/server/src/data/index.ts

@@ -3,9 +3,13 @@ import '../prelude';
 import { Logger } from '@nestjs/common';
 import { CommandFactory } from 'nest-commander';
 
+import { registerInstrumentations } from '../fundamentals/metrics';
+
 async function bootstrap() {
   AFFiNE.metrics.enabled = false;
   AFFiNE.doc.manager.enableUpdateAutoMerging = false;
+
+  registerInstrumentations();
   const { CliAppModule } = await import('./app');
   await CommandFactory.run(CliAppModule, new Logger()).catch(e => {
     console.error(e);

packages/backend/server/src/data/migrations/1703756315970-unamed-account.ts

@@ -9,9 +9,6 @@ export class UnamedAccount1703756315970 {
       const users = await db.$queryRaw<
         User[]
       >`SELECT * FROM users WHERE name ~ E'^[\\s\\u2000-\\u200F]*$';`;
-      console.log(
-        `renaming ${users.map(({ email }) => email).join('|')} users`
-      );
 
       await Promise.all(
         users.map(({ id, email }) =>

packages/backend/server/src/fundamentals/metrics/index.ts

@@ -50,6 +50,7 @@ export class MetricsModule implements OnModuleInit, OnModuleDestroy {
   }
 }
 
+export { registerInstrumentations } from './instrumentations';
 export * from './metrics';
 export * from './utils';
 export { OpentelemetryFactory };

packages/backend/server/src/fundamentals/metrics/instrumentations.ts

@@ -0,0 +1,32 @@
+import { Instrumentation } from '@opentelemetry/instrumentation';
+import { GraphQLInstrumentation } from '@opentelemetry/instrumentation-graphql';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
+import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
+import { SocketIoInstrumentation } from '@opentelemetry/instrumentation-socket.io';
+import prismaInstrument from '@prisma/instrumentation';
+
+const { PrismaInstrumentation } = prismaInstrument;
+
+let instrumentations: Instrumentation[] = [];
+
+export function registerInstrumentations(): void {
+  if (AFFiNE.metrics.enabled) {
+    instrumentations = [
+      new NestInstrumentation(),
+      new IORedisInstrumentation(),
+      new SocketIoInstrumentation({ traceReserved: true }),
+      new GraphQLInstrumentation({
+        mergeItems: true,
+        ignoreTrivialResolveSpans: true,
+        depth: 10,
+      }),
+      new HttpInstrumentation(),
+      new PrismaInstrumentation({ middleware: false }),
+    ];
+  }
+}
+
+export function getRegisteredInstrumentations(): Instrumentation[] {
+  return instrumentations;
+}

packages/backend/server/src/fundamentals/metrics/opentelemetry.ts

@@ -1,52 +1,82 @@
-import { OnModuleDestroy } from '@nestjs/common';
-import { metrics } from '@opentelemetry/api';
+import { Attributes, metrics } from '@opentelemetry/api';
 import {
   CompositePropagator,
   W3CBaggagePropagator,
   W3CTraceContextPropagator,
 } from '@opentelemetry/core';
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
-import { ZipkinExporter } from '@opentelemetry/exporter-zipkin';
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
 import { HostMetrics } from '@opentelemetry/host-metrics';
 import { Instrumentation } from '@opentelemetry/instrumentation';
-import { GraphQLInstrumentation } from '@opentelemetry/instrumentation-graphql';
-import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
-import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
-import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
-import { SocketIoInstrumentation } from '@opentelemetry/instrumentation-socket.io';
 import { Resource } from '@opentelemetry/resources';
 import type { MeterProvider } from '@opentelemetry/sdk-metrics';
-import { MetricProducer, MetricReader } from '@opentelemetry/sdk-metrics';
+import {
+  MetricProducer,
+  MetricReader,
+  PeriodicExportingMetricReader,
+} from '@opentelemetry/sdk-metrics';
 import { NodeSDK } from '@opentelemetry/sdk-node';
 import {
-  BatchSpanProcessor,
   SpanExporter,
   TraceIdRatioBasedSampler,
 } from '@opentelemetry/sdk-trace-node';
 import {
+  SEMRESATTRS_K8S_CLUSTER_NAME,
   SEMRESATTRS_K8S_NAMESPACE_NAME,
-  SEMRESATTRS_SERVICE_NAME,
+  SEMRESATTRS_K8S_POD_NAME,
   SEMRESATTRS_SERVICE_VERSION,
 } from '@opentelemetry/semantic-conventions';
-import prismaInstrument from '@prisma/instrumentation';
 
+import { getRegisteredInstrumentations } from './instrumentations';
 import { PrismaMetricProducer } from './prisma';
 
-const { PrismaInstrumentation } = prismaInstrument;
+function withBuiltinAttributesMetricReader(
+  reader: MetricReader,
+  attrs: Attributes
+) {
+  const collect = reader.collect;
+  reader.collect = async options => {
+    const result = await collect.call(reader, options);
+
+    result.resourceMetrics.scopeMetrics.forEach(metrics => {
+      metrics.metrics.forEach(metric => {
+        metric.dataPoints.forEach(dataPoint => {
+          // @ts-expect-error allow
+          dataPoint.attributes = Object.assign({}, attrs, dataPoint.attributes);
+        });
+      });
+    });
+
+    return result;
+  };
+
+  return reader;
+}
+
+function withBuiltinAttributesSpanExporter(
+  exporter: SpanExporter,
+  attrs: Attributes
+) {
+  const exportSpans = exporter.export;
+  exporter.export = (spans, callback) => {
+    spans.forEach(span => {
+      // patch span attributes
+      // @ts-expect-error allow
+      span.attributes = Object.assign({}, attrs, span.attributes);
+    });
+
+    return exportSpans.call(exporter, spans, callback);
+  };
+
+  return exporter;
+}
 
 export abstract class OpentelemetryFactory {
   abstract getMetricReader(): MetricReader;
   abstract getSpanExporter(): SpanExporter;
 
   getInstractions(): Instrumentation[] {
-    return [
-      new NestInstrumentation(),
-      new IORedisInstrumentation(),
-      new SocketIoInstrumentation({ traceReserved: true }),
-      new GraphQLInstrumentation({ mergeItems: true }),
-      new HttpInstrumentation(),
-      new PrismaInstrumentation(),
-    ];
+    return getRegisteredInstrumentations();
   }
 
   getMetricsProducers(): MetricProducer[] {
@@ -55,20 +85,32 @@ export abstract class OpentelemetryFactory {
 
   getResource() {
     return new Resource({
+      [SEMRESATTRS_K8S_CLUSTER_NAME]: AFFiNE.flavor.type,
       [SEMRESATTRS_K8S_NAMESPACE_NAME]: AFFiNE.AFFINE_ENV,
-      [SEMRESATTRS_SERVICE_NAME]: AFFiNE.flavor.type,
-      [SEMRESATTRS_SERVICE_VERSION]: AFFiNE.version,
+      [SEMRESATTRS_K8S_POD_NAME]: process.env.HOSTNAME ?? process.env.HOST,
     });
   }
 
+  getBuiltinAttributes(): Attributes {
+    return {
+      [SEMRESATTRS_SERVICE_VERSION]: AFFiNE.version,
+    };
+  }
+
   create() {
-    const traceExporter = this.getSpanExporter();
+    const builtinAttributes = this.getBuiltinAttributes();
+
     return new NodeSDK({
       resource: this.getResource(),
       sampler: new TraceIdRatioBasedSampler(0.1),
-      traceExporter,
-      metricReader: this.getMetricReader(),
-      spanProcessor: new BatchSpanProcessor(traceExporter),
+      traceExporter: withBuiltinAttributesSpanExporter(
+        this.getSpanExporter(),
+        builtinAttributes
+      ),
+      metricReader: withBuiltinAttributesMetricReader(
+        this.getMetricReader(),
+        builtinAttributes
+      ),
       textMapPropagator: new CompositePropagator({
         propagators: [
           new W3CBaggagePropagator(),
@@ -81,24 +123,19 @@ export abstract class OpentelemetryFactory {
   }
 }
 
-export class LocalOpentelemetryFactory
-  extends OpentelemetryFactory
-  implements OnModuleDestroy
-{
-  private readonly metricsExporter = new PrometheusExporter({
-    metricProducers: this.getMetricsProducers(),
-  });
-
-  async onModuleDestroy() {
-    await this.metricsExporter.shutdown();
+export class LocalOpentelemetryFactory extends OpentelemetryFactory {
+  override getMetricReader() {
+    return new PeriodicExportingMetricReader({
+      // requires jeager service running in 'http://localhost:4318'
+      // with metrics feature enabled.
+      // see https://www.jaegertracing.io/docs/1.56/spm
+      exporter: new OTLPMetricExporter(),
+    });
   }
 
-  override getMetricReader(): MetricReader {
-    return this.metricsExporter;
-  }
-
-  override getSpanExporter(): SpanExporter {
-    return new ZipkinExporter();
+  override getSpanExporter() {
+    // requires jeager service running in 'http://localhost:4318'
+    return new OTLPTraceExporter();
   }
 }
 

packages/backend/server/src/index.ts

@@ -6,7 +6,9 @@ import { omit } from 'lodash-es';
 
 import { createApp } from './app';
 import { URLHelper } from './fundamentals';
+import { registerInstrumentations } from './fundamentals/metrics';
 
+registerInstrumentations();
 const app = await createApp();
 const listeningHost = AFFiNE.deploy ? '0.0.0.0' : 'localhost';
 await app.listen(AFFiNE.server.port, listeningHost);