mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-25 06:18:45 +08:00
refactor(server): standarderlize metrics and trace with OTEL (#5054)
you can now export span to Zipkin and metrics to Prometheus when developing locally follow the docs of OTEL: https://opentelemetry.io/docs/instrumentation/js/exporters/ <img width="2357" alt="image" src="https://github.com/toeverything/AFFiNE/assets/8281226/ec615e1f-3e91-43f7-9111-d7d2629e9679">
This commit is contained in:
liuyi
@@ -1,18 +0,0 @@
|
||||
import { Controller, Get, Res } from '@nestjs/common';
|
||||
import type { Response } from 'express';
|
||||
import { register } from 'prom-client';
|
||||
|
||||
import { PrismaService } from '../prisma';
|
||||
|
||||
@Controller()
|
||||
export class MetricsController {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
@Get('/metrics')
|
||||
async index(@Res() res: Response): Promise<void> {
|
||||
res.header('Content-Type', register.contentType);
|
||||
const prismaMetrics = await this.prisma.$metrics.prometheus();
|
||||
const appMetrics = await register.metrics();
|
||||
res.send(appMetrics + prismaMetrics);
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,3 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
|
||||
import { MetricsController } from '../metrics/controller';
|
||||
import { Metrics } from './metrics';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [Metrics],
|
||||
exports: [Metrics],
|
||||
controllers: [MetricsController],
|
||||
})
|
||||
export class MetricsModule {}
|
||||
export { Metrics };
|
||||
export * from './metrics';
|
||||
export { start } from './opentelemetry';
|
||||
export * from './utils';
|
||||
|
||||
@@ -1,31 +1,76 @@
|
||||
import { Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||
import { register } from 'prom-client';
|
||||
import opentelemetry, { Attributes, Observable } from '@opentelemetry/api';
|
||||
|
||||
import { metricsCreator } from './utils';
|
||||
interface AsyncMetric {
|
||||
ob: Observable;
|
||||
get value(): any;
|
||||
get attrs(): Attributes | undefined;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class Metrics implements OnModuleDestroy {
|
||||
onModuleDestroy(): void {
|
||||
register.clear();
|
||||
let _metrics: ReturnType<typeof createBusinessMetrics> | undefined = undefined;
|
||||
|
||||
export function getMeter(name = 'business') {
|
||||
return opentelemetry.metrics.getMeter(name);
|
||||
}
|
||||
|
||||
function createBusinessMetrics() {
|
||||
const meter = getMeter();
|
||||
const asyncMetrics: AsyncMetric[] = [];
|
||||
|
||||
function createGauge(name: string) {
|
||||
let value: any;
|
||||
let attrs: Attributes | undefined;
|
||||
const ob = meter.createObservableGauge(name);
|
||||
asyncMetrics.push({
|
||||
ob,
|
||||
get value() {
|
||||
return value;
|
||||
},
|
||||
get attrs() {
|
||||
return attrs;
|
||||
},
|
||||
});
|
||||
|
||||
return (newValue: any, newAttrs?: Attributes) => {
|
||||
value = newValue;
|
||||
attrs = newAttrs;
|
||||
};
|
||||
}
|
||||
|
||||
socketIOEventCounter = metricsCreator.counter('socket_io_counter', ['event']);
|
||||
socketIOEventTimer = metricsCreator.timer('socket_io_timer', ['event']);
|
||||
socketIOConnectionGauge = metricsCreator.gauge(
|
||||
'socket_io_connection_counter'
|
||||
const metrics = {
|
||||
socketIOConnectionGauge: createGauge('socket_io_connection'),
|
||||
|
||||
gqlRequest: meter.createCounter('gql_request'),
|
||||
gqlError: meter.createCounter('gql_error'),
|
||||
gqlTimer: meter.createHistogram('gql_timer'),
|
||||
|
||||
jwstCodecMerge: meter.createCounter('jwst_codec_merge'),
|
||||
jwstCodecDidnotMatch: meter.createCounter('jwst_codec_didnot_match'),
|
||||
jwstCodecFail: meter.createCounter('jwst_codec_fail'),
|
||||
|
||||
authCounter: meter.createCounter('auth'),
|
||||
authFailCounter: meter.createCounter('auth_fail'),
|
||||
|
||||
docHistoryCounter: meter.createCounter('doc_history_created'),
|
||||
docRecoverCounter: meter.createCounter('doc_history_recovered'),
|
||||
};
|
||||
|
||||
meter.addBatchObservableCallback(
|
||||
result => {
|
||||
asyncMetrics.forEach(metric => {
|
||||
result.observe(metric.ob, metric.value, metric.attrs);
|
||||
});
|
||||
},
|
||||
asyncMetrics.map(({ ob }) => ob)
|
||||
);
|
||||
|
||||
gqlRequest = metricsCreator.counter('gql_request', ['operation']);
|
||||
gqlError = metricsCreator.counter('gql_error', ['operation']);
|
||||
gqlTimer = metricsCreator.timer('gql_timer', ['operation']);
|
||||
|
||||
jwstCodecMerge = metricsCreator.counter('jwst_codec_merge');
|
||||
jwstCodecDidnotMatch = metricsCreator.counter('jwst_codec_didnot_match');
|
||||
jwstCodecFail = metricsCreator.counter('jwst_codec_fail');
|
||||
|
||||
authCounter = metricsCreator.counter('auth');
|
||||
authFailCounter = metricsCreator.counter('auth_fail', ['reason']);
|
||||
|
||||
docHistoryCounter = metricsCreator.counter('doc_history_created');
|
||||
docRecoverCounter = metricsCreator.counter('doc_history_recovered');
|
||||
return metrics;
|
||||
}
|
||||
|
||||
export function registerBusinessMetrics() {
|
||||
if (!_metrics) {
|
||||
_metrics = createBusinessMetrics();
|
||||
}
|
||||
|
||||
return _metrics;
|
||||
}
|
||||
export const metrics = registerBusinessMetrics;
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { MetricExporter } from '@google-cloud/opentelemetry-cloud-monitoring-exporter';
|
||||
import { TraceExporter } from '@google-cloud/opentelemetry-cloud-trace-exporter';
|
||||
import {
|
||||
CompositePropagator,
|
||||
W3CBaggagePropagator,
|
||||
W3CTraceContextPropagator,
|
||||
} from '@opentelemetry/core';
|
||||
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
|
||||
import { ZipkinExporter } from '@opentelemetry/exporter-zipkin';
|
||||
import { HostMetrics } from '@opentelemetry/host-metrics';
|
||||
import { Instrumentation } from '@opentelemetry/instrumentation';
|
||||
import { GraphQLInstrumentation } from '@opentelemetry/instrumentation-graphql';
|
||||
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
|
||||
import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
|
||||
import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
|
||||
import { SocketIoInstrumentation } from '@opentelemetry/instrumentation-socket.io';
|
||||
import {
|
||||
ConsoleMetricExporter,
|
||||
MetricReader,
|
||||
PeriodicExportingMetricReader,
|
||||
} from '@opentelemetry/sdk-metrics';
|
||||
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||
import {
|
||||
BatchSpanProcessor,
|
||||
ConsoleSpanExporter,
|
||||
SpanExporter,
|
||||
} from '@opentelemetry/sdk-trace-node';
|
||||
import { PrismaInstrumentation } from '@prisma/instrumentation';
|
||||
|
||||
import { registerBusinessMetrics } from './metrics';
|
||||
|
||||
abstract class OpentelemetryFactor {
|
||||
abstract getMetricReader(): MetricReader;
|
||||
abstract getSpanExporter(): SpanExporter;
|
||||
|
||||
getInstractions(): Instrumentation[] {
|
||||
return [
|
||||
new NestInstrumentation(),
|
||||
new IORedisInstrumentation(),
|
||||
new SocketIoInstrumentation({ traceReserved: true }),
|
||||
new GraphQLInstrumentation({ mergeItems: true }),
|
||||
new HttpInstrumentation(),
|
||||
new PrismaInstrumentation(),
|
||||
];
|
||||
}
|
||||
|
||||
create() {
|
||||
const traceExporter = this.getSpanExporter();
|
||||
return new NodeSDK({
|
||||
traceExporter,
|
||||
metricReader: this.getMetricReader(),
|
||||
spanProcessor: new BatchSpanProcessor(traceExporter),
|
||||
textMapPropagator: new CompositePropagator({
|
||||
propagators: [
|
||||
new W3CBaggagePropagator(),
|
||||
new W3CTraceContextPropagator(),
|
||||
],
|
||||
}),
|
||||
instrumentations: this.getInstractions(),
|
||||
serviceName: 'affine-cloud',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
class GCloudOpentelemetryFactor extends OpentelemetryFactor {
|
||||
override getMetricReader(): MetricReader {
|
||||
return new PeriodicExportingMetricReader({
|
||||
exportIntervalMillis: 30000,
|
||||
exportTimeoutMillis: 60000,
|
||||
exporter: new MetricExporter(),
|
||||
});
|
||||
}
|
||||
|
||||
override getSpanExporter(): SpanExporter {
|
||||
return new TraceExporter();
|
||||
}
|
||||
}
|
||||
|
||||
class LocalOpentelemetryFactor extends OpentelemetryFactor {
|
||||
override getMetricReader(): MetricReader {
|
||||
return new PrometheusExporter();
|
||||
}
|
||||
|
||||
override getSpanExporter(): SpanExporter {
|
||||
return new ZipkinExporter();
|
||||
}
|
||||
}
|
||||
|
||||
class DebugOpentelemetryFactor extends OpentelemetryFactor {
|
||||
override getMetricReader(): MetricReader {
|
||||
return new PeriodicExportingMetricReader({
|
||||
exporter: new ConsoleMetricExporter(),
|
||||
});
|
||||
}
|
||||
|
||||
override getSpanExporter(): SpanExporter {
|
||||
return new ConsoleSpanExporter();
|
||||
}
|
||||
}
|
||||
|
||||
function createSDK() {
|
||||
let factor: OpentelemetryFactor | null = null;
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
factor = new GCloudOpentelemetryFactor();
|
||||
} else if (process.env.DEBUG_METRICS) {
|
||||
factor = new DebugOpentelemetryFactor();
|
||||
} else {
|
||||
factor = new LocalOpentelemetryFactor();
|
||||
}
|
||||
|
||||
return factor?.create();
|
||||
}
|
||||
|
||||
function registerCustomMetrics() {
|
||||
const host = new HostMetrics({ name: 'instance-host-metrics' });
|
||||
host.start();
|
||||
}
|
||||
|
||||
export function start() {
|
||||
const sdk = createSDK();
|
||||
|
||||
if (sdk) {
|
||||
sdk.start();
|
||||
registerCustomMetrics();
|
||||
registerBusinessMetrics();
|
||||
}
|
||||
}
|
||||
@@ -1,99 +1,11 @@
|
||||
import { Counter, Gauge, register, Summary } from 'prom-client';
|
||||
import { Attributes } from '@opentelemetry/api';
|
||||
|
||||
function getOr<T>(name: string, or: () => T): T {
|
||||
return (register.getSingleMetric(name) as T) || or();
|
||||
}
|
||||
|
||||
type LabelValues<T extends string> = Partial<Record<T, string | number>>;
|
||||
type MetricsCreator<T extends string> = (
|
||||
value: number,
|
||||
labels: LabelValues<T>
|
||||
) => void;
|
||||
type TimerMetricsCreator<T extends string> = (
|
||||
labels: LabelValues<T>
|
||||
) => () => number;
|
||||
|
||||
export const metricsCreatorGenerator = () => {
|
||||
const counterCreator = <T extends string>(
|
||||
name: string,
|
||||
labelNames?: T[]
|
||||
): MetricsCreator<T> => {
|
||||
const counter = getOr(
|
||||
name,
|
||||
() =>
|
||||
new Counter({
|
||||
name,
|
||||
help: name,
|
||||
...(labelNames ? { labelNames } : {}),
|
||||
})
|
||||
);
|
||||
|
||||
return (value: number, labels: LabelValues<T>) => {
|
||||
counter.inc(labels, value);
|
||||
};
|
||||
};
|
||||
|
||||
const gaugeCreator = <T extends string>(
|
||||
name: string,
|
||||
labelNames?: T[]
|
||||
): MetricsCreator<T> => {
|
||||
const gauge = getOr(
|
||||
name,
|
||||
() =>
|
||||
new Gauge({
|
||||
name,
|
||||
help: name,
|
||||
...(labelNames ? { labelNames } : {}),
|
||||
})
|
||||
);
|
||||
|
||||
return (value: number, labels: LabelValues<T>) => {
|
||||
gauge.set(labels, value);
|
||||
};
|
||||
};
|
||||
|
||||
const timerCreator = <T extends string>(
|
||||
name: string,
|
||||
labelNames?: T[]
|
||||
): TimerMetricsCreator<T> => {
|
||||
const summary = getOr(
|
||||
name,
|
||||
() =>
|
||||
new Summary({
|
||||
name,
|
||||
help: name,
|
||||
...(labelNames ? { labelNames } : {}),
|
||||
})
|
||||
);
|
||||
|
||||
return (labels: LabelValues<T>) => {
|
||||
const now = process.hrtime();
|
||||
|
||||
return () => {
|
||||
const delta = process.hrtime(now);
|
||||
const value = delta[0] + delta[1] / 1e9;
|
||||
|
||||
summary.observe(labels, value);
|
||||
return value;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
return {
|
||||
counter: counterCreator,
|
||||
gauge: gaugeCreator,
|
||||
timer: timerCreator,
|
||||
};
|
||||
};
|
||||
|
||||
export const metricsCreator = metricsCreatorGenerator();
|
||||
import { getMeter } from './metrics';
|
||||
|
||||
export const CallTimer = (
|
||||
name: string,
|
||||
labels: Record<string, any> = {}
|
||||
attrs?: Attributes
|
||||
): MethodDecorator => {
|
||||
const timer = metricsCreator.timer(name, Object.keys(labels));
|
||||
|
||||
// @ts-expect-error allow
|
||||
return (
|
||||
_target,
|
||||
@@ -106,19 +18,27 @@ export const CallTimer = (
|
||||
}
|
||||
|
||||
desc.value = function (...args: any[]) {
|
||||
const endTimer = timer(labels);
|
||||
const timer = getMeter().createHistogram(name, {
|
||||
description: `function call time costs of ${name}`,
|
||||
});
|
||||
const start = Date.now();
|
||||
|
||||
const end = () => {
|
||||
timer.record(Date.now() - start, attrs);
|
||||
};
|
||||
|
||||
let result: any;
|
||||
try {
|
||||
result = originalMethod.apply(this, args);
|
||||
} catch (e) {
|
||||
endTimer();
|
||||
end();
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (result instanceof Promise) {
|
||||
return result.finally(endTimer);
|
||||
return result.finally(end);
|
||||
} else {
|
||||
endTimer();
|
||||
end();
|
||||
return result;
|
||||
}
|
||||
};
|
||||
@@ -129,10 +49,8 @@ export const CallTimer = (
|
||||
|
||||
export const CallCounter = (
|
||||
name: string,
|
||||
labels: Record<string, any> = {}
|
||||
attrs?: Attributes
|
||||
): MethodDecorator => {
|
||||
const count = metricsCreator.counter(name, Object.keys(labels));
|
||||
|
||||
// @ts-expect-error allow
|
||||
return (
|
||||
_target,
|
||||
@@ -145,7 +63,11 @@ export const CallCounter = (
|
||||
}
|
||||
|
||||
desc.value = function (...args: any[]) {
|
||||
count(1, labels);
|
||||
const count = getMeter().createCounter(name, {
|
||||
description: `function call counter of ${name}`,
|
||||
});
|
||||
|
||||
count.add(1, attrs);
|
||||
return originalMethod.apply(this, args);
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user