Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-02 02:00:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: improve event flow (#14266)

Browse Source
This commit is contained in:
DarkSky
2026-01-16 16:07:27 +08:00
committed by GitHub
parent d4581b839a
commit 924d58603f
43 changed files with 2306 additions and 567 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.docker/selfhost/schema.json
+39
View File
@@ -634,6 +634,45 @@
}
}
},
"telemetry": {
"type": "object",
"description": "Configuration for telemetry module",
"properties": {
"allowedOrigin": {
"type": "array",
"description": "Allowed origins for telemetry collection.\n@default [\"localhost\",\"127.0.0.1\"]",
"default": [
"localhost",
"127.0.0.1"
]
},
"ga4.measurementId": {
"type": "string",
"description": "GA4 Measurement ID for Measurement Protocol.\n@default \"\"\n@environment `GA4_MEASUREMENT_ID`",
"default": ""
},
"ga4.apiSecret": {
"type": "string",
"description": "GA4 API secret for Measurement Protocol.\n@default \"\"\n@environment `GA4_API_SECRET`",
"default": ""
},
"dedupe.ttlHours": {
"type": "number",
"description": "Telemetry dedupe TTL in hours.\n@default 24",
"default": 24
},
"dedupe.maxEntries": {
"type": "number",
"description": "Telemetry dedupe max entries.\n@default 100000",
"default": 100000
},
"batch.maxEvents": {
"type": "number",
"description": "Max events per telemetry batch.\n@default 25",
"default": 25
}
}
},
"client": {
"type": "object",
"description": "Configuration for client module",
packages/backend/server/src/app.module.ts
+3 -1
View File
@@ -45,6 +45,7 @@ import { QuotaModule } from './core/quota';
import { SelfhostModule } from './core/selfhost';
import { StorageModule } from './core/storage';
import { SyncModule } from './core/sync';
import { TelemetryModule } from './core/telemetry';
import { UserModule } from './core/user';
import { VersionModule } from './core/version';
import { WorkspaceModule } from './core/workspaces';
@@ -175,7 +176,7 @@ export function buildAppModule(env: Env) {
// renderer server only
.useIf(() => env.flavors.renderer, DocRendererModule)
// sync server only
.useIf(() => env.flavors.sync, SyncModule)
.useIf(() => env.flavors.sync, SyncModule, TelemetryModule)
// graphql server only
.useIf(
() => env.flavors.graphql,
@@ -191,6 +192,7 @@ export function buildAppModule(env: Env) {
OAuthModule,
CalendarModule,
CustomerIoModule,
TelemetryModule,
CommentModule,
AccessTokenModule,
QueueDashboardModule
packages/backend/server/src/core/telemetry/__tests__/cleaner.spec.ts
+81
View File
@@ -0,0 +1,81 @@
import test from 'ava';
import {
cleanTelemetryEvent,
sanitizeParams,
sanitizeUserProperties,
} from '../cleaner';
import { TelemetryDeduper } from '../deduper';
test('sanitizeParams applies renames, drops, and normalization', t => {
const params = {
page: 'home',
status: true,
docId: 'doc-1',
other: { foo: 'bar' },
nested: { value: 'x' },
enabled: false,
};
const sanitized = sanitizeParams(params);
t.is(sanitized.ui_page, 'home');
t.is(sanitized.result, 'success');
t.is(sanitized.nested_setting_value, 'x');
t.is(sanitized.enabled, 'off');
t.false('doc_id' in sanitized);
t.false(Object.keys(sanitized).some(key => key.includes('other')));
});
test('sanitizeUserProperties filters and maps values', t => {
const props = {
appVersion: '1.2.3',
pro: true,
$email: 'test@example.com',
isMobile: true,
};
const sanitized = sanitizeUserProperties(props);
t.deepEqual(sanitized, {
app_version: '1.2.3',
plan_tier: 'pro',
is_mobile: '1',
});
});
test('cleanTelemetryEvent maps page view metadata', t => {
const cleaned = cleanTelemetryEvent(
{
schemaVersion: 1,
eventName: 'track_pageview',
clientId: 'client-1',
eventId: 'event-1',
params: {
location: 'https://example.com/docs?tab=1',
title: 'Example',
},
context: {
url: 'https://example.com/docs?tab=1',
},
},
{
timestampMicros: 123456,
}
);
t.truthy(cleaned);
t.is(cleaned?.eventName, 'page_view');
t.is(cleaned?.params.page_location, 'https://example.com/docs?tab=1');
t.is(cleaned?.params.page_path, '/docs?tab=1');
t.is(cleaned?.params.page_title, 'Example');
});
test('TelemetryDeduper drops duplicates within ttl', t => {
const deduper = new TelemetryDeduper(1000, 10);
const key = 'client-1:event-1';
t.false(deduper.isDuplicate(key, 0));
t.true(deduper.isDuplicate(key, 500));
t.false(deduper.isDuplicate(key, 1500));
});
packages/backend/server/src/core/telemetry/cleaner.ts
+444
View File
@@ -0,0 +1,444 @@
import { TelemetryEvent } from './types';
export type Scalar = string | number;
export type CleanedTelemetryEvent = {
clientId: string;
userId?: string;
eventId: string;
eventName: string;
params: Record<string, Scalar>;
userProperties: Record<string, string>;
timestampMicros: number;
};
const EVENT_NAME_RE = /^[A-Za-z][A-Za-z0-9_]{0,39}$/;
const PARAM_NAME_RE = /^[A-Za-z][A-Za-z0-9_]{0,39}$/;
const USER_PROP_NAME_RE = /^[A-Za-z][A-Za-z0-9_]{0,23}$/;
const EVENT_NAME_ALIAS: Record<string, string> = {
track_pageview: 'page_view',
};
const PARAM_RENAME_MAP = new Map<string, string>([
['page', 'ui_page'],
['segment', 'ui_segment'],
['module', 'ui_module'],
['arg', 'ui_arg'],
['control', 'ui_control'],
['option', 'ui_option'],
['key', 'setting_key'],
['value', 'setting_value'],
['docId', 'doc_id'],
['workspaceId', 'workspace_id'],
['serverId', 'server_id'],
['docType', 'doc_type'],
['docCount', 'doc_count'],
['unreadCount', 'unread_count'],
['withAttachment', 'with_attachment'],
['withMention', 'with_mention'],
['appName', 'app_name'],
['recurring', 'billing_cycle'],
['plan', 'plan_name'],
['time', 'duration_ms'],
['error', 'error_code'],
['status', 'result'],
['success', 'result'],
['to', 'target'],
['on', 'enabled'],
['pageLocation', 'page_location'],
['pageReferrer', 'page_referrer'],
['pagePath', 'page_path'],
['pageTitle', 'page_title'],
]);
const USER_PROP_RENAME_MAP = new Map<string, string>([
['appVersion', 'app_version'],
['editorVersion', 'editor_version'],
['environment', 'environment'],
['isDesktop', 'is_desktop'],
['isMobile', 'is_mobile'],
['distribution', 'distribution'],
['isSelfHosted', 'is_self_hosted'],
['ai', 'ai_enabled'],
['pro', 'plan_tier'],
['quota', 'quota_tier'],
]);
const DROP_PARAM_SEGMENTS = new Set(['other', 'instruction', 'operation']);
const DROP_MAPPED_PARAMS = new Set(['doc_id', 'workspace_id', 'server_id']);
const PRIORITY_KEYS = new Set([
'event_id',
'session_id',
'ui_page',
'ui_segment',
'ui_module',
'ui_control',
'ui_option',
'ui_arg',
'type',
'method',
'mode',
'plan_name',
'billing_cycle',
'role',
'result',
'error_code',
'category',
'doc_type',
'item',
'action',
'target',
'enabled',
'setting_key',
'setting_value',
'duration_ms',
'doc_count',
'unread_count',
'with_attachment',
'with_mention',
'page_location',
'page_path',
'page_referrer',
'page_title',
]);
export function cleanTelemetryEvent(
event: TelemetryEvent,
{
userId,
timestampMicros,
maxParams = 25,
}: {
userId?: string;
timestampMicros: number;
maxParams?: number;
}
): CleanedTelemetryEvent | null {
if (!event || event.schemaVersion !== 1) {
return null;
}
if (
!event.clientId ||
typeof event.clientId !== 'string' ||
!event.eventId ||
typeof event.eventId !== 'string' ||
!event.eventName ||
typeof event.eventName !== 'string'
) {
return null;
}
const mappedEventName = mapEventName(event.eventName);
if (!EVENT_NAME_RE.test(mappedEventName)) {
return null;
}
const contextParams = buildContextParams(event.context);
const baseParams = isPlainObject(event.params) ? event.params : {};
const mergedParams: Record<string, unknown> = {
...baseParams,
...contextParams,
eventId: event.eventId,
};
if (event.sessionId) {
mergedParams.sessionId = event.sessionId;
}
const contextUserProps = buildContextUserProps(event.context);
const baseUserProps = isPlainObject(event.userProperties)
? event.userProperties
: {};
const mergedUserProps = {
...contextUserProps,
...baseUserProps,
};
const sanitizedUserProps = sanitizeUserProperties(mergedUserProps);
const sanitizedParams =
mappedEventName === 'page_view'
? sanitizePageViewParams(mergedParams, event.context, maxParams)
: sanitizeParams(mergedParams, maxParams);
if (!Object.keys(sanitizedParams).length) {
sanitizedParams.event_id = event.eventId;
}
return {
clientId: event.clientId,
userId: event.userId ?? userId,
eventId: event.eventId,
eventName: mappedEventName,
params: sanitizedParams,
userProperties: sanitizedUserProps,
timestampMicros,
};
}
function buildContextParams(context?: TelemetryEvent['context']) {
if (!context) {
return {};
}
const params: Record<string, unknown> = {};
if (context.url) {
params.pageLocation = context.url;
}
if (context.referrer) {
params.pageReferrer = context.referrer;
}
if (context.locale) {
params.locale = context.locale;
}
if (context.timezone) {
params.timezone = context.timezone;
}
if (context.channel) {
params.channel = context.channel;
}
return params;
}
function buildContextUserProps(context?: TelemetryEvent['context']) {
if (!context) {
return {};
}
const props: Record<string, unknown> = {};
if (context.appVersion) {
props.appVersion = context.appVersion;
}
if (context.editorVersion) {
props.editorVersion = context.editorVersion;
}
if (context.environment) {
props.environment = context.environment;
}
if (context.distribution) {
props.distribution = context.distribution;
}
if (context.isDesktop !== undefined) {
props.isDesktop = context.isDesktop;
}
if (context.isMobile !== undefined) {
props.isMobile = context.isMobile;
}
return props;
}
function sanitizePageViewParams(
input: Record<string, unknown>,
context: TelemetryEvent['context'] | undefined,
maxParams: number
) {
const customParams = { ...input };
const rawLocation = customParams.location;
const rawTitle = customParams.title ?? customParams.pageTitle;
delete customParams.location;
delete customParams.title;
delete customParams.pageTitle;
const { pageLocation, pagePath, pageTitle } = resolvePageViewMeta(
rawLocation,
context?.url,
rawTitle
);
const sanitized = sanitizeParams(customParams, Math.max(maxParams - 3, 1));
const merged: Record<string, Scalar> = { ...sanitized };
if (pageLocation) {
merged.page_location = pageLocation;
}
if (pagePath) {
merged.page_path = pagePath;
}
if (pageTitle) {
merged.page_title = pageTitle;
}
return merged;
}
function resolvePageViewMeta(
locationValue: unknown,
contextUrl: string | undefined,
titleValue: unknown
) {
let pageLocation =
typeof locationValue === 'string' ? locationValue : contextUrl;
let pagePath: string | undefined;
if (pageLocation) {
try {
const url = contextUrl
? new URL(pageLocation, contextUrl)
: new URL(pageLocation);
pagePath = url.pathname + url.search;
if (!pageLocation.startsWith('http')) {
pageLocation = url.toString();
}
} catch {
if (pageLocation.startsWith('/')) {
pagePath = pageLocation;
}
}
}
const pageTitle = typeof titleValue === 'string' ? titleValue : undefined;
return { pageLocation, pagePath, pageTitle };
}
function toSnakeCase(input: string): string {
return input
.replace(/([a-z0-9])([A-Z])/g, '$1_$2')
.replace(/([A-Z]+)([A-Z][a-z0-9]+)/g, '$1_$2')
.replace(/\s+/g, '_')
.replace(/[^a-zA-Z0-9_]/g, '_')
.replace(/__+/g, '_')
.toLowerCase();
}
function toScalar(value: unknown): Scalar | undefined {
if (value === null || value === undefined) {
return undefined;
}
if (typeof value === 'number' && Number.isFinite(value)) {
return value;
}
if (typeof value === 'boolean') {
return value ? 1 : 0;
}
if (typeof value === 'string') {
return value.length > 100 ? value.slice(0, 100) : value;
}
if (value instanceof Date) {
return value.toISOString();
}
try {
const serialized = JSON.stringify(value);
return serialized.length > 100 ? serialized.slice(0, 100) : serialized;
} catch {
return undefined;
}
}
function normalizeValue(key: string, value: unknown): Scalar | undefined {
if (key === 'result' && typeof value === 'boolean') {
return value ? 'success' : 'failure';
}
if (key === 'enabled' && typeof value === 'boolean') {
return value ? 'on' : 'off';
}
return toScalar(value);
}
function isPlainObject(value: unknown): value is Record<string, unknown> {
if (!value || typeof value !== 'object') return false;
if (value instanceof Date) return false;
if (Array.isArray(value)) return false;
return Object.getPrototypeOf(value) === Object.prototype;
}
function flattenProps(
input: Record<string, unknown>,
prefix = ''
): Record<string, unknown> {
const out: Record<string, unknown> = {};
for (const [key, value] of Object.entries(input)) {
const path = prefix ? `${prefix}.${key}` : key;
if (isPlainObject(value)) {
Object.assign(out, flattenProps(value, path));
} else {
out[path] = value;
}
}
return out;
}
function mapParamKey(path: string): string {
const segments = path.split('.');
const mappedSegments = segments.map(
segment => PARAM_RENAME_MAP.get(segment) ?? segment
);
return toSnakeCase(mappedSegments.join('_'));
}
function mapEventName(name: string): string {
const alias = EVENT_NAME_ALIAS[name];
return toSnakeCase(alias ?? name);
}
function shouldDropPath(path: string): boolean {
const segments = path.split('.');
return segments.some(segment => DROP_PARAM_SEGMENTS.has(segment));
}
export function sanitizeParams(
input: Record<string, unknown>,
maxParams = 25
): Record<string, Scalar> {
const flattened = flattenProps(input);
const mappedEntries: Array<[string, Scalar]> = [];
for (const [path, value] of Object.entries(flattened)) {
if (shouldDropPath(path)) continue;
const mappedKey = mapParamKey(path);
if (!mappedKey || !PARAM_NAME_RE.test(mappedKey)) continue;
if (DROP_MAPPED_PARAMS.has(mappedKey)) continue;
const normalized = normalizeValue(mappedKey, value);
if (normalized === undefined) continue;
mappedEntries.push([mappedKey, normalized]);
}
mappedEntries.sort((a, b) => {
const aPriority = PRIORITY_KEYS.has(a[0]);
const bPriority = PRIORITY_KEYS.has(b[0]);
if (aPriority === bPriority) return 0;
return aPriority ? -1 : 1;
});
const out: Record<string, Scalar> = {};
for (const [key, value] of mappedEntries) {
if (Object.keys(out).length >= maxParams) break;
if (key in out) continue;
out[key] = value;
}
return out;
}
function mapUserPropKey(key: string): string | undefined {
if (key.startsWith('$')) return undefined;
const mapped = USER_PROP_RENAME_MAP.get(key) ?? key;
return toSnakeCase(mapped);
}
export function sanitizeUserProperties(
props: Record<string, unknown>
): Record<string, string> {
const sanitized: Record<string, string> = {};
for (const [key, value] of Object.entries(props)) {
const mappedKey = mapUserPropKey(key);
if (!mappedKey || !USER_PROP_NAME_RE.test(mappedKey)) continue;
let mappedValue = value;
if (key === 'pro' && typeof value === 'boolean') {
mappedValue = value ? 'pro' : 'free';
}
const scalar = toScalar(mappedValue);
if (scalar === undefined) continue;
const stringValue = String(scalar);
sanitized[mappedKey] =
stringValue.length > 36 ? stringValue.slice(0, 36) : stringValue;
}
return sanitized;
}
packages/backend/server/src/core/telemetry/config.ts
+51
View File
@@ -0,0 +1,51 @@
import { defineModuleConfig } from '../../base';
export interface TelemetryConfig {
allowedOrigin: ConfigItem<string[]>;
ga4: {
measurementId: ConfigItem<string>;
apiSecret: ConfigItem<string>;
};
dedupe: {
ttlHours: ConfigItem<number>;
maxEntries: ConfigItem<number>;
};
batch: {
maxEvents: ConfigItem<number>;
};
}
declare global {
interface AppConfigSchema {
telemetry: TelemetryConfig;
}
}
defineModuleConfig('telemetry', {
allowedOrigin: {
desc: 'Allowed origins for telemetry collection.',
default: ['localhost', '127.0.0.1'],
},
'ga4.measurementId': {
desc: 'GA4 Measurement ID for Measurement Protocol.',
default: '',
env: 'GA4_MEASUREMENT_ID',
},
'ga4.apiSecret': {
desc: 'GA4 API secret for Measurement Protocol.',
default: '',
env: 'GA4_API_SECRET',
},
'dedupe.ttlHours': {
desc: 'Telemetry dedupe TTL in hours.',
default: 24,
},
'dedupe.maxEntries': {
desc: 'Telemetry dedupe max entries.',
default: 100000,
},
'batch.maxEvents': {
desc: 'Max events per telemetry batch.',
default: 25,
},
});
packages/backend/server/src/core/telemetry/controller.ts
+66
View File
@@ -0,0 +1,66 @@
import { Body, Controller, Options, Post, Req, Res } from '@nestjs/common';
import type { Request, Response } from 'express';
import { BadRequest, Throttle, UseNamedGuard } from '../../base';
import type { CurrentUser as CurrentUserType } from '../auth';
import { Public } from '../auth';
import { CurrentUser } from '../auth';
import { TelemetryService } from './service';
import { TelemetryAck, type TelemetryBatch } from './types';
@Public()
@UseNamedGuard('version')
@Throttle('default')
@Controller('/api/telemetry')
export class TelemetryController {
constructor(private readonly telemetry: TelemetryService) {}
@Options('/collect')
collectOptions(@Req() req: Request, @Res() res: Response) {
const origin = req.headers.origin;
const referer = req.headers.referer;
if (!this.telemetry.isOriginAllowed(origin, referer)) {
throw new BadRequest(`Invalid origin: ${origin}, referer: ${referer}`);
}
return res
.status(200)
.header({
...this.telemetry.getCorsHeaders(origin),
'Access-Control-Allow-Methods': 'POST, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type, x-affine-version',
})
.send();
}
@Post('/collect')
async collect(
@Req() req: Request,
@Res({ passthrough: true }) res: Response,
@Body() batch: TelemetryBatch,
@CurrentUser() user?: CurrentUserType
): Promise<TelemetryAck> {
const origin = req.headers.origin;
const referer = req.headers.referer;
if (!this.telemetry.isOriginAllowed(origin, referer)) {
throw new BadRequest('Invalid origin: ' + origin);
}
res.header({
...this.telemetry.getCorsHeaders(origin),
});
const patchedBatch =
user && Array.isArray(batch?.events)
? {
...batch,
events: batch.events.map(event => ({
...event,
userId: event.userId ?? user.id,
})),
}
: batch;
return this.telemetry.collectBatch(patchedBatch);
}
}
packages/backend/server/src/core/telemetry/deduper.ts
+31
View File
@@ -0,0 +1,31 @@
export class TelemetryDeduper {
private readonly entries = new Map<string, number>();
constructor(
private readonly ttlMs: number,
private readonly maxEntries: number
) {}
isDuplicate(key: string, now = Date.now()) {
const existing = this.entries.get(key);
if (existing && existing > now) {
return true;
}
this.entries.set(key, now + this.ttlMs);
this.prune(now);
return false;
}
private prune(now: number) {
for (const [key, expiresAt] of this.entries) {
if (expiresAt <= now || this.entries.size > this.maxEntries) {
this.entries.delete(key);
continue;
}
if (this.entries.size <= this.maxEntries) {
break;
}
}
}
}
packages/backend/server/src/core/telemetry/ga4-client.ts
+124
View File
@@ -0,0 +1,124 @@
import { CleanedTelemetryEvent, Scalar } from './cleaner';
const GA4_ENDPOINT = 'https://www.google-analytics.com/mp/collect';
type Ga4Payload = {
client_id: string;
user_id?: string;
user_properties?: Record<string, { value: string }>;
events: Array<{
name: string;
params: Record<string, Scalar>;
timestamp_micros?: number;
}>;
};
export class Ga4Client {
constructor(
private readonly measurementId: string,
private readonly apiSecret: string,
private readonly maxEvents: number
) {}
async send(events: CleanedTelemetryEvent[]) {
if (!events.length) {
return;
}
if (!this.measurementId || !this.apiSecret) {
return;
}
const groups = groupEvents(events);
for (const group of groups) {
for (const chunk of chunkEvents(group.events, this.maxEvents)) {
const payload: Ga4Payload = {
client_id: group.clientId,
user_id: group.userId,
user_properties: toUserProperties(group.userProperties),
events: chunk.map(event => ({
name: event.eventName,
params: event.params,
timestamp_micros: event.timestampMicros,
})),
};
await this.post(payload);
}
}
}
private async post(payload: Ga4Payload) {
const url = new URL(GA4_ENDPOINT);
url.searchParams.set('measurement_id', this.measurementId);
url.searchParams.set('api_secret', this.apiSecret);
const response = await fetch(url, {
method: 'POST',
headers: {
'content-type': 'application/json',
},
body: JSON.stringify(payload),
});
if (!response.ok) {
const body = await response.text().catch(() => '');
throw new Error(
`GA4 request failed with ${response.status}: ${body || 'unknown error'}`
);
}
}
}
type GroupKey = {
clientId: string;
userId?: string;
userProperties: Record<string, string>;
events: CleanedTelemetryEvent[];
};
function groupEvents(events: CleanedTelemetryEvent[]): GroupKey[] {
const grouped = new Map<string, GroupKey>();
for (const event of events) {
const key = `${event.clientId}::${event.userId ?? ''}::${serializeUserProps(event.userProperties)}`;
const existing = grouped.get(key);
if (existing) {
existing.events.push(event);
} else {
grouped.set(key, {
clientId: event.clientId,
userId: event.userId,
userProperties: event.userProperties,
events: [event],
});
}
}
return Array.from(grouped.values());
}
function serializeUserProps(props: Record<string, string>) {
const entries = Object.entries(props).sort(([a], [b]) => a.localeCompare(b));
return JSON.stringify(entries);
}
function toUserProperties(props: Record<string, string>) {
const userProperties: Record<string, { value: string }> = {};
for (const [key, value] of Object.entries(props)) {
userProperties[key] = { value };
}
return Object.keys(userProperties).length ? userProperties : undefined;
}
function chunkEvents<T>(events: T[], size: number) {
if (events.length <= size) {
return [events];
}
const chunks: T[][] = [];
for (let i = 0; i < events.length; i += size) {
chunks.push(events.slice(i, i + size));
}
return chunks;
}
packages/backend/server/src/core/telemetry/gateway.ts
+51
View File
@@ -0,0 +1,51 @@
import { applyDecorators, UseInterceptors } from '@nestjs/common';
import {
ConnectedSocket,
MessageBody,
SubscribeMessage as RawSubscribeMessage,
WebSocketGateway,
} from '@nestjs/websockets';
import { ClsInterceptor } from 'nestjs-cls';
import { Socket } from 'socket.io';
import { BadRequest, GatewayErrorWrapper } from '../../base';
import { CurrentUser } from '../auth';
import { TelemetryService } from './service';
import { TelemetryAck, type TelemetryBatch } from './types';
const SubscribeMessage = (event: string) =>
applyDecorators(GatewayErrorWrapper(event), RawSubscribeMessage(event));
type EventResponse<Data = any> = [Data] extends [never]
? { data?: never }
: { data: Data };
@WebSocketGateway()
@UseInterceptors(ClsInterceptor)
export class TelemetryGateway {
constructor(private readonly telemetry: TelemetryService) {}
@SubscribeMessage('telemetry:batch')
async onBatch(
@CurrentUser() user: CurrentUser,
@ConnectedSocket() client: Socket,
@MessageBody() batch: TelemetryBatch
): Promise<EventResponse<TelemetryAck>> {
const origin = client.handshake.headers.origin;
const referer = client.handshake.headers.referer;
if (!this.telemetry.isOriginAllowed(origin, referer)) {
throw new BadRequest(`Invalid origin: ${origin}, referer: ${referer}`);
}
const ack = await this.telemetry.collectBatch({
...batch,
transport: 'ws',
events: batch?.events?.map(event => ({
...event,
userId: event.userId ?? user?.id,
})),
});
return { data: ack };
}
}
packages/backend/server/src/core/telemetry/index.ts
+13
View File
@@ -0,0 +1,13 @@
import './config';
import { Module } from '@nestjs/common';
import { TelemetryController } from './controller';
import { TelemetryGateway } from './gateway';
import { TelemetryService } from './service';
@Module({
providers: [TelemetryService, TelemetryGateway],
controllers: [TelemetryController],
})
export class TelemetryModule {}
packages/backend/server/src/core/telemetry/service.ts
+181
View File
@@ -0,0 +1,181 @@
import { Injectable, Logger } from '@nestjs/common';
import { Config, OnEvent, URLHelper } from '../../base';
import { cleanTelemetryEvent } from './cleaner';
import { TelemetryDeduper } from './deduper';
import { Ga4Client } from './ga4-client';
import { TelemetryAck, TelemetryBatch } from './types';
@Injectable()
export class TelemetryService {
private readonly logger = new Logger(TelemetryService.name);
private allowedOrigins: string[] = [];
private ga4Client!: Ga4Client;
private readonly deduper: TelemetryDeduper;
constructor(
private readonly config: Config,
private readonly url: URLHelper
) {
this.deduper = new TelemetryDeduper(
this.config.telemetry.dedupe.ttlHours * 60 * 60 * 1000,
this.config.telemetry.dedupe.maxEntries
);
this.refreshConfig();
}
@OnEvent('config.init')
onConfigInit() {
this.refreshConfig();
}
@OnEvent('config.changed')
onConfigChanged(event: Events['config.changed']) {
if ('telemetry' in event.updates) {
this.refreshConfig();
}
}
getCorsHeaders(origin?: string | string[] | null) {
const normalized = Array.isArray(origin) ? origin[0] : origin;
if (normalized) {
return {
'Access-Control-Allow-Origin': normalized,
};
}
return {};
}
isOriginAllowed(origin?: string | string[] | null, referer?: string | null) {
const normalizedOrigin = Array.isArray(origin) ? origin[0] : origin;
if (!normalizedOrigin && !referer) {
return true;
}
const originAllowed = normalizedOrigin
? this.allowedOrigins.includes(normalizedOrigin)
: false;
if (originAllowed) {
return true;
}
if (referer) {
try {
const refererOrigin = new URL(referer).origin;
return this.allowedOrigins.includes(refererOrigin);
} catch {
return false;
}
}
return false;
}
async collectBatch(batch: TelemetryBatch): Promise<TelemetryAck> {
if (!batch || batch.schemaVersion !== 1 || !Array.isArray(batch.events)) {
return {
ok: true,
accepted: 0,
dropped: 0,
};
}
const events = batch.events;
let dropped = 0;
const cleanedEvents = [];
const fallbackTimestamp =
typeof batch.sentAt === 'number' && Number.isFinite(batch.sentAt)
? batch.sentAt * 1000
: Date.now() * 1000;
for (const event of events) {
const timestampMicros =
typeof event.timestampMicros === 'number' &&
Number.isFinite(event.timestampMicros)
? event.timestampMicros
: fallbackTimestamp;
const cleaned = cleanTelemetryEvent(event, {
userId: event.userId,
timestampMicros,
maxParams: 25,
});
if (!cleaned) {
dropped++;
continue;
}
const dedupeKey = `${cleaned.clientId}:${cleaned.eventId}`;
if (this.deduper.isDuplicate(dedupeKey)) {
dropped++;
continue;
}
cleanedEvents.push(cleaned);
}
if (!cleanedEvents.length) {
return {
ok: true,
accepted: 0,
dropped,
};
}
try {
await this.ga4Client.send(cleanedEvents);
return {
ok: true,
accepted: cleanedEvents.length,
dropped,
};
} catch (error) {
const err = error as Error;
this.logger.error('Telemetry forwarding failed', err);
return {
ok: false,
error: {
name: err?.name ?? 'TelemetryForwardingError',
message: err?.message ?? 'Telemetry forwarding failed',
},
};
}
}
private refreshConfig() {
const normalizeOrigin = (input: string) => {
const candidate = input.includes('://') ? input : `http://${input}`;
try {
const url = new URL(candidate);
if (!['http:', 'https:', 'assets:'].includes(url.protocol)) {
return null;
} else if (url.protocol === 'assets:') {
return url.href;
}
return url.origin;
} catch {
return null;
}
};
const configOrigins = this.config.telemetry.allowedOrigin
.map(origin => normalizeOrigin(origin))
.filter((origin): origin is string => Boolean(origin));
this.allowedOrigins = Array.from(
new Set([...configOrigins, ...this.url.allowedOrigins])
);
this.logger.log(
`Telemetry allowed origins updated: ${this.allowedOrigins.join(', ')}`
);
this.ga4Client = new Ga4Client(
this.config.telemetry.ga4.measurementId,
this.config.telemetry.ga4.apiSecret,
Math.max(1, this.config.telemetry.batch.maxEvents)
);
}
}
packages/backend/server/src/core/telemetry/types.ts
+35
View File
@@ -0,0 +1,35 @@
export type TelemetryEvent = {
schemaVersion: 1;
eventName: string;
params?: Record<string, unknown>;
userProperties?: Record<string, unknown>;
userId?: string;
clientId: string;
sessionId?: string;
eventId: string;
timestampMicros?: number;
context?: {
appVersion?: string;
editorVersion?: string;
environment?: string;
distribution?: string;
channel?: 'stable' | 'beta' | 'internal' | 'canary';
isDesktop?: boolean;
isMobile?: boolean;
locale?: string;
timezone?: string;
url?: string;
referrer?: string;
};
};
export type TelemetryBatch = {
schemaVersion: 1;
transport: 'http' | 'ws';
sentAt: number;
events: TelemetryEvent[];
};
export type TelemetryAck =
| { ok: true; accepted: number; dropped: number }
| { ok: false; error: { name: string; message: string } };
packages/common/nbstore/src/impls/cloud/socket.ts
+3
View File
@@ -4,6 +4,7 @@ import {
} from 'socket.io-client';
import { AutoReconnectConnection } from '../../connection';
import type { TelemetryAck, TelemetryBatch } from '../../telemetry/types';
import { throwIfAborted } from '../../utils/throw-if-aborted';
// TODO(@forehalo): use [UserFriendlyError]
@@ -104,6 +105,8 @@ interface ClientEvents {
},
];
'space:delete-doc': { spaceType: string; spaceId: string; docId: string };
'telemetry:batch': [TelemetryBatch, TelemetryAck];
}
export type ServerEventsMap = {
packages/common/nbstore/src/telemetry/__tests__/manager.spec.ts
+75
View File
@@ -0,0 +1,75 @@
import 'fake-indexeddb/auto';
import { expect, test, vi } from 'vitest';
import { TelemetryManager } from '../manager';
import type { TelemetryContext, TelemetryEvent } from '../types';
const context: TelemetryContext = {
isAuthed: false,
isSelfHosted: false,
channel: 'stable',
officialEndpoint: 'https://example.com',
};
const baseEvent: TelemetryEvent = {
schemaVersion: 1,
eventName: 'openDoc',
clientId: 'client-1',
eventId: 'event-1',
};
test('telemetry manager retries with backoff and flushes on success', async () => {
const fetchMock = vi
.fn()
.mockResolvedValueOnce({
ok: false,
status: 500,
text: async () => 'fail',
})
.mockResolvedValueOnce({
ok: true,
json: async () => ({ ok: true, accepted: 1, dropped: 0 }),
});
globalThis.fetch = fetchMock as any;
(globalThis as any).BUILD_CONFIG = { appVersion: 'test' };
const manager = new TelemetryManager({
retryBaseMs: 10,
retryMaxMs: 10,
maxBatchEvents: 5,
});
await manager.setContext(context);
await manager.track(baseEvent);
const first = await manager.flush();
expect(first.ok).toBe(false);
expect(manager.getQueueState().size).toBe(1);
expect(manager.getQueueState().nextRetryAt).toBeDefined();
await vi.waitFor(() => {
expect(fetchMock).toHaveBeenCalledTimes(2);
expect(manager.getQueueState().size).toBe(0);
});
});
test('telemetry queue caps entries and drops oldest', async () => {
(globalThis as any).BUILD_CONFIG = { appVersion: 'test' };
globalThis.fetch = vi.fn() as any;
const manager = new TelemetryManager({
maxQueueEntries: 2,
maxQueueBytes: 10_000,
});
await manager.setContext({
...context,
officialEndpoint: '',
});
await manager.track({ ...baseEvent, eventId: 'event-1' });
await manager.track({ ...baseEvent, eventId: 'event-2' });
await manager.track({ ...baseEvent, eventId: 'event-3' });
expect(manager.getQueueState().size).toBe(2);
});
packages/common/nbstore/src/telemetry/manager.ts
+316
View File
@@ -0,0 +1,316 @@
import { SocketConnection } from '../impls/cloud/socket';
import { TelemetryQueue } from './queue';
import type {
TelemetryAck,
TelemetryBatch,
TelemetryContext,
TelemetryEvent,
} from './types';
const DEFAULT_MAX_QUEUE_ENTRIES = 2000;
const DEFAULT_MAX_QUEUE_BYTES = 2 * 1024 * 1024;
const DEFAULT_MAX_BATCH_EVENTS = 25;
const DEFAULT_RETRY_BASE_MS = 1000;
const DEFAULT_RETRY_MAX_MS = 5 * 60 * 1000;
type TelemetryManagerOptions = {
maxQueueEntries?: number;
maxQueueBytes?: number;
maxBatchEvents?: number;
retryBaseMs?: number;
retryMaxMs?: number;
};
export class TelemetryManager {
private context: TelemetryContext = {
isAuthed: false,
isSelfHosted: false,
channel: 'stable',
officialEndpoint: '',
};
private readonly queue: TelemetryQueue;
private readonly maxBatchEvents: number;
private readonly retryBaseMs: number;
private readonly retryMaxMs: number;
private retryAttempt = 0;
private retryTimer: ReturnType<typeof setTimeout> | null = null;
private nextRetryAt?: number;
private lastError?: string;
private flushPromise?: Promise<TelemetryAck>;
private socketConnection?: SocketConnection;
private socketEndpoint?: string;
constructor(options: TelemetryManagerOptions = {}) {
const maxQueueEntries =
options.maxQueueEntries ?? DEFAULT_MAX_QUEUE_ENTRIES;
const maxQueueBytes = options.maxQueueBytes ?? DEFAULT_MAX_QUEUE_BYTES;
this.queue = new TelemetryQueue(maxQueueEntries, maxQueueBytes);
this.maxBatchEvents = options.maxBatchEvents ?? DEFAULT_MAX_BATCH_EVENTS;
this.retryBaseMs = options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS;
this.retryMaxMs = options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS;
}
async setContext(context: TelemetryContext) {
this.context = { ...context };
this.updateSocketConnection();
this.scheduleFlush(true);
}
async track(event: TelemetryEvent) {
await this.queue.enqueue(event);
this.scheduleFlush(false);
return { queued: true };
}
async pageview(event: TelemetryEvent) {
return this.track(event);
}
async flush(): Promise<TelemetryAck> {
if (this.flushPromise) {
return this.flushPromise;
}
this.flushPromise = this.flushInternal().finally(() => {
this.flushPromise = undefined;
});
return this.flushPromise;
}
getQueueState() {
return {
size: this.queue.size,
lastError: this.lastError,
nextRetryAt: this.nextRetryAt,
};
}
private async flushInternal(): Promise<TelemetryAck> {
if (!this.context.officialEndpoint) {
return {
ok: false,
error: {
name: 'TelemetryEndpointMissing',
message: 'Telemetry official endpoint is not configured',
},
};
}
let accepted = 0;
let dropped = 0;
while (true) {
const items = await this.queue.peek(this.maxBatchEvents);
if (!items.length) {
this.resetRetry();
return { ok: true, accepted, dropped };
}
const events = items.map(item => this.mergeContext(item.event));
const ack = await this.sendBatch(events);
if (!ack.ok) {
this.recordFailure(ack.error.message);
return ack;
}
accepted += ack.accepted;
dropped += ack.dropped;
await this.queue.remove(items.map(item => item.id));
}
}
private mergeContext(event: TelemetryEvent): TelemetryEvent {
const mergedUserProps = {
...(this.context.userProperties ?? {}),
...(event.userProperties ?? {}),
};
const mergedContext = {
...event.context,
channel: event.context?.channel ?? this.context.channel,
};
return {
...event,
schemaVersion: 1,
userId: event.userId ?? this.context.userId,
userProperties: mergedUserProps,
context: mergedContext,
};
}
private async sendBatch(events: TelemetryEvent[]): Promise<TelemetryAck> {
const useWebsocket = this.context.isAuthed && !this.context.isSelfHosted;
const transport = useWebsocket ? 'ws' : 'http';
const batch: TelemetryBatch = {
schemaVersion: 1,
transport,
sentAt: Date.now(),
events,
};
try {
if (useWebsocket) {
return await this.sendWs(batch);
}
return await this.sendHttp(batch);
} catch (error) {
const err = error as Error;
return {
ok: false,
error: {
name: err?.name ?? 'TelemetrySendError',
message: err?.message ?? 'Telemetry send failed',
},
};
}
}
private async sendHttp(batch: TelemetryBatch): Promise<TelemetryAck> {
const url = new URL(
'/api/telemetry/collect',
this.context.officialEndpoint
);
const abortController = new AbortController();
const timeoutId = setTimeout(() => {
abortController.abort();
}, 10000);
const response = await fetch(url, {
method: 'POST',
headers: {
'content-type': 'application/json',
'x-affine-version': BUILD_CONFIG.appVersion,
},
body: JSON.stringify(batch),
signal: abortController.signal,
});
if (!response.ok) {
const text = await response.text().catch(() => '');
throw new Error(
`Telemetry HTTP failed with ${response.status}: ${text || 'unknown error'}`
);
} else {
clearTimeout(timeoutId);
}
const payload = (await response.json().catch(() => null)) as TelemetryAck;
if (!payload || typeof payload.ok !== 'boolean') {
throw new Error('Invalid telemetry response');
}
return payload;
}
private async sendWs(batch: TelemetryBatch): Promise<TelemetryAck> {
const socketConnection = this.ensureSocketConnection();
socketConnection.connect();
await socketConnection.waitForConnected();
const res = await socketConnection.inner.socket.emitWithAck(
'telemetry:batch',
batch
);
if ('error' in res) {
return {
ok: false,
error: {
name: res.error.name ?? 'TelemetryWebsocketError',
message: res.error.message ?? 'Telemetry websocket error',
},
};
}
return res.data as TelemetryAck;
}
private scheduleFlush(force: boolean) {
if (force) {
this.clearRetry();
}
if (this.retryTimer && !force) {
return;
}
queueMicrotask(() => {
this.flush().catch(() => {
return;
});
});
}
private recordFailure(message: string) {
this.lastError = message;
const delay = this.nextBackoffDelay();
this.retryAttempt += 1;
this.nextRetryAt = Date.now() + delay;
this.clearRetry();
this.retryTimer = setTimeout(() => {
this.retryTimer = null;
this.flush().catch(() => {
return;
});
}, delay);
}
private resetRetry() {
this.retryAttempt = 0;
this.nextRetryAt = undefined;
this.lastError = undefined;
this.clearRetry();
}
private clearRetry() {
if (this.retryTimer) {
clearTimeout(this.retryTimer);
this.retryTimer = null;
}
}
private nextBackoffDelay() {
const exp = Math.min(this.retryAttempt, 10);
const base = this.retryBaseMs * Math.pow(2, exp);
const delay = Math.min(this.retryMaxMs, base);
const jitter = Math.random() * delay * 0.2;
return delay + jitter;
}
private ensureSocketConnection() {
if (
this.socketConnection &&
this.socketEndpoint === this.context.officialEndpoint
) {
return this.socketConnection;
}
if (this.socketConnection) {
this.socketConnection.disconnect(true);
}
this.socketEndpoint = this.context.officialEndpoint;
this.socketConnection = new SocketConnection(
this.context.officialEndpoint,
this.context.isSelfHosted
);
return this.socketConnection;
}
private updateSocketConnection() {
const useWebsocket = this.context.isAuthed && !this.context.isSelfHosted;
if (!useWebsocket) {
if (this.socketConnection) {
this.socketConnection.disconnect(true);
}
this.socketConnection = undefined;
this.socketEndpoint = undefined;
return;
}
this.ensureSocketConnection();
}
}
packages/common/nbstore/src/telemetry/queue.ts
+132
View File
@@ -0,0 +1,132 @@
import { type DBSchema, openDB } from 'idb';
import type { TelemetryEvent } from './types';
interface TelemetryQueueDB extends DBSchema {
events: {
key: number;
value: {
id?: number;
event: TelemetryEvent;
size: number;
addedAt: number;
};
};
}
export type TelemetryQueueItem = {
id: number;
event: TelemetryEvent;
size: number;
addedAt: number;
};
export class TelemetryQueue {
private readonly dbPromise = openDB<TelemetryQueueDB>('affine-telemetry', 1, {
upgrade(db) {
if (!db.objectStoreNames.contains('events')) {
db.createObjectStore('events', { keyPath: 'id', autoIncrement: true });
}
},
});
private readonly ready = this.load();
private items: TelemetryQueueItem[] = [];
private totalSize = 0;
constructor(
private readonly maxEntries: number,
private readonly maxBytes: number
) {}
get size() {
return this.items.length;
}
async enqueue(event: TelemetryEvent) {
await this.ready;
const size = estimateSize(event);
const addedAt = Date.now();
const db = await this.dbPromise;
const id = await db.add('events', { event, size, addedAt });
const item = { id: Number(id), event, size, addedAt };
this.items.push(item);
this.totalSize += size;
await this.enforceLimits();
}
async peek(limit: number) {
await this.ready;
return this.items.slice(0, limit);
}
async remove(ids: number[]) {
await this.ready;
if (!ids.length) {
return;
}
const db = await this.dbPromise;
const tx = db.transaction('events', 'readwrite');
await Promise.all(ids.map(id => tx.store.delete(id)));
await tx.done;
const removeSet = new Set(ids);
this.items = this.items.filter(item => {
if (removeSet.has(item.id)) {
this.totalSize -= item.size;
return false;
}
return true;
});
}
private async load() {
const db = await this.dbPromise;
const all = await db.getAll('events');
this.items = all
.filter(item => typeof item.id === 'number')
.map(item => ({
id: item.id as number,
event: item.event,
size: item.size,
addedAt: item.addedAt,
}))
.sort((a, b) => a.id - b.id);
this.totalSize = this.items.reduce((sum, item) => sum + item.size, 0);
}
private async enforceLimits() {
if (
this.items.length <= this.maxEntries &&
this.totalSize <= this.maxBytes
) {
return;
}
const db = await this.dbPromise;
const tx = db.transaction('events', 'readwrite');
const deletions: Promise<unknown>[] = [];
while (
this.items.length > this.maxEntries ||
this.totalSize > this.maxBytes
) {
const removed = this.items.shift();
if (!removed) {
break;
}
this.totalSize -= removed.size;
deletions.push(tx.store.delete(removed.id));
}
await Promise.all(deletions);
await tx.done;
}
}
function estimateSize(event: TelemetryEvent) {
try {
return JSON.stringify(event).length;
} catch {
return 0;
}
}
packages/common/nbstore/src/telemetry/types.ts
+50
View File
@@ -0,0 +1,50 @@
export type TelemetryEvent = {
schemaVersion: 1;
eventName: string;
params?: Record<string, unknown>;
userProperties?: Record<string, unknown>;
userId?: string;
clientId: string;
sessionId?: string;
eventId: string;
timestampMicros?: number;
context?: {
appVersion?: string;
editorVersion?: string;
environment?: string;
distribution?: string;
channel?: 'stable' | 'beta' | 'internal' | 'canary';
isDesktop?: boolean;
isMobile?: boolean;
locale?: string;
timezone?: string;
url?: string;
referrer?: string;
};
};
export type TelemetryBatch = {
schemaVersion: 1;
transport: 'http' | 'ws';
sentAt: number;
events: TelemetryEvent[];
};
export type TelemetryAck =
| { ok: true; accepted: number; dropped: number }
| { ok: false; error: { name: string; message: string } };
export interface TelemetryContext {
isAuthed: boolean;
isSelfHosted: boolean;
channel: 'stable' | 'beta' | 'internal' | 'canary';
userId?: string;
userProperties?: Record<string, unknown>;
officialEndpoint: string;
}
export interface TelemetryQueueState {
size: number;
lastError?: string;
nextRetryAt?: number;
}
packages/common/nbstore/src/worker/client.ts
+35 -1
View File
@@ -28,6 +28,12 @@ import type { AwarenessSync } from '../sync/awareness';
import type { BlobSync } from '../sync/blob';
import type { DocSync } from '../sync/doc';
import type { IndexerPreferOptions, IndexerSync } from '../sync/indexer';
import type {
TelemetryAck,
TelemetryContext,
TelemetryEvent,
TelemetryQueueState,
} from '../telemetry/types';
import type { StoreInitOptions, WorkerManagerOps, WorkerOps } from './ops';
export type { StoreInitOptions as WorkerInitOptions } from './ops';
@@ -41,7 +47,11 @@ export class StoreManagerClient {
}
>();
constructor(private readonly client: OpClient<WorkerManagerOps>) {}
constructor(private readonly client: OpClient<WorkerManagerOps>) {
this.telemetry = new TelemetryClient(this.client);
}
readonly telemetry: TelemetryClient;
open(key: string, options: StoreInitOptions) {
const { port1, port2 } = new MessageChannel();
@@ -104,6 +114,30 @@ export class StoreManagerClient {
}
}
class TelemetryClient {
constructor(private readonly client: OpClient<WorkerManagerOps>) {}
setContext(context: TelemetryContext): Promise<void> {
return this.client.call('telemetry.setContext', context);
}
track(event: TelemetryEvent): Promise<{ queued: boolean }> {
return this.client.call('telemetry.track', event);
}
pageview(event: TelemetryEvent): Promise<{ queued: boolean }> {
return this.client.call('telemetry.pageview', event);
}
flush(): Promise<TelemetryAck> {
return this.client.call('telemetry.flush');
}
getQueueState(): Promise<TelemetryQueueState> {
return this.client.call('telemetry.getQueueState');
}
}
export class StoreClient {
constructor(private readonly client: OpClient<WorkerOps>) {
this.docStorage = new WorkerDocStorage(this.client);
packages/common/nbstore/src/worker/consumer.ts
+7
View File
@@ -6,6 +6,7 @@ import { SpaceStorage } from '../storage';
import type { AwarenessRecord } from '../storage/awareness';
import { Sync } from '../sync';
import type { PeerStorageOptions } from '../sync/types';
import { TelemetryManager } from '../telemetry/manager';
import { MANUALLY_STOP } from '../utils/throw-if-aborted';
import type { StoreInitOptions, WorkerManagerOps, WorkerOps } from './ops';
@@ -338,6 +339,7 @@ export class StoreManagerConsumer {
string,
{ store: StoreConsumer; refCount: number }
>();
private readonly telemetry = new TelemetryManager();
constructor(
private readonly availableStorageImplementations: StorageConstructor[]
@@ -386,6 +388,11 @@ export class StoreManagerConsumer {
workerDisposer();
this.storeDisposers.delete(key);
},
'telemetry.setContext': context => this.telemetry.setContext(context),
'telemetry.track': event => this.telemetry.track(event),
'telemetry.pageview': event => this.telemetry.pageview(event),
'telemetry.flush': () => this.telemetry.flush(),
'telemetry.getQueueState': () => this.telemetry.getQueueState(),
});
}
}
packages/common/nbstore/src/worker/ops.ts
+11
View File
@@ -16,6 +16,12 @@ import type { AwarenessRecord } from '../storage/awareness';
import type { BlobSyncBlobState, BlobSyncState } from '../sync/blob';
import type { DocSyncDocState, DocSyncState } from '../sync/doc';
import type { IndexerDocSyncState, IndexerSyncState } from '../sync/indexer';
import type {
TelemetryAck,
TelemetryContext,
TelemetryEvent,
TelemetryQueueState,
} from '../telemetry/types';
type StorageInitOptions = Values<{
[key in keyof AvailableStorageImplementations]: {
@@ -178,4 +184,9 @@ export type WorkerManagerOps = {
string,
];
close: [string, void];
'telemetry.setContext': [TelemetryContext, void];
'telemetry.track': [TelemetryEvent, { queued: boolean }];
'telemetry.pageview': [TelemetryEvent, { queued: boolean }];
'telemetry.flush': [void, TelemetryAck];
'telemetry.getQueueState': [void, TelemetryQueueState];
};
packages/frontend/admin/src/config.json
+28
View File
@@ -245,6 +245,34 @@
"env": "DOC_SERVICE_ENDPOINT"
}
},
"telemetry": {
"allowedOrigin": {
"type": "Array",
"desc": "Allowed origins for telemetry collection."
},
"ga4.measurementId": {
"type": "String",
"desc": "GA4 Measurement ID for Measurement Protocol.",
"env": "GA4_MEASUREMENT_ID"
},
"ga4.apiSecret": {
"type": "String",
"desc": "GA4 API secret for Measurement Protocol.",
"env": "GA4_API_SECRET"
},
"dedupe.ttlHours": {
"type": "Number",
"desc": "Telemetry dedupe TTL in hours."
},
"dedupe.maxEntries": {
"type": "Number",
"desc": "Telemetry dedupe max entries."
},
"batch.maxEvents": {
"type": "Number",
"desc": "Max events per telemetry batch."
}
},
"client": {
"versionControl.enabled": {
"type": "Boolean",
packages/frontend/apps/android/package.json
+1
View File
@@ -16,6 +16,7 @@
"@affine/env": "workspace:*",
"@affine/i18n": "workspace:*",
"@affine/nbstore": "workspace:*",
"@affine/track": "workspace:*",
"@blocksuite/affine": "workspace:*",
"@blocksuite/icons": "^2.2.17",
"@capacitor/android": "^7.0.0",
packages/frontend/apps/android/src/app.tsx
+2
View File
@@ -31,6 +31,7 @@ import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspac
import { getWorkerUrl } from '@affine/env/worker';
import { I18n } from '@affine/i18n';
import { StoreManagerClient } from '@affine/nbstore/worker/client';
import { setTelemetryTransport } from '@affine/track';
import { Container } from '@blocksuite/affine/global/di';
import {
docLinkBaseURLMiddleware,
@@ -56,6 +57,7 @@ import { NbStoreNativeDBApis } from './plugins/nbstore';
import { writeEndpointToken } from './proxy';
const storeManagerClient = createStoreManagerClient();
setTelemetryTransport(storeManagerClient.telemetry);
window.addEventListener('beforeunload', () => {
storeManagerClient.dispose();
});
packages/frontend/apps/android/tsconfig.json
+1
View File
@@ -12,6 +12,7 @@
{ "path": "../../../common/env" },
{ "path": "../../i18n" },
{ "path": "../../../common/nbstore" },
{ "path": "../../track" },
{ "path": "../../../../blocksuite/affine/all" },
{ "path": "../../../common/infra" }
]
packages/frontend/apps/electron-renderer/src/app/effects/store-manager.ts
+2
View File
@@ -1,6 +1,7 @@
import { NbstoreProvider } from '@affine/core/modules/storage';
import { apis } from '@affine/electron-api';
import { StoreManagerClient } from '@affine/nbstore/worker/client';
import { setTelemetryTransport } from '@affine/track';
import type { Framework } from '@toeverything/infra';
import { OpClient } from '@toeverything/infra/op';
import { v4 as uuid } from 'uuid';
@@ -43,6 +44,7 @@ function createStoreManagerClient() {
export function setupStoreManager(framework: Framework) {
const storeManagerClient = createStoreManagerClient();
setTelemetryTransport(storeManagerClient.telemetry);
window.addEventListener('beforeunload', () => {
storeManagerClient.dispose();
});
packages/frontend/apps/ios/package.json
+1
View File
@@ -20,6 +20,7 @@
"@affine/graphql": "workspace:*",
"@affine/i18n": "workspace:*",
"@affine/nbstore": "workspace:*",
"@affine/track": "workspace:*",
"@blocksuite/affine": "workspace:*",
"@blocksuite/icons": "^2.2.17",
"@capacitor/app": "^7.0.0",
packages/frontend/apps/ios/src/app.tsx
+2
View File
@@ -45,6 +45,7 @@ import {
} from '@affine/graphql';
import { I18n } from '@affine/i18n';
import { StoreManagerClient } from '@affine/nbstore/worker/client';
import { setTelemetryTransport } from '@affine/track';
import { Container } from '@blocksuite/affine/global/di';
import {
docLinkBaseURLMiddleware,
@@ -74,6 +75,7 @@ import { writeEndpointToken } from './proxy';
import { enableNavigationGesture$ } from './web-navigation-control';
const storeManagerClient = createStoreManagerClient();
setTelemetryTransport(storeManagerClient.telemetry);
window.addEventListener('beforeunload', () => {
storeManagerClient.dispose();
});
packages/frontend/apps/ios/tsconfig.json
+1
View File
@@ -13,6 +13,7 @@
{ "path": "../../../common/graphql" },
{ "path": "../../i18n" },
{ "path": "../../../common/nbstore" },
{ "path": "../../track" },
{ "path": "../../../../blocksuite/affine/all" },
{ "path": "../../../common/infra" },
{ "path": "../../../../tools/cli" },
packages/frontend/apps/mobile/package.json
+1
View File
@@ -14,6 +14,7 @@
"@affine/env": "workspace:*",
"@affine/i18n": "workspace:*",
"@affine/nbstore": "workspace:*",
"@affine/track": "workspace:*",
"@blocksuite/affine": "workspace:*",
"@blocksuite/icons": "^2.2.17",
"@sentry/react": "^9.47.1",
packages/frontend/apps/mobile/src/app.tsx
+2
View File
@@ -16,6 +16,7 @@ import { configureBrowserWorkbenchModule } from '@affine/core/modules/workbench'
import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine';
import { getWorkerUrl } from '@affine/env/worker';
import { StoreManagerClient } from '@affine/nbstore/worker/client';
import { setTelemetryTransport } from '@affine/track';
import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra';
import { OpClient } from '@toeverything/infra/op';
import { Suspense } from 'react';
@@ -31,6 +32,7 @@ if (window.SharedWorker) {
const worker = new Worker(workerUrl);
storeManagerClient = new StoreManagerClient(new OpClient(worker));
}
setTelemetryTransport(storeManagerClient.telemetry);
window.addEventListener('beforeunload', () => {
storeManagerClient.dispose();
});
packages/frontend/apps/mobile/tsconfig.json
+1
View File
@@ -12,6 +12,7 @@
{ "path": "../../../common/env" },
{ "path": "../../i18n" },
{ "path": "../../../common/nbstore" },
{ "path": "../../track" },
{ "path": "../../../../blocksuite/affine/all" },
{ "path": "../../../common/infra" }
]
packages/frontend/apps/web/src/app.tsx
+2
View File
@@ -14,6 +14,7 @@ import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspac
import createEmotionCache from '@affine/core/utils/create-emotion-cache';
import { getWorkerUrl } from '@affine/env/worker';
import { StoreManagerClient } from '@affine/nbstore/worker/client';
import { setTelemetryTransport } from '@affine/track';
import { CacheProvider } from '@emotion/react';
import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra';
import { OpClient } from '@toeverything/infra/op';
@@ -38,6 +39,7 @@ if (
const worker = new Worker(workerUrl);
storeManagerClient = new StoreManagerClient(new OpClient(worker));
}
setTelemetryTransport(storeManagerClient.telemetry);
window.addEventListener('beforeunload', () => {
storeManagerClient.dispose();
});
packages/frontend/core/src/bootstrap/telemetry.ts
+3 -5
View File
@@ -1,4 +1,4 @@
import { ga4, sentry, tracker } from '@affine/track';
import { sentry, tracker } from '@affine/track';
import { APP_SETTINGS_STORAGE_KEY } from '@toeverything/infra/atom';
tracker.init();
@@ -14,10 +14,8 @@ if (typeof localStorage !== 'undefined') {
}
if (!enabled) {
// NOTE(@forehalo): mixpanel will read local storage flag and doesn't need to be manually opt_out at startup time.
// see: https://docs.mixpanel.com/docs/privacy/protecting-user-data
// mixpanel.opt_out_tracking();
// NOTE: telemetry setting is respected by tracker and sentry.
sentry.disable();
ga4.setEnabled(false);
tracker.opt_out_tracking();
}
}
packages/frontend/core/src/modules/cloud/constant.ts
+29
View File
@@ -187,3 +187,32 @@ export const BUILD_IN_SERVERS: (ServerMetadata & { config: ServerConfig })[] =
},
]
: [];
export type TelemetryChannel =
| 'stable'
| 'beta'
| 'internal'
| 'canary'
| 'local';
const OFFICIAL_TELEMETRY_ENDPOINTS: Record<TelemetryChannel, string> = {
stable: 'https://app.affine.pro',
beta: 'https://insider.affine.pro',
internal: 'https://insider.affine.pro',
canary: 'https://affine.fail',
local: 'http://localhost:8080',
};
export function getOfficialTelemetryEndpoint(
channel = BUILD_CONFIG.appBuildType
): string {
if (BUILD_CONFIG.debug) {
return BUILD_CONFIG.isNative
? OFFICIAL_TELEMETRY_ENDPOINTS.local
: location.origin;
} else if (['beta', 'internal', 'canary', 'stable'].includes(channel)) {
return OFFICIAL_TELEMETRY_ENDPOINTS[channel];
}
return OFFICIAL_TELEMETRY_ENDPOINTS.stable;
}
packages/frontend/core/src/modules/telemetry/services/telemetry.ts
+27 -1
View File
@@ -1,12 +1,16 @@
import { shallowEqual } from '@affine/component';
import { DebugLogger } from '@affine/debug';
import { ServerDeploymentType } from '@affine/graphql';
import { tracker } from '@affine/track';
import { flushTelemetry, setTelemetryContext, tracker } from '@affine/track';
import { LiveData, OnEvent, Service } from '@toeverything/infra';
import type { AuthAccountInfo, Server, ServersService } from '../../cloud';
import { getOfficialTelemetryEndpoint } from '../../cloud/constant';
import type { GlobalContextService } from '../../global-context';
import { ApplicationStarted } from '../../lifecycle';
const logger = new DebugLogger('telemetry-service');
@OnEvent(ApplicationStarted, e => e.onApplicationStart)
export class TelemetryService extends Service {
private readonly disposableFns: (() => void)[] = [];
@@ -46,6 +50,21 @@ export class TelemetryService extends Service {
let prevSelfHosted: boolean | undefined = undefined;
const unsubscribe = this.currentAccount$.subscribe(
({ account, selfHosted }) => {
const channel =
BUILD_CONFIG.appBuildType === 'beta' ||
BUILD_CONFIG.appBuildType === 'internal' ||
BUILD_CONFIG.appBuildType === 'canary' ||
BUILD_CONFIG.appBuildType === 'stable'
? BUILD_CONFIG.appBuildType
: 'stable';
setTelemetryContext({
isAuthed: !!account,
isSelfHosted: !!selfHosted,
channel,
officialEndpoint: getOfficialTelemetryEndpoint(channel),
});
if (prevAccount) {
tracker.reset();
}
@@ -64,6 +83,13 @@ export class TelemetryService extends Service {
$name: account.label,
$avatar: account.avatar,
});
void flushTelemetry().catch(error => {
logger.error('failed to flush telemetry after login', error);
});
} else if (prevAccount) {
void flushTelemetry().catch(error => {
logger.error('failed to flush telemetry after logout', error);
});
}
}
);
packages/frontend/track/package.json
+1 -2
View File
@@ -9,11 +9,10 @@
"dependencies": {
"@affine/debug": "workspace:*",
"@sentry/react": "^9.47.1",
"mixpanel-browser": "^2.56.0",
"nanoid": "^5.1.6",
"react-router-dom": "^6.30.3"
},
"devDependencies": {
"@types/mixpanel-browser": "^2.50.2",
"@types/react": "^19.0.1",
"vitest": "^3.2.4"
}
packages/frontend/track/src/ga4.ts
-333
View File
@@ -1,333 +0,0 @@
type Scalar = string | number;
declare global {
interface Window {
gtag?: (...args: any[]) => void;
dataLayer?: unknown[];
}
}
const EVENT_NAME_RE = /^[A-Za-z][A-Za-z0-9_]{0,39}$/;
const PARAM_NAME_RE = /^[A-Za-z][A-Za-z0-9_]{0,39}$/;
const USER_PROP_NAME_RE = /^[A-Za-z][A-Za-z0-9_]{0,23}$/;
const GA4_MEASUREMENT_ID = BUILD_CONFIG.GA4_MEASUREMENT_ID;
const GTAG_SCRIPT_ID = 'ga4-gtag';
const EVENT_NAME_ALIAS: Record<string, string> = {
track_pageview: 'page_view',
};
const PARAM_RENAME_MAP = new Map<string, string>([
['page', 'ui_page'],
['segment', 'ui_segment'],
['module', 'ui_module'],
['arg', 'ui_arg'],
['control', 'ui_control'],
['option', 'ui_option'],
['key', 'setting_key'],
['value', 'setting_value'],
['docId', 'doc_id'],
['workspaceId', 'workspace_id'],
['serverId', 'server_id'],
['docType', 'doc_type'],
['docCount', 'doc_count'],
['unreadCount', 'unread_count'],
['withAttachment', 'with_attachment'],
['withMention', 'with_mention'],
['appName', 'app_name'],
['recurring', 'billing_cycle'],
['plan', 'plan_name'],
['time', 'duration_ms'],
['error', 'error_code'],
['status', 'result'],
['success', 'result'],
['to', 'target'],
['on', 'enabled'],
]);
const USER_PROP_RENAME_MAP = new Map<string, string>([
['appVersion', 'app_version'],
['editorVersion', 'editor_version'],
['environment', 'environment'],
['isDesktop', 'is_desktop'],
['distribution', 'distribution'],
['isSelfHosted', 'is_self_hosted'],
['ai', 'ai_enabled'],
['pro', 'plan_tier'],
['quota', 'quota_tier'],
]);
const DROP_PARAM_SEGMENTS = new Set(['other', 'instruction', 'operation']);
const DROP_MAPPED_PARAMS = new Set(['doc_id', 'workspace_id', 'server_id']);
const PRIORITY_KEYS = [
'ui_page',
'ui_segment',
'ui_module',
'ui_control',
'ui_option',
'ui_arg',
'type',
'method',
'mode',
'plan_name',
'billing_cycle',
'role',
'result',
'error_code',
'category',
'doc_type',
'item',
'action',
'target',
'enabled',
'setting_key',
'setting_value',
'duration_ms',
'doc_count',
'unread_count',
'with_attachment',
'with_mention',
];
let enabled = true;
let configured = false;
function ensureGtagLoaded(): boolean {
if (!enabled || !GA4_MEASUREMENT_ID) return false;
if (typeof window === 'undefined' || typeof document === 'undefined') {
return false;
}
if (!window.gtag) {
window.dataLayer = window.dataLayer || [];
window.gtag = (...args: any[]) => {
window.dataLayer?.push(args);
};
}
if (!document.getElementById(GTAG_SCRIPT_ID)) {
const script = document.createElement('script');
script.id = GTAG_SCRIPT_ID;
script.async = true;
script.src = `https://www.googletagmanager.com/gtag/js?id=${encodeURIComponent(
GA4_MEASUREMENT_ID
)}`;
(document.head || document.body || document.documentElement).appendChild(
script
);
}
if (!configured) {
configured = true;
window.gtag('js', new Date());
window.gtag('config', GA4_MEASUREMENT_ID, { send_page_view: false });
}
return true;
}
function toSnakeCase(input: string): string {
return input
.replace(/([a-z0-9])([A-Z])/g, '$1_$2')
.replace(/([A-Z]+)([A-Z][a-z0-9]+)/g, '$1_$2')
.replace(/\s+/g, '_')
.replace(/[^a-zA-Z0-9_]/g, '_')
.replace(/__+/g, '_')
.toLowerCase();
}
function toScalar(v: unknown): Scalar | undefined {
if (v === null || v === undefined) return undefined;
if (typeof v === 'number' && Number.isFinite(v)) return v;
if (typeof v === 'boolean') return v ? 1 : 0;
if (typeof v === 'string') return v.length > 100 ? v.slice(0, 100) : v;
if (v instanceof Date) return v.toISOString();
try {
const s = JSON.stringify(v);
return s.length > 100 ? s.slice(0, 100) : s;
} catch {
return undefined;
}
}
function normalizeValue(key: string, value: unknown): Scalar | undefined {
if (key === 'result' && typeof value === 'boolean') {
return value ? 'success' : 'failure';
}
if (key === 'enabled' && typeof value === 'boolean') {
return value ? 'on' : 'off';
}
return toScalar(value);
}
function isPlainObject(value: unknown): value is Record<string, unknown> {
if (!value || typeof value !== 'object') return false;
if (value instanceof Date) return false;
if (Array.isArray(value)) return false;
return Object.getPrototypeOf(value) === Object.prototype;
}
function flattenProps(
input: Record<string, unknown>,
prefix = ''
): Record<string, unknown> {
const out: Record<string, unknown> = {};
for (const [key, value] of Object.entries(input)) {
const path = prefix ? `${prefix}.${key}` : key;
if (isPlainObject(value)) {
const nested = flattenProps(value, path);
Object.assign(out, nested);
} else {
out[path] = value;
}
}
return out;
}
function mapParamKey(path: string): string {
const segments = path.split('.');
const mappedSegments = segments.map(
segment => PARAM_RENAME_MAP.get(segment) ?? segment
);
return toSnakeCase(mappedSegments.join('_'));
}
function mapEventName(name: string): string {
const alias = EVENT_NAME_ALIAS[name];
return toSnakeCase(alias ?? name);
}
function shouldDropPath(path: string): boolean {
const segments = path.split('.');
return segments.some(segment => DROP_PARAM_SEGMENTS.has(segment));
}
function sanitizeParams(
input: Record<string, unknown>,
maxParams = 25
): Record<string, Scalar> {
const flattened = flattenProps(input);
const mappedEntries: Array<[string, Scalar]> = [];
for (const [path, value] of Object.entries(flattened)) {
if (shouldDropPath(path)) continue;
const mappedKey = mapParamKey(path);
if (!mappedKey || !PARAM_NAME_RE.test(mappedKey)) continue;
if (DROP_MAPPED_PARAMS.has(mappedKey)) continue;
const normalized = normalizeValue(mappedKey, value);
if (normalized === undefined) continue;
mappedEntries.push([mappedKey, normalized]);
}
const prioritySet = new Set(PRIORITY_KEYS);
mappedEntries.sort((a, b) => {
const aPriority = prioritySet.has(a[0]);
const bPriority = prioritySet.has(b[0]);
if (aPriority === bPriority) return 0;
return aPriority ? -1 : 1;
});
const out: Record<string, Scalar> = {};
for (const [key, value] of mappedEntries) {
if (Object.keys(out).length >= maxParams) break;
if (key in out) continue;
out[key] = value;
}
return out;
}
function mapUserPropKey(key: string): string | undefined {
if (key.startsWith('$')) return undefined;
const mapped = USER_PROP_RENAME_MAP.get(key) ?? key;
return toSnakeCase(mapped);
}
function sanitizeUserProperties(
props: Record<string, unknown>
): Record<string, string> {
const sanitized: Record<string, string> = {};
for (const [key, value] of Object.entries(props)) {
const mappedKey = mapUserPropKey(key);
if (!mappedKey || !USER_PROP_NAME_RE.test(mappedKey)) continue;
let mappedValue = value;
if (key === 'pro' && typeof value === 'boolean') {
mappedValue = value ? 'pro' : 'free';
}
const scalar = toScalar(mappedValue);
if (scalar === undefined) continue;
const stringValue = String(scalar);
sanitized[mappedKey] =
stringValue.length > 36 ? stringValue.slice(0, 36) : stringValue;
}
return sanitized;
}
export const ga4 = {
setEnabled(v: boolean) {
enabled = v;
if (enabled) {
ensureGtagLoaded();
}
},
reset() {
if (!ensureGtagLoaded()) return;
window.gtag?.('set', 'user_id', undefined);
window.gtag?.('set', 'user_properties', {});
},
setUserId(userId?: string) {
if (!ensureGtagLoaded()) return;
window.gtag?.('set', 'user_id', userId ? String(userId) : undefined);
},
setUserProperties(props: Record<string, unknown>) {
if (!ensureGtagLoaded()) return;
const sanitized = sanitizeUserProperties(props);
if (Object.keys(sanitized).length === 0) return;
window.gtag?.('set', 'user_properties', sanitized);
},
track(eventName: string, props: Record<string, unknown> = {}) {
if (!ensureGtagLoaded()) return;
const mappedEvent = mapEventName(eventName);
if (!EVENT_NAME_RE.test(mappedEvent)) return;
const sanitized = sanitizeParams(props);
window.gtag?.('event', mappedEvent, sanitized);
},
pageview(props: Record<string, unknown> = {}) {
if (!ensureGtagLoaded()) return;
const pageLocation =
typeof props.location === 'string' ? props.location : location.href;
let pagePath = location.pathname + location.search;
if (typeof pageLocation === 'string') {
try {
const url = new URL(pageLocation, location.origin);
pagePath = url.pathname + url.search;
} catch {
pagePath = location.pathname + location.search;
}
}
const customParams = { ...props };
delete customParams.location;
const sanitized = sanitizeParams(customParams, 22);
window.gtag?.('event', 'page_view', {
page_location: pageLocation,
page_path: pagePath,
page_title: document.title,
...sanitized,
});
},
};
packages/frontend/track/src/index.ts
+16 -3
View File
@@ -1,11 +1,24 @@
import { enableAutoTrack, makeTracker } from './auto';
import { type EventArgs, type Events } from './events';
import { ga4 } from './ga4';
import { tracker } from './mixpanel';
import { sentry } from './sentry';
import {
flushTelemetry,
setTelemetryContext,
setTelemetryTransport,
} from './telemetry';
import { tracker } from './tracker';
export const track = makeTracker((event, props) => {
tracker.track(event, props);
});
export { enableAutoTrack, type EventArgs, type Events, ga4, sentry, tracker };
export {
enableAutoTrack,
type EventArgs,
type Events,
flushTelemetry,
sentry,
setTelemetryContext,
setTelemetryTransport,
tracker,
};
export default track;
packages/frontend/track/src/mixpanel.ts
-129
View File
@@ -1,129 +0,0 @@
import { DebugLogger } from '@affine/debug';
import type { Dict, OverridedMixpanel } from 'mixpanel-browser';
import mixpanelBrowser from 'mixpanel-browser';
import { ga4 } from './ga4';
const logger = new DebugLogger('mixpanel');
type Middleware = (
name: string,
properties?: Record<string, unknown>
) => Record<string, unknown>;
function createMixpanel() {
let mixpanel;
if (BUILD_CONFIG.MIXPANEL_TOKEN) {
mixpanelBrowser.init(BUILD_CONFIG.MIXPANEL_TOKEN || '', {
track_pageview: true,
persistence: 'localStorage',
api_host: 'https://telemetry.affine.run',
ignore_dnt: true,
});
mixpanel = mixpanelBrowser;
} else {
mixpanel = new Proxy(
function () {} as unknown as OverridedMixpanel,
createProxyHandler()
);
}
const middlewares = new Set<Middleware>();
const wrapped = {
init() {
const defaultProps = {
appVersion: BUILD_CONFIG.appVersion,
environment: BUILD_CONFIG.appBuildType,
editorVersion: BUILD_CONFIG.editorVersion,
isDesktop: BUILD_CONFIG.isElectron,
distribution: BUILD_CONFIG.distribution,
};
this.register(defaultProps);
},
// provide a way to override the default properties
register(props: Dict) {
logger.debug('register with', props);
mixpanel.register(props);
ga4.setUserProperties(props);
},
reset() {
mixpanel.reset();
ga4.reset();
this.init();
},
track(event_name: string, properties?: Record<string, any>) {
const middlewareProperties = Array.from(middlewares).reduce(
(acc, middleware) => {
return middleware(event_name, acc);
},
properties as Record<string, unknown>
);
logger.debug('track', event_name, middlewareProperties);
mixpanel.track(event_name as string, middlewareProperties);
ga4.track(event_name as string, middlewareProperties);
},
middleware(cb: Middleware): () => void {
middlewares.add(cb);
return () => {
middlewares.delete(cb);
};
},
opt_out_tracking() {
mixpanel.opt_out_tracking();
ga4.setEnabled(false);
},
opt_in_tracking() {
mixpanel.opt_in_tracking();
ga4.setEnabled(true);
},
has_opted_in_tracking() {
mixpanel.has_opted_in_tracking();
},
has_opted_out_tracking() {
mixpanel.has_opted_out_tracking();
},
identify(unique_id?: string) {
mixpanel.identify(unique_id);
ga4.setUserId(unique_id);
},
get people() {
const people = mixpanel.people;
return {
set: (props: Dict) => {
people?.set?.(props);
ga4.setUserProperties(props);
},
} as typeof mixpanel.people;
},
track_pageview(properties?: { location?: string }) {
const middlewareProperties = Array.from(middlewares).reduce(
(acc, middleware) => {
return middleware('track_pageview', acc);
},
properties as Record<string, unknown>
);
logger.debug('track_pageview', middlewareProperties);
mixpanel.track_pageview(middlewareProperties);
ga4.pageview(middlewareProperties);
},
};
return wrapped;
}
export const tracker = createMixpanel();
function createProxyHandler() {
const handler = {
get: () => {
return new Proxy(
function () {} as unknown as OverridedMixpanel,
createProxyHandler()
);
},
apply: () => {},
} as ProxyHandler<OverridedMixpanel>;
return handler;
}
packages/frontend/track/src/telemetry.ts
+155
View File
@@ -0,0 +1,155 @@
import { DebugLogger } from '@affine/debug';
export type TelemetryEvent = {
schemaVersion: 1;
eventName: string;
params?: Record<string, unknown>;
userProperties?: Record<string, unknown>;
userId?: string;
clientId: string;
sessionId?: string;
eventId: string;
timestampMicros?: number;
context?: {
appVersion?: string;
editorVersion?: string;
environment?: string;
distribution?: string;
channel?: 'stable' | 'beta' | 'internal' | 'canary';
isDesktop?: boolean;
isMobile?: boolean;
locale?: string;
timezone?: string;
url?: string;
referrer?: string;
};
};
export type TelemetryContext = {
isAuthed: boolean;
isSelfHosted: boolean;
channel: 'stable' | 'beta' | 'internal' | 'canary';
userId?: string;
userProperties?: Record<string, unknown>;
officialEndpoint: string;
};
export type TelemetryAck =
| { ok: true; accepted: number; dropped: number }
| { ok: false; error: { name: string; message: string } };
export type TelemetryTransport = {
setContext: (context: TelemetryContext) => Promise<void> | void;
track: (event: TelemetryEvent) => Promise<{ queued: boolean }> | void;
pageview?: (event: TelemetryEvent) => Promise<{ queued: boolean }> | void;
flush?: () => Promise<TelemetryAck> | void;
};
type TelemetryContextUpdate = Partial<TelemetryContext> & {
userProperties?: Record<string, unknown>;
};
type TelemetryContextUpdateOptions = {
replaceUserProperties?: boolean;
};
const logger = new DebugLogger('telemetry');
const pendingEvents: TelemetryEvent[] = [];
const PENDING_LIMIT = 500;
const defaultChannel =
BUILD_CONFIG.appBuildType === 'beta' ||
BUILD_CONFIG.appBuildType === 'internal' ||
BUILD_CONFIG.appBuildType === 'canary' ||
BUILD_CONFIG.appBuildType === 'stable'
? BUILD_CONFIG.appBuildType
: 'stable';
let context: TelemetryContext = {
isAuthed: false,
isSelfHosted: false,
channel: defaultChannel,
officialEndpoint: '',
userProperties: {},
};
let transport: TelemetryTransport | null = null;
export function setTelemetryTransport(next: TelemetryTransport | null) {
transport = next;
if (!transport) {
return;
}
applyTransportContext(context);
flushPending().catch(error => {
logger.error('failed to flush pending telemetry events', error);
});
}
export function setTelemetryContext(
update: TelemetryContextUpdate,
options: TelemetryContextUpdateOptions = {}
) {
const nextUserProps = options.replaceUserProperties
? (update.userProperties ?? {})
: {
...(context.userProperties ?? {}),
...(update.userProperties ?? {}),
};
context = {
...context,
...update,
userProperties: nextUserProps,
};
applyTransportContext(context);
}
export function getTelemetryContext() {
return context;
}
export async function sendTelemetryEvent(event: TelemetryEvent) {
if (!transport) {
enqueuePending(event);
return { queued: true };
}
return await transport.track(event);
}
export async function flushTelemetry() {
if (!transport?.flush) {
return { ok: true, accepted: 0, dropped: 0 } as const;
}
return await transport.flush();
}
async function flushPending() {
if (!transport || pendingEvents.length === 0) {
return;
}
const events = pendingEvents.splice(0, pendingEvents.length);
for (const event of events) {
await transport.track(event);
}
}
function enqueuePending(event: TelemetryEvent) {
if (pendingEvents.length >= PENDING_LIMIT) {
pendingEvents.shift();
}
pendingEvents.push(event);
}
function applyTransportContext(next: TelemetryContext) {
if (!transport) {
return;
}
void Promise.resolve(transport.setContext(next)).catch(error => {
logger.error('failed to set telemetry context', error);
});
}
packages/frontend/track/src/tracker.ts
+260
View File
@@ -0,0 +1,260 @@
import { DebugLogger } from '@affine/debug';
import { nanoid } from 'nanoid';
import type { TelemetryEvent } from './telemetry';
import { sendTelemetryEvent, setTelemetryContext } from './telemetry';
const logger = new DebugLogger('telemetry');
type TrackProperties = Record<string, unknown> | undefined;
type RawTrackProperties = Record<string, unknown> | object | undefined;
type Middleware = (
name: string,
properties?: TrackProperties
) => Record<string, unknown>;
const CLIENT_ID_KEY = 'affine_telemetry_client_id';
const SESSION_ID_KEY = 'affine_telemetry_session_id';
let enabled = true;
let clientId = readPersistentId(CLIENT_ID_KEY, localStorageSafe());
let sessionId = readPersistentId(SESSION_ID_KEY, sessionStorageSafe());
let userId: string | undefined;
let userProperties: Record<string, unknown> = {};
const middlewares = new Set<Middleware>();
export const tracker = {
init() {
this.register({
appVersion: BUILD_CONFIG.appVersion,
environment: BUILD_CONFIG.appBuildType,
editorVersion: BUILD_CONFIG.editorVersion,
isDesktop: BUILD_CONFIG.isElectron,
isMobile: BUILD_CONFIG.isMobileEdition,
distribution: BUILD_CONFIG.distribution,
});
},
register(props: Record<string, unknown>) {
userProperties = {
...userProperties,
...props,
};
setTelemetryContext({ userProperties });
},
reset() {
userId = undefined;
userProperties = {};
sessionId = readPersistentId(SESSION_ID_KEY, sessionStorageSafe(), true);
setTelemetryContext(
{ userId, userProperties },
{ replaceUserProperties: true }
);
this.init();
},
track(eventName: string, properties?: RawTrackProperties) {
if (!enabled) {
return;
}
const middlewareProperties = Array.from(middlewares).reduce(
(acc, middleware) => {
return middleware(eventName, acc);
},
normalizeProperties(properties)
);
logger.debug('track', eventName, middlewareProperties);
const event = buildEvent(eventName, middlewareProperties);
void sendTelemetryEvent(event).catch(error => {
logger.error('failed to send telemetry event', error);
});
},
track_pageview(properties?: { location?: string; [key: string]: unknown }) {
if (!enabled) {
return;
}
const middlewareProperties = Array.from(middlewares).reduce(
(acc, middleware) => {
return middleware('track_pageview', acc);
},
normalizeProperties(properties)
);
const pageLocation =
typeof middlewareProperties?.location === 'string'
? middlewareProperties.location
: getLocationHref();
const pageTitle = getDocumentTitle();
const params = {
...middlewareProperties,
location: pageLocation,
pageTitle: pageTitle ?? middlewareProperties?.pageTitle,
};
logger.debug('track_pageview', params);
const event = buildEvent('track_pageview', params);
void sendTelemetryEvent(event).catch(error => {
logger.error('failed to send telemetry pageview', error);
});
},
middleware(cb: Middleware): () => void {
middlewares.add(cb);
return () => {
middlewares.delete(cb);
};
},
opt_out_tracking() {
enabled = false;
},
opt_in_tracking() {
enabled = true;
},
has_opted_in_tracking() {
return enabled;
},
has_opted_out_tracking() {
return !enabled;
},
identify(nextUserId?: string) {
userId = nextUserId ? String(nextUserId) : undefined;
setTelemetryContext({ userId });
},
get people() {
return {
set: (props: Record<string, unknown>) => {
userProperties = {
...userProperties,
...props,
};
setTelemetryContext({ userProperties });
},
};
},
};
function buildEvent(
eventName: string,
params?: Record<string, unknown>
): TelemetryEvent {
return {
schemaVersion: 1,
eventName,
params,
userId,
userProperties,
clientId,
sessionId,
eventId: nanoid(),
timestampMicros: Date.now() * 1000,
context: buildContext(),
};
}
function buildContext(): TelemetryEvent['context'] {
return {
appVersion: BUILD_CONFIG.appVersion,
editorVersion: BUILD_CONFIG.editorVersion,
environment: BUILD_CONFIG.appBuildType,
distribution: BUILD_CONFIG.distribution,
channel: BUILD_CONFIG.appBuildType as NonNullable<
TelemetryEvent['context']
>['channel'],
isDesktop: BUILD_CONFIG.isElectron,
isMobile: BUILD_CONFIG.isMobileEdition,
locale: getLocale(),
timezone: getTimezone(),
url: getLocationHref(),
referrer: getReferrer(),
};
}
function normalizeProperties(properties?: RawTrackProperties): TrackProperties {
if (!properties) {
return undefined;
}
return properties as Record<string, unknown>;
}
function readPersistentId(key: string, storage: Storage | null, renew = false) {
if (!storage) {
return nanoid();
}
if (!renew) {
const existing = storage.getItem(key);
if (existing) {
return existing;
}
}
const id = nanoid();
try {
storage.setItem(key, id);
} catch {
return id;
}
return id;
}
function localStorageSafe(): Storage | null {
try {
return typeof localStorage === 'undefined' ? null : localStorage;
} catch {
return null;
}
}
function sessionStorageSafe(): Storage | null {
try {
return typeof sessionStorage === 'undefined' ? null : sessionStorage;
} catch {
return null;
}
}
function getLocale() {
try {
return typeof navigator === 'undefined' ? undefined : navigator.language;
} catch {
return undefined;
}
}
function getTimezone() {
try {
return Intl.DateTimeFormat().resolvedOptions().timeZone;
} catch {
return undefined;
}
}
function getLocationHref() {
try {
return typeof location === 'undefined' ? undefined : location.href;
} catch {
return undefined;
}
}
function getReferrer() {
try {
return typeof document === 'undefined' ? undefined : document.referrer;
} catch {
return undefined;
}
}
function getDocumentTitle() {
try {
return typeof document === 'undefined' ? undefined : document.title;
} catch {
return undefined;
}
}
tools/utils/src/workspace.gen.ts
+3
View File
@@ -1243,6 +1243,7 @@ export const PackageList = [
'packages/common/env',
'packages/frontend/i18n',
'packages/common/nbstore',
'packages/frontend/track',
'blocksuite/affine/all',
'packages/common/infra',
],
@@ -1284,6 +1285,7 @@ export const PackageList = [
'packages/common/graphql',
'packages/frontend/i18n',
'packages/common/nbstore',
'packages/frontend/track',
'blocksuite/affine/all',
'packages/common/infra',
'tools/cli',
@@ -1300,6 +1302,7 @@ export const PackageList = [
'packages/common/env',
'packages/frontend/i18n',
'packages/common/nbstore',
'packages/frontend/track',
'blocksuite/affine/all',
'packages/common/infra',
],
yarn.lock
+20 -92
View File
@@ -251,6 +251,7 @@ __metadata:
"@affine/env": "workspace:*"
"@affine/i18n": "workspace:*"
"@affine/nbstore": "workspace:*"
"@affine/track": "workspace:*"
"@blocksuite/affine": "workspace:*"
"@blocksuite/icons": "npm:^2.2.17"
"@capacitor/android": "npm:^7.0.0"
@@ -686,6 +687,7 @@ __metadata:
"@affine/i18n": "workspace:*"
"@affine/native": "workspace:*"
"@affine/nbstore": "workspace:*"
"@affine/track": "workspace:*"
"@blocksuite/affine": "workspace:*"
"@blocksuite/icons": "npm:^2.2.17"
"@capacitor/app": "npm:^7.0.0"
@@ -753,6 +755,7 @@ __metadata:
"@affine/env": "workspace:*"
"@affine/i18n": "workspace:*"
"@affine/nbstore": "workspace:*"
"@affine/track": "workspace:*"
"@blocksuite/affine": "workspace:*"
"@blocksuite/icons": "npm:^2.2.17"
"@sentry/react": "npm:^9.47.1"
@@ -885,6 +888,17 @@ __metadata:
languageName: unknown
linkType: soft
"@affine/revert-update@workspace:tools/revert-update":
version: 0.0.0-use.local
resolution: "@affine/revert-update@workspace:tools/revert-update"
dependencies:
"@affine-tools/cli": "workspace:*"
"@types/node": "npm:^22.0.0"
typescript: "npm:^5.7.2"
yjs: "npm:^13.6.27"
languageName: unknown
linkType: soft
"@affine/routes@workspace:*, @affine/routes@workspace:packages/frontend/routes":
version: 0.0.0-use.local
resolution: "@affine/routes@workspace:packages/frontend/routes"
@@ -1059,9 +1073,8 @@ __metadata:
dependencies:
"@affine/debug": "workspace:*"
"@sentry/react": "npm:^9.47.1"
"@types/mixpanel-browser": "npm:^2.50.2"
"@types/react": "npm:^19.0.1"
mixpanel-browser: "npm:^2.56.0"
nanoid: "npm:^5.1.6"
react-router-dom: "npm:^6.30.3"
vitest: "npm:^3.2.4"
languageName: unknown
@@ -15625,20 +15638,6 @@ __metadata:
languageName: node
linkType: hard
"@rrweb/types@npm:^2.0.0-alpha.18":
version: 2.0.0-alpha.18
resolution: "@rrweb/types@npm:2.0.0-alpha.18"
checksum: 10/ddb632d49490ac6c20d011825b7b44e28a3783bbdabdf72f09649fdf2c5e107f756e7d926e5074b827c2311f28562905e8fa1911bbe8119ca014b3e24fb87f72
languageName: node
linkType: hard
"@rrweb/utils@npm:^2.0.0-alpha.18":
version: 2.0.0-alpha.18
resolution: "@rrweb/utils@npm:2.0.0-alpha.18"
checksum: 10/d0ca790639816b13dcd353b9669059373c1f56585a5ad4a03c7559fd1d6cdae104e63f50f0076053b67ae1635be0f7b0e9bcb42ab9f9e094358952760499133d
languageName: node
linkType: hard
"@scarf/scarf@npm:=1.4.0":
version: 1.4.0
resolution: "@scarf/scarf@npm:1.4.0"
@@ -17886,13 +17885,6 @@ __metadata:
languageName: node
linkType: hard
"@types/css-font-loading-module@npm:0.0.7":
version: 0.0.7
resolution: "@types/css-font-loading-module@npm:0.0.7"
checksum: 10/f70b9098ee3b2e006f5f6d5cecc627dcc7b898f266bfc594e73a8720636f1a3bc5f8c38fa0e8f7e5b7878038b46fd70da0c797c3288e072af984097210f4c056
languageName: node
linkType: hard
"@types/d3-array@npm:*":
version: 3.2.2
resolution: "@types/d3-array@npm:3.2.2"
@@ -18565,13 +18557,6 @@ __metadata:
languageName: node
linkType: hard
"@types/mixpanel-browser@npm:^2.50.2":
version: 2.60.0
resolution: "@types/mixpanel-browser@npm:2.60.0"
checksum: 10/99c1f4601f5d56e86635237ff086d46593bbe826da1c0ff53e28cec146b63b8a70d15ab7fc0950c955f9e75213bac80299938478d1118c1ca3ac8b46c2b9244a
languageName: node
linkType: hard
"@types/mixpanel@npm:^2.14.9":
version: 2.14.9
resolution: "@types/mixpanel@npm:2.14.9"
@@ -19750,13 +19735,6 @@ __metadata:
languageName: node
linkType: hard
"@xstate/fsm@npm:^1.4.0":
version: 1.6.5
resolution: "@xstate/fsm@npm:1.6.5"
checksum: 10/deae1501169d41d5395ce1581d7b08bde17911b7dec1533eacd5bee17060d22273055d6d6bc7e8f32222a502e4dcf510cd29064a5c9c5fa5aa2ced0ad60b2512
languageName: node
linkType: hard
"@xtuc/ieee754@npm:^1.2.0":
version: 1.2.0
resolution: "@xtuc/ieee754@npm:1.2.0"
@@ -20609,7 +20587,7 @@ __metadata:
languageName: node
linkType: hard
"base64-arraybuffer@npm:^1.0.1, base64-arraybuffer@npm:^1.0.2":
"base64-arraybuffer@npm:^1.0.2":
version: 1.0.2
resolution: "base64-arraybuffer@npm:1.0.2"
checksum: 10/15e6400d2d028bf18be4ed97702b11418f8f8779fb8c743251c863b726638d52f69571d4cc1843224da7838abef0949c670bde46936663c45ad078e89fee5c62
@@ -20638,11 +20616,11 @@ __metadata:
linkType: hard
"baseline-browser-mapping@npm:^2.8.25":
version: 2.8.28
resolution: "baseline-browser-mapping@npm:2.8.28"
version: 2.9.14
resolution: "baseline-browser-mapping@npm:2.9.14"
bin:
baseline-browser-mapping: dist/cli.js
checksum: 10/f54abab4d61b2105c628581ae89220924ed83bcf43dbc621672e4c2498af527a054aa38f9e14dd73a11550290e6bcef6f60eaccd1f1de4f557b941b43162a5e4
checksum: 10/a329881e5f673c0834843640e9c954c478f643fb983449c99850392e48cf52dfb1dc3de8d81c6a6a2802c86310833accc5e3deb6bef5fb6e329989e28ca5489b
languageName: node
linkType: hard
@@ -30902,22 +30880,6 @@ __metadata:
languageName: node
linkType: hard
"mitt@npm:^3.0.0":
version: 3.0.1
resolution: "mitt@npm:3.0.1"
checksum: 10/287c70d8e73ffc25624261a4989c783768aed95ecb60900f051d180cf83e311e3e59865bfd6e9d029cdb149dc20ba2f128a805e9429c5c4ce33b1416c65bbd14
languageName: node
linkType: hard
"mixpanel-browser@npm:^2.56.0":
version: 2.65.0
resolution: "mixpanel-browser@npm:2.65.0"
dependencies:
rrweb: "npm:2.0.0-alpha.18"
checksum: 10/0cc14191ef6bc9ec051bba62807f85426cabc48154a7ee8ba0b9770237f5701a10cd8cbc6178c12ee4ddac2ebc037d81335107c41f6c633ed9bd1f8f3d40abc7
languageName: node
linkType: hard
"mixpanel@npm:^0.18.0":
version: 0.18.1
resolution: "mixpanel@npm:0.18.1"
@@ -33273,7 +33235,7 @@ __metadata:
languageName: node
linkType: hard
"postcss@npm:^8.4.33, postcss@npm:^8.4.38, postcss@npm:^8.4.41, postcss@npm:^8.4.49, postcss@npm:^8.5.6":
"postcss@npm:^8.4.33, postcss@npm:^8.4.41, postcss@npm:^8.4.49, postcss@npm:^8.5.6":
version: 8.5.6
resolution: "postcss@npm:8.5.6"
dependencies:
@@ -34971,40 +34933,6 @@ __metadata:
languageName: node
linkType: hard
"rrdom@npm:^2.0.0-alpha.18":
version: 2.0.0-alpha.18
resolution: "rrdom@npm:2.0.0-alpha.18"
dependencies:
rrweb-snapshot: "npm:^2.0.0-alpha.18"
checksum: 10/4b02e60a6828dd29893b2a003e7611f8db275bb21b24a0b8db97ecbfd75020b6d4be26e90ada0cae4a9b28fdc75fa258dd174d21e33d30c33aded797cc23b9bc
languageName: node
linkType: hard
"rrweb-snapshot@npm:^2.0.0-alpha.18":
version: 2.0.0-alpha.18
resolution: "rrweb-snapshot@npm:2.0.0-alpha.18"
dependencies:
postcss: "npm:^8.4.38"
checksum: 10/5dbc717cf80057855d43c7afdbffc117af5074dc627c5b1375234512f1b04c03756836605cf8cb9615639a244fe6d5b2a139955a4ab131a2050dab7808332aa2
languageName: node
linkType: hard
"rrweb@npm:2.0.0-alpha.18":
version: 2.0.0-alpha.18
resolution: "rrweb@npm:2.0.0-alpha.18"
dependencies:
"@rrweb/types": "npm:^2.0.0-alpha.18"
"@rrweb/utils": "npm:^2.0.0-alpha.18"
"@types/css-font-loading-module": "npm:0.0.7"
"@xstate/fsm": "npm:^1.4.0"
base64-arraybuffer: "npm:^1.0.1"
mitt: "npm:^3.0.0"
rrdom: "npm:^2.0.0-alpha.18"
rrweb-snapshot: "npm:^2.0.0-alpha.18"
checksum: 10/44efc0475a70c0a53f8eafc08159ccaaab56396c2cf2233a132bccb4d98c801b555d6d1bab7267a51bf5312766f908952bd75659d3d84e31c1a1e51cc76631ee
languageName: node
linkType: hard
"run-applescript@npm:^7.0.0":
version: 7.0.0
resolution: "run-applescript@npm:7.0.0"