feat(server): adapt gemini3.1 preview (#14583)

#### PR Dependency Tree


* **PR #14583** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added Gemini 3.1 Pro Preview support (text, image, audio) and new
GPT‑5 variants as defaults; centralized persistent telemetry state for
more reliable client identity.

* **UX**
  * Improved model submenu placement in chat preferences.
* More robust mindmap parsing, preview, regeneration and replace
behavior.

* **Chores**
  * Bumped AI SDK and related dependencies.

* **Tests**
  * Expanded/updated tests and increased timeouts for flaky flows.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-03-08 00:53:16 +08:00
committed by GitHub
parent 9742e9735e
commit 9c55edeb62
36 changed files with 980 additions and 375 deletions
@@ -26,10 +26,17 @@ import {
ThinkingIcon,
} from '@blocksuite/icons/lit';
import { ShadowlessElement } from '@blocksuite/std';
import { autoPlacement, offset, shift } from '@floating-ui/dom';
import { computed } from '@preact/signals-core';
import { css, html } from 'lit';
import { property } from 'lit/decorators.js';
const modelSubMenuMiddleware = [
autoPlacement({ allowedPlacements: ['right-start', 'left-start'] }),
offset({ mainAxis: 4, crossAxis: 0 }),
shift({ crossAxis: true, padding: 8 }),
];
export class ChatInputPreference extends SignalWatcher(
WithDisposable(ShadowlessElement)
) {
@@ -140,6 +147,7 @@ export class ChatInputPreference extends SignalWatcher(
menu.subMenu({
name: 'Model',
prefix: AiOutlineIcon(),
middleware: modelSubMenuMiddleware,
postfix: html`
<span class="ai-active-model-name"> ${this.model.value?.name} </span>
`,
@@ -99,4 +99,69 @@ describe('markdownToMindmap: convert markdown list to a mind map tree', () => {
expect(nodes).toEqual(null);
});
test('accepts leading plain text before the markdown list', () => {
const markdown = `Here is the regenerated mind map:
- Text A
- Text B`;
const collection = new TestWorkspace();
collection.meta.initialize();
const doc = collection.createDoc().getStore();
const nodes = markdownToMindmap(markdown, doc, provider);
expect(nodes).toEqual({
text: 'Text A',
children: [
{
text: 'Text B',
children: [],
},
],
});
});
test('accepts markdown lists wrapped in a code block', () => {
const markdown = `\`\`\`markdown
- Text A
- Text B
\`\`\``;
const collection = new TestWorkspace();
collection.meta.initialize();
const doc = collection.createDoc().getStore();
const nodes = markdownToMindmap(markdown, doc, provider);
expect(nodes).toEqual({
text: 'Text A',
children: [
{
text: 'Text B',
children: [],
},
],
});
});
test('keeps inline markdown content inside node labels', () => {
const markdown = `
- Root with [link](https://example.com) and [^1]
- Child with \`code\`
[^1]: footnote
`;
const collection = new TestWorkspace();
collection.meta.initialize();
const doc = collection.createDoc().getStore();
const nodes = markdownToMindmap(markdown, doc, provider);
expect(nodes).toEqual({
text: 'Root with link and',
children: [
{
text: 'Child with code',
children: [],
},
],
});
});
});
@@ -19,7 +19,7 @@ import { css, html, LitElement, nothing } from 'lit';
import { property, query } from 'lit/decorators.js';
import { repeat } from 'lit/directives/repeat.js';
import { styleMap } from 'lit/directives/style-map.js';
import type { Root } from 'mdast';
import type { Root, RootContent } from 'mdast';
import { Doc as YDoc } from 'yjs';
import { MiniMindmapSchema, MiniMindmapSpecs } from './spec.js';
@@ -234,19 +234,68 @@ type Node = {
children: Node[];
};
type MarkdownNode =
| RootContent
| { alt?: string | null; children?: MarkdownNode[]; value?: string };
export const markdownToMindmap = (
answer: string,
doc: Store,
provider: ServiceProvider
) => {
let result: Node | null = null;
const transformer = doc.getTransformer();
const markdown = new MarkdownAdapter(transformer, provider);
const ast: Root = markdown['_markdownToAst'](answer);
const astToMindmap = (ast: Root): Node | null => {
const findList = (
nodes: Root['children']
): Unpacked<Root['children']> | null => {
for (const node of nodes) {
if (node.type === 'list') {
return node;
}
if (node.type === 'code' && node.value) {
const nestedAst: Root = markdown['_markdownToAst'](node.value);
const nestedList = findList(nestedAst.children);
if (nestedList) {
return nestedList;
}
}
}
return null;
};
const list = findList(ast.children);
if (!list) {
return null;
}
return traverse(list, true);
};
const traverse = (
markdownNode: Unpacked<(typeof ast)['children']>,
markdownNode: Unpacked<Root['children']>,
firstLevel = false
): Node | null => {
const toPlainText = (node: MarkdownNode): string => {
if ('value' in node && typeof node.value === 'string') {
return node.value;
}
if ('alt' in node && typeof node.alt === 'string') {
return node.alt;
}
if ('children' in node && Array.isArray(node.children)) {
return node.children
.map((child: MarkdownNode) => toPlainText(child))
.join('');
}
return '';
};
switch (markdownNode.type) {
case 'list':
{
@@ -267,11 +316,11 @@ export const markdownToMindmap = (
children: [],
};
if (
paragraph?.type === 'paragraph' &&
paragraph.children[0]?.type === 'text'
) {
node.text = paragraph.children[0].value;
if (paragraph?.type === 'paragraph') {
node.text = paragraph.children
.map((child: MarkdownNode) => toPlainText(child))
.join('')
.trim();
}
if (list?.type === 'list') {
@@ -287,9 +336,5 @@ export const markdownToMindmap = (
return null;
};
if (ast?.children?.[0]?.type === 'list') {
result = traverse(ast.children[0], true);
}
return result;
return astToMindmap(markdown['_markdownToAst'](answer));
};
@@ -12,6 +12,7 @@ import {
ThemeService,
} from '@blocksuite/affine/shared/services';
import { BlockViewExtension, FlavourExtension } from '@blocksuite/affine/std';
import { ToolController } from '@blocksuite/affine/std/gfx';
import type { BlockSchema, ExtensionType } from '@blocksuite/affine/store';
import { literal } from 'lit/static-html.js';
import type { z } from 'zod';
@@ -24,6 +25,7 @@ export const MiniMindmapSpecs: ExtensionType[] = [
ThemeService,
FlavourExtension('affine:page'),
MindmapService,
ToolController,
BlockViewExtension('affine:page', literal`mini-mindmap-root-block`),
FlavourExtension('affine:surface'),
MindMapView,
@@ -4,7 +4,7 @@ import {
type UploadFileResponse,
} from '@google/generative-ai/server';
const DEFAULT_MODEL = 'gemini-2.0-flash';
const DEFAULT_MODEL = 'gemini-2.5-pro';
export interface TranscriptionResult {
title: string;
@@ -75,7 +75,7 @@ Output in JSON format with the following structure:
export async function gemini(
audioFilePath: string,
options?: {
model?: 'gemini-2.0-flash' | 'gemini-1.5-flash';
model?: 'gemini-2.5-flash' | 'gemini-2.5-pro';
mode?: 'transcript' | 'summary';
}
) {
@@ -3,8 +3,13 @@
*/
import { beforeEach, describe, expect, test, vi } from 'vitest';
const sendTelemetryEvent = vi.fn().mockResolvedValue({ queued: true });
const setTelemetryContext = vi.fn();
import { resetTrackerState } from '../state';
import { tracker } from '../tracker';
const { sendTelemetryEvent, setTelemetryContext } = vi.hoisted(() => ({
sendTelemetryEvent: vi.fn().mockResolvedValue({ queued: true }),
setTelemetryContext: vi.fn(),
}));
vi.mock('../telemetry', () => ({
sendTelemetryEvent,
@@ -27,17 +32,11 @@ beforeEach(() => {
sendTelemetryEvent.mockClear();
setTelemetryContext.mockClear();
vi.useRealTimers();
vi.resetModules();
resetTrackerState();
});
async function loadTracker() {
return await import('../tracker');
}
describe('tracker session signals', () => {
test('sends first_visit and session_start on first event', async () => {
const { tracker } = await loadTracker();
test('sends first_visit and session_start on first event', () => {
tracker.track('test_event');
const events = sendTelemetryEvent.mock.calls.map(call => call[0]);
@@ -48,14 +47,12 @@ describe('tracker session signals', () => {
]);
const firstVisit = events[0];
expect(typeof (firstVisit.params as any).session_id).toBe('number');
expect((firstVisit.params as any).session_number).toBe(1);
expect((firstVisit.params as any).engagement_time_msec).toBe(1);
expect(typeof firstVisit.params?.session_id).toBe('number');
expect(firstVisit.params?.session_number).toBe(1);
expect(firstVisit.params?.engagement_time_msec).toBe(1);
});
test('does not repeat first_visit for later events', async () => {
const { tracker } = await loadTracker();
test('does not repeat first_visit for later events', () => {
tracker.track('event_a');
tracker.track('event_b');
@@ -64,10 +61,9 @@ describe('tracker session signals', () => {
expect(names.filter(name => name === 'session_start')).toHaveLength(1);
});
test('increments session_number after idle timeout', async () => {
test('increments session_number after idle timeout', () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2024-01-01T00:00:00Z'));
const { tracker } = await loadTracker();
tracker.track('event_a');
sendTelemetryEvent.mockClear();
+99
View File
@@ -0,0 +1,99 @@
import { nanoid } from 'nanoid';
export type TrackProperties = Record<string, unknown> | undefined;
export type Middleware = (
name: string,
properties?: TrackProperties
) => Record<string, unknown>;
type TrackerState = {
enabled: boolean;
clientStorage: Storage | null;
clientId: string;
pendingFirstVisit: boolean;
sessionId: number;
sessionNumber: number;
lastActivityMs: number;
sessionStartSent: boolean;
engagementTrackingEnabled: boolean;
visibleSinceMs: number | null;
pendingEngagementMs: number;
visibilityChangeHandler: (() => void) | null;
pageHideHandler: (() => void) | null;
userId: string | undefined;
userProperties: Record<string, unknown>;
middlewares: Set<Middleware>;
};
const CLIENT_ID_KEY = 'affine_telemetry_client_id';
export let trackerState = createTrackerState();
export function resetTrackerState() {
cleanupTrackerState(trackerState);
trackerState = createTrackerState();
}
function createTrackerState(): TrackerState {
const clientStorage = localStorageSafe();
const hasClientId = !!clientStorage?.getItem(CLIENT_ID_KEY);
return {
enabled: true,
clientStorage,
clientId: readPersistentId(CLIENT_ID_KEY, clientStorage),
pendingFirstVisit: !hasClientId,
sessionId: 0,
sessionNumber: 0,
lastActivityMs: 0,
sessionStartSent: false,
engagementTrackingEnabled: false,
visibleSinceMs: null,
pendingEngagementMs: 0,
visibilityChangeHandler: null,
pageHideHandler: null,
userId: undefined,
userProperties: {},
middlewares: new Set<Middleware>(),
};
}
function cleanupTrackerState(state: TrackerState) {
if (state.visibilityChangeHandler && typeof document !== 'undefined') {
document.removeEventListener(
'visibilitychange',
state.visibilityChangeHandler
);
}
if (state.pageHideHandler && typeof window !== 'undefined') {
window.removeEventListener('pagehide', state.pageHideHandler);
}
}
function readPersistentId(key: string, storage: Storage | null, renew = false) {
if (!storage) {
return nanoid();
}
if (!renew) {
const existing = storage.getItem(key);
if (existing) {
return existing;
}
}
const id = nanoid();
try {
storage.setItem(key, id);
} catch {
return id;
}
return id;
}
function localStorageSafe(): Storage | null {
try {
return typeof localStorage === 'undefined' ? null : localStorage;
} catch {
return null;
}
}
+99 -115
View File
@@ -1,43 +1,20 @@
import { DebugLogger } from '@affine/debug';
import { nanoid } from 'nanoid';
import { type Middleware, trackerState, type TrackProperties } from './state';
import type { TelemetryEvent } from './telemetry';
import { sendTelemetryEvent, setTelemetryContext } from './telemetry';
const logger = new DebugLogger('telemetry');
type TrackProperties = Record<string, unknown> | undefined;
type RawTrackProperties = Record<string, unknown> | object | undefined;
type Middleware = (
name: string,
properties?: TrackProperties
) => Record<string, unknown>;
const CLIENT_ID_KEY = 'affine_telemetry_client_id';
const SESSION_ID_KEY = 'affine_telemetry_session_id';
const SESSION_NUMBER_KEY = 'affine_telemetry_session_number';
const SESSION_NUMBER_CURRENT_KEY = 'affine_telemetry_session_number_current';
const LAST_ACTIVITY_KEY = 'affine_telemetry_last_activity_ms';
const SESSION_TIMEOUT_MS = 30 * 60 * 1000;
let enabled = true;
const clientStorage = localStorageSafe();
const hasClientId = clientStorage?.getItem(CLIENT_ID_KEY);
let clientId = readPersistentId(CLIENT_ID_KEY, clientStorage);
let pendingFirstVisit = !hasClientId;
let sessionId = 0;
let sessionNumber = 0;
let lastActivityMs = 0;
let sessionStartSent = false;
let engagementTrackingEnabled = false;
let visibleSinceMs: number | null = null;
let pendingEngagementMs = 0;
let userId: string | undefined;
let userProperties: Record<string, unknown> = {};
const middlewares = new Set<Middleware>();
export const tracker = {
init() {
this.register({
@@ -51,29 +28,32 @@ export const tracker = {
},
register(props: Record<string, unknown>) {
userProperties = {
...userProperties,
trackerState.userProperties = {
...trackerState.userProperties,
...props,
};
setTelemetryContext({ userProperties });
setTelemetryContext({ userProperties: trackerState.userProperties });
},
reset() {
userId = undefined;
userProperties = {};
trackerState.userId = undefined;
trackerState.userProperties = {};
startNewSession(Date.now(), sessionStorageSafe());
setTelemetryContext(
{ userId, userProperties },
{
userId: trackerState.userId,
userProperties: trackerState.userProperties,
},
{ replaceUserProperties: true }
);
this.init();
},
track(eventName: string, properties?: RawTrackProperties) {
if (!enabled) {
if (!trackerState.enabled) {
return;
}
const middlewareProperties = Array.from(middlewares).reduce(
const middlewareProperties = Array.from(trackerState.middlewares).reduce(
(acc, middleware) => {
return middleware(eventName, acc);
},
@@ -84,10 +64,10 @@ export const tracker = {
},
track_pageview(properties?: { location?: string; [key: string]: unknown }) {
if (!enabled) {
if (!trackerState.enabled) {
return;
}
const middlewareProperties = Array.from(middlewares).reduce(
const middlewareProperties = Array.from(trackerState.middlewares).reduce(
(acc, middleware) => {
return middleware('track_pageview', acc);
},
@@ -108,41 +88,41 @@ export const tracker = {
},
middleware(cb: Middleware): () => void {
middlewares.add(cb);
trackerState.middlewares.add(cb);
return () => {
middlewares.delete(cb);
trackerState.middlewares.delete(cb);
};
},
opt_out_tracking() {
enabled = false;
trackerState.enabled = false;
},
opt_in_tracking() {
enabled = true;
trackerState.enabled = true;
},
has_opted_in_tracking() {
return enabled;
return trackerState.enabled;
},
has_opted_out_tracking() {
return !enabled;
return !trackerState.enabled;
},
identify(nextUserId?: string) {
userId = nextUserId ? String(nextUserId) : undefined;
setTelemetryContext({ userId });
trackerState.userId = nextUserId ? String(nextUserId) : undefined;
setTelemetryContext({ userId: trackerState.userId });
},
get people() {
return {
set: (props: Record<string, unknown>) => {
userProperties = {
...userProperties,
trackerState.userProperties = {
...trackerState.userProperties,
...props,
};
setTelemetryContext({ userProperties });
setTelemetryContext({ userProperties: trackerState.userProperties });
},
};
},
@@ -193,45 +173,62 @@ function prepareSession(now: number) {
if (expired) {
startNewSession(now, sessionStorage);
} else {
sessionId = storedSessionId;
sessionNumber = readCurrentSessionNumber(sessionStorage, clientStorage);
trackerState.sessionId = storedSessionId;
trackerState.sessionNumber = readCurrentSessionNumber(
sessionStorage,
trackerState.clientStorage
);
updateLastActivity(now, sessionStorage);
}
} else {
const expired =
!sessionId ||
!lastActivityMs ||
now - lastActivityMs > SESSION_TIMEOUT_MS;
!trackerState.sessionId ||
!trackerState.lastActivityMs ||
now - trackerState.lastActivityMs > SESSION_TIMEOUT_MS;
if (expired) {
startNewSession(now, null);
} else {
lastActivityMs = now;
if (!sessionNumber) {
sessionNumber = 1;
trackerState.lastActivityMs = now;
if (!trackerState.sessionNumber) {
trackerState.sessionNumber = 1;
}
}
}
const preEvents: TelemetryEvent[] = [];
if (pendingFirstVisit) {
pendingFirstVisit = false;
if (trackerState.pendingFirstVisit) {
trackerState.pendingFirstVisit = false;
preEvents.push(
buildEvent(
'first_visit',
mergeSessionParams({}, sessionId, sessionNumber, 1)
mergeSessionParams(
{},
trackerState.sessionId,
trackerState.sessionNumber,
1
)
)
);
}
if (!sessionStartSent) {
sessionStartSent = true;
if (!trackerState.sessionStartSent) {
trackerState.sessionStartSent = true;
preEvents.push(
buildEvent(
'session_start',
mergeSessionParams({}, sessionId, sessionNumber, 1)
mergeSessionParams(
{},
trackerState.sessionId,
trackerState.sessionNumber,
1
)
)
);
}
return { sessionId, sessionNumber, preEvents };
return {
sessionId: trackerState.sessionId,
sessionNumber: trackerState.sessionNumber,
preEvents,
};
}
function mergeSessionParams(
@@ -256,62 +253,76 @@ function mergeSessionParams(
}
function startNewSession(now: number, sessionStorage: Storage | null) {
sessionId = Math.floor(now / 1000);
sessionNumber = incrementSessionNumber(clientStorage, sessionStorage);
trackerState.sessionId = Math.floor(now / 1000);
trackerState.sessionNumber = incrementSessionNumber(
trackerState.clientStorage,
sessionStorage
);
updateLastActivity(now, sessionStorage);
writeNumber(sessionStorage, SESSION_ID_KEY, sessionId);
sessionStartSent = false;
writeNumber(sessionStorage, SESSION_ID_KEY, trackerState.sessionId);
trackerState.sessionStartSent = false;
resetEngagementState(now);
}
function updateLastActivity(now: number, sessionStorage: Storage | null) {
lastActivityMs = now;
trackerState.lastActivityMs = now;
writeNumber(sessionStorage, LAST_ACTIVITY_KEY, now);
}
function consumeEngagementTime(now: number) {
initEngagementTracking(now);
if (visibleSinceMs !== null) {
pendingEngagementMs += now - visibleSinceMs;
visibleSinceMs = now;
if (trackerState.visibleSinceMs !== null) {
trackerState.pendingEngagementMs += now - trackerState.visibleSinceMs;
trackerState.visibleSinceMs = now;
}
const engagementMs = Math.max(0, Math.round(pendingEngagementMs));
pendingEngagementMs = 0;
const engagementMs = Math.max(
0,
Math.round(trackerState.pendingEngagementMs)
);
trackerState.pendingEngagementMs = 0;
return engagementMs;
}
function resetEngagementState(now: number) {
pendingEngagementMs = 0;
visibleSinceMs = isDocumentVisible() ? now : null;
trackerState.pendingEngagementMs = 0;
trackerState.visibleSinceMs = isDocumentVisible() ? now : null;
}
function initEngagementTracking(now: number) {
if (engagementTrackingEnabled || typeof document === 'undefined') {
if (
trackerState.engagementTrackingEnabled ||
typeof document === 'undefined'
) {
return;
}
engagementTrackingEnabled = true;
trackerState.engagementTrackingEnabled = true;
resetEngagementState(now);
document.addEventListener('visibilitychange', () => {
trackerState.visibilityChangeHandler = () => {
const now = Date.now();
if (visibleSinceMs !== null) {
pendingEngagementMs += now - visibleSinceMs;
if (trackerState.visibleSinceMs !== null) {
trackerState.pendingEngagementMs += now - trackerState.visibleSinceMs;
}
visibleSinceMs = isDocumentVisible() ? now : null;
trackerState.visibleSinceMs = isDocumentVisible() ? now : null;
if (!isDocumentVisible()) {
dispatchUserEngagement(now);
}
});
};
document.addEventListener(
'visibilitychange',
trackerState.visibilityChangeHandler
);
if (typeof window !== 'undefined') {
window.addEventListener('pagehide', () => {
trackerState.pageHideHandler = () => {
dispatchUserEngagement(Date.now());
});
};
window.addEventListener('pagehide', trackerState.pageHideHandler);
}
}
function dispatchUserEngagement(now: number) {
if (!enabled) {
if (!trackerState.enabled) {
return;
}
const engagementMs = consumeEngagementTime(now);
@@ -377,7 +388,7 @@ function readCurrentSessionNumber(
const fallback = localStorage
? (readPositiveNumber(localStorage, SESSION_NUMBER_KEY) ?? 1)
: sessionNumber || 1;
: trackerState.sessionNumber || 1;
writeNumber(sessionStorage, SESSION_NUMBER_CURRENT_KEY, fallback);
if (localStorage && !readPositiveNumber(localStorage, SESSION_NUMBER_KEY)) {
@@ -391,7 +402,7 @@ function incrementSessionNumber(
sessionStorage: Storage | null
) {
if (!localStorage) {
const next = (sessionNumber || 0) + 1;
const next = (trackerState.sessionNumber || 0) + 1;
writeNumber(sessionStorage, SESSION_NUMBER_CURRENT_KEY, next);
return next;
}
@@ -410,10 +421,10 @@ function buildEvent(
schemaVersion: 1,
eventName,
params,
userId,
userProperties,
clientId,
sessionId,
userId: trackerState.userId,
userProperties: trackerState.userProperties,
clientId: trackerState.clientId,
sessionId: trackerState.sessionId,
eventId: nanoid(),
timestampMicros: Date.now() * 1000,
context: buildContext(),
@@ -445,33 +456,6 @@ function normalizeProperties(properties?: RawTrackProperties): TrackProperties {
return properties as Record<string, unknown>;
}
function readPersistentId(key: string, storage: Storage | null, renew = false) {
if (!storage) {
return nanoid();
}
if (!renew) {
const existing = storage.getItem(key);
if (existing) {
return existing;
}
}
const id = nanoid();
try {
storage.setItem(key, id);
} catch {
return id;
}
return id;
}
function localStorageSafe(): Storage | null {
try {
return typeof localStorage === 'undefined' ? null : localStorage;
} catch {
return null;
}
}
function sessionStorageSafe(): Storage | null {
try {
return typeof sessionStorage === 'undefined' ? null : sessionStorage;