From 78949044ec0bce3f55fbff096450b308e8a1a4f5 Mon Sep 17 00:00:00 2001 From: DarkSky <25152247+darkskygit@users.noreply.github.com> Date: Sat, 27 Dec 2025 08:22:37 +0800 Subject: [PATCH] feat: improve idb perf (#14159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary by CodeRabbit * **Performance** * Optimized database operations through improved batch processing to accelerate data retrieval, updates, and deletion operations for better efficiency. * **Reliability** * Enhanced transaction durability handling to strengthen data consistency and ensure more reliable persistence of database changes and updates. ✏️ Tip: You can customize this high-level summary in your review settings. --- packages/common/nbstore/src/impls/idb/blob.ts | 60 +++++++++++-------- packages/common/nbstore/src/impls/idb/db.ts | 32 ++++++++++ packages/common/nbstore/src/impls/idb/doc.ts | 46 +++++++++++--- 3 files changed, 104 insertions(+), 34 deletions(-) diff --git a/packages/common/nbstore/src/impls/idb/blob.ts b/packages/common/nbstore/src/impls/idb/blob.ts index 9abb02c773..d06292951d 100644 --- a/packages/common/nbstore/src/impls/idb/blob.ts +++ b/packages/common/nbstore/src/impls/idb/blob.ts @@ -1,9 +1,5 @@ import { share } from '../../connection'; -import { - type BlobRecord, - BlobStorageBase, - type ListedBlobRecord, -} from '../../storage'; +import { type BlobRecord, BlobStorageBase } from '../../storage'; import { IDBConnection, type IDBConnectionOptions } from './db'; export class IndexedDBBlobStorage extends BlobStorageBase { @@ -36,7 +32,9 @@ export class IndexedDBBlobStorage extends BlobStorageBase { } override async set(blob: BlobRecord) { - const trx = this.db.transaction(['blobs', 'blobData'], 'readwrite'); + const trx = this.db.transaction(['blobs', 'blobData'], 'readwrite', { + durability: 'relaxed', + }); await trx.objectStore('blobs').put({ key: blob.key, mime: blob.mime, @@ -52,11 +50,15 @@ export class IndexedDBBlobStorage extends BlobStorageBase { override async delete(key: string, permanently: boolean) { if (permanently) { - const trx = this.db.transaction(['blobs', 'blobData'], 'readwrite'); + const trx = this.db.transaction(['blobs', 'blobData'], 'readwrite', { + durability: 'relaxed', + }); await trx.objectStore('blobs').delete(key); await trx.objectStore('blobData').delete(key); } else { - const trx = this.db.transaction('blobs', 'readwrite'); + const trx = this.db.transaction('blobs', 'readwrite', { + durability: 'relaxed', + }); const blob = await trx.store.get(key); if (blob) { await trx.store.put({ @@ -68,29 +70,37 @@ export class IndexedDBBlobStorage extends BlobStorageBase { } override async release() { - const trx = this.db.transaction(['blobs', 'blobData'], 'readwrite'); + const trx = this.db.transaction(['blobs', 'blobData'], 'readwrite', { + durability: 'relaxed', + }); - const it = trx.objectStore('blobs').iterate(); + const store = trx.objectStore('blobs'); + const getAllRecords = store.getAllRecords?.bind(store); + const blobs = + typeof getAllRecords === 'function' + ? (await getAllRecords()).map(record => record.value) + : await store.getAll(); - for await (const item of it) { - if (item.value.deletedAt) { - await item.delete(); - await trx.objectStore('blobData').delete(item.value.key); - } - } + const deleted = blobs.filter(blob => blob.deletedAt); + + await Promise.all( + deleted.map(blob => + Promise.all([ + store.delete(blob.key), + trx.objectStore('blobData').delete(blob.key), + ]) + ) + ); } override async list() { const trx = this.db.transaction('blobs', 'readonly'); - const it = trx.store.iterate(); + const getAllRecords = trx.store.getAllRecords?.bind(trx.store); + const blobs = + typeof getAllRecords === 'function' + ? (await getAllRecords()).map(record => record.value) + : await trx.store.getAll(); - const blobs: ListedBlobRecord[] = []; - for await (const item of it) { - if (!item.value.deletedAt) { - blobs.push(item.value); - } - } - - return blobs; + return blobs.filter(blob => !blob.deletedAt); } } diff --git a/packages/common/nbstore/src/impls/idb/db.ts b/packages/common/nbstore/src/impls/idb/db.ts index 0326ab91e2..1e29f49d64 100644 --- a/packages/common/nbstore/src/impls/idb/db.ts +++ b/packages/common/nbstore/src/impls/idb/db.ts @@ -4,6 +4,38 @@ import { AutoReconnectConnection } from '../../connection'; import type { SpaceType } from '../../utils/universal-id'; import { type DocStorageSchema, migrator } from './schema'; +declare module 'idb' { + interface IDBPObjectStore { + getAllRecords?( + query?: IDBValidKey | IDBKeyRange | null, + count?: number | { direction?: IDBCursorDirection; count?: number } + ): Promise; + } + interface IDBPIndex { + getAllRecords?( + query?: IDBValidKey | IDBKeyRange | null, + count?: number | { direction?: IDBCursorDirection; count?: number } + ): Promise; + } + interface IDBObjectStore { + getAllRecords?( + query?: IDBValidKey | IDBKeyRange | null, + count?: number | { direction?: IDBCursorDirection; count?: number } + ): Promise; + } + interface IDBIndex { + getAllRecords?( + query?: IDBValidKey | IDBKeyRange | null, + count?: number | { direction?: IDBCursorDirection; count?: number } + ): Promise; + } + interface IDBRecord { + key: IDBValidKey; + primaryKey: IDBValidKey; + value: any; + } +} + export interface IDBConnectionOptions { flavour: string; type: SpaceType; diff --git a/packages/common/nbstore/src/impls/idb/doc.ts b/packages/common/nbstore/src/impls/idb/doc.ts index 46ca0bc4bc..c6977f6baa 100644 --- a/packages/common/nbstore/src/impls/idb/doc.ts +++ b/packages/common/nbstore/src/impls/idb/doc.ts @@ -37,7 +37,9 @@ export class IndexedDBDocStorage extends DocStorageBase { while (true) { try { - const trx = this.db.transaction(['updates', 'clocks'], 'readwrite'); + const trx = this.db.transaction(['updates', 'clocks'], 'readwrite', { + durability: 'relaxed', + }); await trx.objectStore('updates').add({ ...update, @@ -103,15 +105,15 @@ export class IndexedDBDocStorage extends DocStorageBase { override async deleteDoc(docId: string) { const trx = this.db.transaction( ['snapshots', 'updates', 'clocks'], - 'readwrite' + 'readwrite', + { durability: 'relaxed' } ); - const idx = trx.objectStore('updates').index('docId'); - const iter = idx.iterate(IDBKeyRange.only(docId)); + const updates = trx.objectStore('updates'); + const idx = updates.index('docId'); + const keys = await idx.getAllKeys(IDBKeyRange.only(docId)); - for await (const { value } of iter) { - await trx.objectStore('updates').delete([value.docId, value.createdAt]); - } + await Promise.all(keys.map(key => updates.delete(key))); await trx.objectStore('snapshots').delete(docId); await trx.objectStore('clocks').delete(docId); @@ -120,6 +122,18 @@ export class IndexedDBDocStorage extends DocStorageBase { override async getDocTimestamps(after: Date = new Date(0)) { const trx = this.db.transaction('clocks', 'readonly'); + const getAllRecords = trx.store.getAllRecords?.bind(trx.store); + + if (typeof getAllRecords === 'function') { + const records = await getAllRecords(); + return records.reduce((ret, cur) => { + if (cur.value.timestamp > after) { + ret[cur.value.docId] = cur.value.timestamp; + } + return ret; + }, {} as DocClocks); + } + const clocks = await trx.store.getAll(); return clocks.reduce((ret, cur) => { @@ -157,7 +171,19 @@ export class IndexedDBDocStorage extends DocStorageBase { protected override async getDocUpdates(docId: string): Promise { const trx = this.db.transaction('updates', 'readonly'); - const updates = await trx.store.index('docId').getAll(docId); + const idx = trx.store.index('docId'); + const getAllRecords = idx.getAllRecords?.bind(idx); + + if (typeof getAllRecords === 'function') { + const records = await getAllRecords(IDBKeyRange.only(docId)); + return records.map(record => ({ + docId, + bin: record.value.bin, + timestamp: record.value.createdAt, + })); + } + + const updates = await idx.getAll(docId); return updates.map(update => ({ docId, @@ -170,7 +196,9 @@ export class IndexedDBDocStorage extends DocStorageBase { docId: string, updates: DocRecord[] ): Promise { - const trx = this.db.transaction('updates', 'readwrite'); + const trx = this.db.transaction('updates', 'readwrite', { + durability: 'relaxed', + }); await Promise.all( updates.map(update => trx.store.delete([docId, update.timestamp]))