Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-01 17:50:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(native): move sqlite operation into Rust (#2497)

Browse Source 
Co-authored-by: Peng Xiao <pengxiao@outlook.com>
This commit is contained in:
LongYinan
2023-06-07 14:52:19 +08:00
committed by GitHub
parent 541011ba90
commit d28c887237
36 changed files with 1910 additions and 545 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/build-rust/action.yml
+8 -6
View File
@@ -29,21 +29,23 @@ runs:
if: ${{ inputs.target != 'x86_64-unknown-linux-gnu' && inputs.target != 'aarch64-unknown-linux-gnu' }}
shell: bash
run: yarn workspace @affine/native build --target ${{ inputs.target }}
env:
CARGO_BUILD_INCREMENTAL: 'false'
- name: Build
if: ${{ inputs.target == 'x86_64-unknown-linux-gnu' }}
uses: addnab/docker-run-action@v3
with:
image: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
options: --user 0:0 -e CARGO_BUILD_INCREMENTAL=false -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build
run: yarn workspace @affine/native build --target ${{ inputs.target }}
options: --user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build
run: >-
export CC=x86_64-unknown-linux-gnu-gcc &&
export CC_x86_64_unknown_linux_gnu=x86_64-unknown-linux-gnu-gcc &&
yarn workspace @affine/native build --target ${{ inputs.target }}
- name: Build
if: ${{ inputs.target == 'aarch64-unknown-linux-gnu' }}
uses: addnab/docker-run-action@v3
with:
image: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
options: --user 0:0 -e CARGO_BUILD_INCREMENTAL=false -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build
run: yarn workspace @affine/native build --target ${{ inputs.target }}
options: --user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build
run: >-
yarn workspace @affine/native build --target ${{ inputs.target }}
.github/workflows/build.yml
+2 -12
View File
@@ -118,8 +118,7 @@ jobs:
COVERAGE: true
- name: Export static resources
run: yarn export
working-directory: apps/web
run: yarn workspace @affine/web export
- name: Upload static resources artifact
uses: actions/upload-artifact@v3
@@ -327,10 +326,7 @@ jobs:
- name: Run unit tests
if: ${{ matrix.spec.test }}
shell: bash
run: |
rm -rf apps/electron/node_modules/better-sqlite3/build
yarn --cwd apps/electron/node_modules/better-sqlite3 run install
yarn test:unit
run: yarn test:unit
env:
NATIVE_TEST: 'true'
- name: Build layers
@@ -342,12 +338,6 @@ jobs:
name: next-js-static
path: ./apps/electron/resources/web-static
- name: Rebuild Electron dependences
shell: bash
run: |
rm -rf apps/electron/node_modules/better-sqlite3/build
yarn workspace @affine/electron rebuild:for-electron --arch=${{ matrix.spec.arch }}
- name: Run desktop tests
if: ${{ matrix.spec.test && matrix.spec.os == 'ubuntu-latest' }}
run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine/electron test
.github/workflows/nightly-build.yml
-5
View File
@@ -128,11 +128,6 @@ jobs:
with:
name: before-make-web-static
path: apps/electron/resources/web-static
- name: Rebuild Electron dependences
shell: bash
run: |
rm -rf apps/electron/node_modules/better-sqlite3/build
yarn workspace @affine/electron rebuild:for-electron --arch=${{ matrix.spec.arch }}
- name: Build layers
run: yarn workspace @affine/electron build-layers
.github/workflows/release-desktop-app.yml
-6
View File
@@ -126,12 +126,6 @@ jobs:
name: before-make-web-static
path: apps/electron/resources/web-static
- name: Rebuild Electron dependences
shell: bash
run: |
rm -rf apps/electron/node_modules/better-sqlite3/build
yarn workspace @affine/electron rebuild:for-electron --arch=${{ matrix.spec.arch }}
- name: Build layers
run: yarn workspace @affine/electron build-layers
.gitignore
+1
View File
@@ -73,3 +73,4 @@ target
*.node
tsconfig.node.tsbuildinfo
lib
affine.db
.vscode/settings.template.json
Vendored
+3 -1
View File
@@ -38,5 +38,7 @@
"tests/unit/**/*.spec.ts",
"tests/unit/**/*.spec.tsx"
],
"deepscan.enable": true
"rust-analyzer.check.extraEnv": {
"DATABASE_URL": "sqlite:affine.db"
}
}
Cargo.lock
Generated
+1385 -48
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+4 -1
View File
@@ -1,5 +1,8 @@
[workspace]
members = ["./packages/native"]
members = ["./packages/native", "./packages/native/schema"]
[profile.dev.package.sqlx-macros]
opt-level = 3
[profile.release]
lto = true
apps/electron/README.md
-16
View File
@@ -17,22 +17,6 @@ yarn dev # or yarn prod for production build
## Troubleshooting
### better-sqlite3 error
When running tests or starting electron, you may encounter the following error:
> Error: The module 'apps/electron/node_modules/better-sqlite3/build/Release/better_sqlite3.node'
This is due to the fact that the `better-sqlite3` package is built for the Node.js version in Electron & in your machine. To fix this, run the following command based on different cases:
```sh
# for running unit tests, we are not using Electron's node:
yarn rebuild better-sqlite3
# for running Electron, we are using Electron's node:
yarn postinstall
```
## Credits
Most of the boilerplate code is generously borrowed from the following
apps/electron/layers/main/src/__tests__/integration.spec.ts
+16 -12
View File
@@ -1,5 +1,6 @@
import assert from 'node:assert';
import path from 'node:path';
import { setTimeout } from 'node:timers/promises';
import fs from 'fs-extra';
import { v4 } from 'uuid';
@@ -13,8 +14,6 @@ const registeredHandlers = new Map<
((...args: any[]) => Promise<any>)[]
>();
const delay = (ms: number) => new Promise(r => setTimeout(r, ms));
type WithoutFirstParameter<T> = T extends (_: any, ...args: infer P) => infer R
? (...args: P) => R
: T;
@@ -72,11 +71,14 @@ const nativeTheme = {
themeSource: 'light',
};
function compareBuffer(a: Uint8Array | null, b: Uint8Array | null) {
function compareBuffer(
a: Uint8Array | null | undefined,
b: Uint8Array | null | undefined
) {
if (
(a === null && b === null) ||
a === null ||
b === null ||
(a == null && b == null) ||
a == null ||
b == null ||
a.length !== b.length
) {
return false;
@@ -105,11 +107,11 @@ const electronModule = {
handlers.push(callback);
registeredHandlers.set(name, handlers);
},
addEventListener: (...args: any[]) => {
addListener: (...args: any[]) => {
// @ts-ignore
electronModule.app.on(...args);
},
removeEventListener: () => {},
removeListener: () => {},
},
BrowserWindow: {
getAllWindows: () => {
@@ -135,7 +137,6 @@ beforeEach(async () => {
const { registerEvents } = await import('../events');
registerEvents();
await fs.mkdirp(SESSION_DATA_PATH);
await import('../db/ensure-db');
registeredHandlers.get('ready')?.forEach(fn => fn());
});
@@ -143,7 +144,10 @@ beforeEach(async () => {
afterEach(async () => {
// reset registered handlers
registeredHandlers.get('before-quit')?.forEach(fn => fn());
// wait for the db to be closed on Windows
if (process.platform === 'win32') {
await setTimeout(200);
}
await fs.remove(SESSION_DATA_PATH);
});
@@ -175,7 +179,7 @@ describe('ensureSQLiteDB', () => {
const fileExists = await fs.pathExists(file);
expect(fileExists).toBe(true);
registeredHandlers.get('before-quit')?.forEach(fn => fn());
await delay(100);
await setTimeout(100);
expect(workspaceDB.db).toBe(null);
});
});
@@ -254,7 +258,7 @@ describe('db handlers', () => {
test('get non existent blob', async () => {
const workspaceId = v4();
const bin = await dispatch('db', 'getBlob', workspaceId, 'non-existent-id');
expect(bin).toBeNull();
expect(bin).toBeUndefined();
});
test('list blobs (empty)', async () => {
apps/electron/layers/main/src/db/__tests__/ensure-db.spec.ts
+31 -13
View File
@@ -1,4 +1,5 @@
import path from 'node:path';
import { setTimeout } from 'node:timers/promises';
import fs from 'fs-extra';
import { v4 } from 'uuid';
@@ -30,18 +31,20 @@ const electronModule = {
handlers.push(callback);
registeredHandlers.set(name, handlers);
},
addEventListener: (...args: any[]) => {
addListener: (...args: any[]) => {
// @ts-ignore
electronModule.app.on(...args);
},
removeEventListener: () => {},
removeListener: () => {},
},
shell: {} as Partial<Electron.Shell>,
dialog: {} as Partial<Electron.Dialog>,
};
const runHandler = (key: string) => {
registeredHandlers.get(key)?.forEach(handler => handler());
const runHandler = async (key: string) => {
await Promise.all(
(registeredHandlers.get(key) ?? []).map(handler => handler())
);
};
// dynamically import handlers so that we can inject local variables to mocks
@@ -51,6 +54,7 @@ vi.doMock('electron', () => {
const constructorStub = vi.fn();
const destroyStub = vi.fn();
destroyStub.mockReturnValue(Promise.resolve());
vi.doMock('../secondary-db', () => {
return {
@@ -59,6 +63,10 @@ vi.doMock('../secondary-db', () => {
constructorStub(...args);
}
connectIfNeeded = () => Promise.resolve();
pull = () => Promise.resolve();
destroy = destroyStub;
},
};
@@ -69,7 +77,7 @@ beforeEach(() => {
});
afterEach(async () => {
runHandler('before-quit');
await runHandler('before-quit');
await fs.remove(tmpDir);
vi.useRealTimers();
});
@@ -98,12 +106,24 @@ test('db should be destroyed when app quits', async () => {
expect(db0.db).not.toBeNull();
expect(db1.db).not.toBeNull();
runHandler('before-quit');
await runHandler('before-quit');
// wait the async `db.destroy()` to be called
await setTimeout(100);
expect(db0.db).toBeNull();
expect(db1.db).toBeNull();
});
test('db should be removed in db$Map after destroyed', async () => {
const { ensureSQLiteDB, db$Map } = await import('../ensure-db');
const workspaceId = v4();
const db = await ensureSQLiteDB(workspaceId);
await db.destroy();
await setTimeout(100);
expect(db$Map.has(workspaceId)).toBe(false);
});
test('if db has a secondary db path, we should also poll that', async () => {
const { ensureSQLiteDB } = await import('../ensure-db');
const { appContext } = await import('../../context');
@@ -115,10 +135,7 @@ test('if db has a secondary db path, we should also poll that', async () => {
const db = await ensureSQLiteDB(workspaceId);
await vi.advanceTimersByTimeAsync(1500);
// not sure why but we still need to wait with real timer
await new Promise(resolve => setTimeout(resolve, 100));
await setTimeout(10);
expect(constructorStub).toBeCalledTimes(1);
expect(constructorStub).toBeCalledWith(path.join(tmpDir, 'secondary.db'), db);
@@ -128,7 +145,8 @@ test('if db has a secondary db path, we should also poll that', async () => {
secondaryDBPath: path.join(tmpDir, 'secondary2.db'),
});
await vi.advanceTimersByTimeAsync(1500);
// wait the async `db.destroy()` to be called
await setTimeout(100);
expect(constructorStub).toBeCalledTimes(2);
expect(destroyStub).toBeCalledTimes(1);
@@ -141,7 +159,7 @@ test('if db has a secondary db path, we should also poll that', async () => {
expect(destroyStub).toBeCalledTimes(1);
// if primary is destroyed, secondary should also be destroyed
db.destroy();
await new Promise(resolve => setTimeout(resolve, 100));
await db.destroy();
await setTimeout(100);
expect(destroyStub).toBeCalledTimes(2);
});
apps/electron/layers/main/src/db/__tests__/workspace-db-adapter.spec.ts
+6 -9
View File
@@ -16,10 +16,7 @@ const testAppContext: AppContext = {
};
afterEach(async () => {
if (process.platform !== 'win32') {
// hmmm ....
await fs.remove(tmpDir);
}
await fs.remove(tmpDir);
});
function getTestUpdates() {
@@ -40,7 +37,7 @@ test('can create new db file if not exists', async () => {
`storage.db`
);
expect(await fs.exists(dbPath)).toBe(true);
db.destroy();
await db.destroy();
});
test('on applyUpdate (from self), will not trigger update', async () => {
@@ -52,7 +49,7 @@ test('on applyUpdate (from self), will not trigger update', async () => {
db.update$.subscribe(onUpdate);
db.applyUpdate(getTestUpdates(), 'self');
expect(onUpdate).not.toHaveBeenCalled();
db.destroy();
await db.destroy();
});
test('on applyUpdate (from renderer), will trigger update', async () => {
@@ -67,7 +64,7 @@ test('on applyUpdate (from renderer), will trigger update', async () => {
db.applyUpdate(getTestUpdates(), 'renderer');
expect(onUpdate).toHaveBeenCalled(); // not yet updated
sub.unsubscribe();
db.destroy();
await db.destroy();
});
test('on applyUpdate (from external), will trigger update & send external update event', async () => {
@@ -83,7 +80,7 @@ test('on applyUpdate (from external), will trigger update & send external update
expect(onUpdate).toHaveBeenCalled();
expect(onExternalUpdate).toHaveBeenCalled();
sub.unsubscribe();
db.destroy();
await db.destroy();
});
test('on destroy, check if resources have been released', async () => {
@@ -95,7 +92,7 @@ test('on destroy, check if resources have been released', async () => {
next: vi.fn(),
};
db.update$ = updateSub as any;
db.destroy();
await db.destroy();
expect(db.db).toBe(null);
expect(updateSub.complete).toHaveBeenCalled();
});
apps/electron/layers/main/src/db/base-db-adapter.ts
+33 -87
View File
@@ -1,120 +1,76 @@
import { SqliteConnection } from '@affine/native';
import assert from 'assert';
import type { Database } from 'better-sqlite3';
import sqlite from 'better-sqlite3';
import { logger } from '../logger';
const schemas = [
`CREATE TABLE IF NOT EXISTS "updates" (
id INTEGER PRIMARY KEY AUTOINCREMENT,
data BLOB NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
)`,
`CREATE TABLE IF NOT EXISTS "blobs" (
key TEXT PRIMARY KEY NOT NULL,
data BLOB NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
)`,
];
interface UpdateRow {
id: number;
data: Buffer;
timestamp: string;
}
interface BlobRow {
key: string;
data: Buffer;
timestamp: string;
}
/**
* A base class for SQLite DB adapter that provides basic methods around updates & blobs
*/
export abstract class BaseSQLiteAdapter {
db: Database | null = null;
db: SqliteConnection | null = null;
abstract role: string;
constructor(public path: string) {}
ensureTables() {
assert(this.db, 'db is not connected');
this.db.exec(schemas.join(';'));
constructor(public readonly path: string) {
logger.info(`[SQLiteAdapter]`, 'path:', path);
}
// todo: what if SQLite DB wrapper later is not sync?
connect(): Database | undefined {
if (this.db) {
return this.db;
async connectIfNeeded() {
if (!this.db) {
this.db = new SqliteConnection(this.path);
await this.db.connect();
}
logger.log(`[SQLiteAdapter][${this.role}] open db`, this.path);
const db = (this.db = sqlite(this.path));
this.ensureTables();
return db;
return this.db;
}
destroy() {
this.db?.close();
async destroy() {
const { db } = this;
this.db = null;
await db?.close();
}
addBlob(key: string, data: Uint8Array) {
async addBlob(key: string, data: Uint8Array) {
try {
assert(this.db, 'db is not connected');
const statement = this.db.prepare(
'INSERT INTO blobs (key, data) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET data = ?'
);
statement.run(key, data, data);
return key;
assert(this.db, `${this.path} is not connected`);
await this.db.addBlob(key, data);
} catch (error) {
logger.error('addBlob', error);
}
}
getBlob(key: string) {
async getBlob(key: string) {
try {
assert(this.db, 'db is not connected');
const statement = this.db.prepare('SELECT data FROM blobs WHERE key = ?');
const row = statement.get(key) as BlobRow;
if (!row) {
return null;
}
return row.data;
assert(this.db, `${this.path} is not connected`);
const blob = await this.db.getBlob(key);
return blob?.data;
} catch (error) {
logger.error('getBlob', error);
return null;
}
}
deleteBlob(key: string) {
async deleteBlob(key: string) {
try {
assert(this.db, 'db is not connected');
const statement = this.db.prepare('DELETE FROM blobs WHERE key = ?');
statement.run(key);
assert(this.db, `${this.path} is not connected`);
await this.db.deleteBlob(key);
} catch (error) {
logger.error('deleteBlob', error);
logger.error(`${this.path} delete blob failed`, error);
}
}
getBlobKeys() {
async getBlobKeys() {
try {
assert(this.db, 'db is not connected');
const statement = this.db.prepare('SELECT key FROM blobs');
const rows = statement.all() as BlobRow[];
return rows.map(row => row.key);
assert(this.db, `${this.path} is not connected`);
return await this.db.getBlobKeys();
} catch (error) {
logger.error('getBlobKeys', error);
logger.error(`getBlobKeys failed`, error);
return [];
}
}
getUpdates() {
async getUpdates() {
try {
assert(this.db, 'db is not connected');
const statement = this.db.prepare('SELECT * FROM updates');
const rows = statement.all() as UpdateRow[];
return rows;
assert(this.db, `${this.path} is not connected`);
return await this.db.getUpdates();
} catch (error) {
logger.error('getUpdates', error);
return [];
@@ -122,22 +78,12 @@ export abstract class BaseSQLiteAdapter {
}
// add a single update to SQLite
addUpdateToSQLite(updates: Uint8Array[]) {
async addUpdateToSQLite(db: SqliteConnection, updates: Uint8Array[]) {
// batch write instead write per key stroke?
try {
assert(this.db, 'db is not connected');
const start = performance.now();
const statement = this.db.prepare(
'INSERT INTO updates (data) VALUES (?)'
);
const insertMany = this.db.transaction(updates => {
for (const d of updates) {
statement.run(d);
}
});
insertMany(updates);
await db.connect();
await db.insertUpdates(updates);
logger.debug(
`[SQLiteAdapter][${this.role}] addUpdateToSQLite`,
'length:',
apps/electron/layers/main/src/db/ensure-db.ts
+92 -56
View File
@@ -1,12 +1,14 @@
import { app } from 'electron';
import type { Subject } from 'rxjs';
import { Observable } from 'rxjs';
import {
concat,
defer,
firstValueFrom,
from,
fromEvent,
interval,
lastValueFrom,
merge,
Observable,
} from 'rxjs';
import {
distinctUntilChanged,
@@ -17,41 +19,83 @@ import {
shareReplay,
startWith,
switchMap,
take,
takeUntil,
tap,
} from 'rxjs/operators';
import { appContext } from '../context';
import { logger } from '../logger';
import { getWorkspaceMeta$ } from '../workspace';
import { getWorkspaceMeta, workspaceSubjects } from '../workspace';
import { SecondaryWorkspaceSQLiteDB } from './secondary-db';
import type { WorkspaceSQLiteDB } from './workspace-db-adapter';
import { openWorkspaceDatabase } from './workspace-db-adapter';
const db$Map = new Map<string, Observable<WorkspaceSQLiteDB>>();
// export for testing
export const db$Map = new Map<string, Observable<WorkspaceSQLiteDB>>();
// use defer to prevent `app` is undefined while running tests
const beforeQuit$ = defer(() => fromEvent(app, 'before-quit'));
// return a stream that emit a single event when the subject completes
function completed<T>(subject: Subject<T>) {
return new Observable(subscriber => {
const sub = subject.subscribe({
complete: () => {
subscriber.next();
subscriber.complete();
},
});
return () => sub.unsubscribe();
});
}
function getWorkspaceDB$(id: string) {
if (!db$Map.has(id)) {
db$Map.set(
id,
from(openWorkspaceDatabase(appContext, id)).pipe(
shareReplay(1),
switchMap(db => {
return startPollingSecondaryDB(db).pipe(
ignoreElements(),
startWith(db),
takeUntil(beforeQuit$),
tap({
complete: () => {
logger.info('[ensureSQLiteDB] close db connection');
db.destroy();
db$Map.delete(id);
},
})
);
tap({
next: db => {
logger.info(
'[ensureSQLiteDB] db connection established',
db.workspaceId
);
},
}),
switchMap(db =>
// takeUntil the polling stream, and then destroy the db
concat(
startPollingSecondaryDB(db).pipe(
ignoreElements(),
startWith(db),
takeUntil(merge(beforeQuit$, completed(db.update$))),
last(),
tap({
next() {
logger.info(
'[ensureSQLiteDB] polling secondary db complete',
db.workspaceId
);
},
})
),
defer(async () => {
try {
await db.destroy();
db$Map.delete(id);
logger.info(
'[ensureSQLiteDB] db connection destroyed',
db.workspaceId
);
return db;
} catch (err) {
logger.error('[ensureSQLiteDB] destroy db failed', err);
throw err;
}
})
).pipe(startWith(db))
),
shareReplay(1)
)
);
@@ -60,51 +104,43 @@ function getWorkspaceDB$(id: string) {
}
function startPollingSecondaryDB(db: WorkspaceSQLiteDB) {
const meta$ = getWorkspaceMeta$(db.workspaceId);
const secondaryDB$ = meta$.pipe(
return merge(
getWorkspaceMeta(appContext, db.workspaceId),
workspaceSubjects.meta.pipe(
map(({ meta }) => meta),
filter(meta => meta.id === db.workspaceId)
)
).pipe(
map(meta => meta?.secondaryDBPath),
distinctUntilChanged(),
filter((p): p is string => !!p),
distinctUntilChanged(),
switchMap(path => {
return new Observable<SecondaryWorkspaceSQLiteDB>(observer => {
const secondaryDB = new SecondaryWorkspaceSQLiteDB(path, db);
observer.next(secondaryDB);
return () => {
logger.info(
'[ensureSQLiteDB] close secondary db connection',
secondaryDB.path
);
secondaryDB.destroy();
};
// on secondary db path change, destroy the old db and create a new one
const secondaryDB = new SecondaryWorkspaceSQLiteDB(path, db);
return new Observable<SecondaryWorkspaceSQLiteDB>(subscriber => {
subscriber.next(secondaryDB);
return () => secondaryDB.destroy();
});
}),
takeUntil(db.update$.pipe(last())),
shareReplay(1)
switchMap(secondaryDB => {
return interval(300000).pipe(
startWith(0),
tap({
next: () => {
secondaryDB.pull();
},
error: err => {
logger.error(`[ensureSQLiteDB] polling secondary db error`, err);
},
complete: () => {
logger.info('[ensureSQLiteDB] polling secondary db complete');
},
})
);
})
);
const firstDelayedTick$ = defer(() => {
return new Promise<number>(resolve =>
setTimeout(() => {
resolve(0);
}, 1000)
);
});
// pull every 30 seconds
const poll$ = merge(firstDelayedTick$, interval(30000)).pipe(
switchMap(() => secondaryDB$),
tap({
next: secondaryDB => {
secondaryDB.pull();
},
}),
takeUntil(db.update$.pipe(last())),
shareReplay(1)
);
return poll$;
}
export function ensureSQLiteDB(id: string) {
return firstValueFrom(getWorkspaceDB$(id));
return lastValueFrom(getWorkspaceDB$(id).pipe(take(1)));
}
apps/electron/layers/main/src/db/helper.ts
-38
View File
@@ -1,38 +0,0 @@
import type { Database } from 'better-sqlite3';
import sqlite from 'better-sqlite3';
import { logger } from '../logger';
export function isValidateDB(db: Database) {
// check if db has two tables, one for updates and one for blobs
const statement = db.prepare(
`SELECT name FROM sqlite_schema WHERE type='table'`
);
const rows = statement.all() as { name: string }[];
const tableNames = rows.map(row => row.name);
if (!tableNames.includes('updates') || !tableNames.includes('blobs')) {
return false;
}
}
export function isValidDBFile(path: string) {
let db: Database | null = null;
try {
db = sqlite(path);
// check if db has two tables, one for updates and one for blobs
const statement = db.prepare(
`SELECT name FROM sqlite_schema WHERE type='table'`
);
const rows = statement.all() as { name: string }[];
const tableNames = rows.map(row => row.name);
if (!tableNames.includes('updates') || !tableNames.includes('blobs')) {
return false;
}
return true;
} catch (error) {
logger.error('isValidDBFile', error);
return false;
} finally {
db?.close();
}
}
apps/electron/layers/main/src/db/secondary-db.ts
+41 -35
View File
@@ -1,3 +1,6 @@
import assert from 'node:assert';
import type { SqliteConnection } from '@affine/native';
import { debounce } from 'lodash-es';
import * as Y from 'yjs';
@@ -30,16 +33,12 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
logger.debug('[SecondaryWorkspaceSQLiteDB] created', this.workspaceId);
}
close() {
this.db?.close();
this.db = null;
}
override destroy() {
this.flushUpdateQueue();
override async destroy() {
const { db } = this;
await this.flushUpdateQueue(db);
this.unsubscribers.forEach(unsub => unsub());
this.db?.close();
this.yDoc.destroy();
await super.destroy();
}
get workspaceId() {
@@ -48,12 +47,15 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
// do not update db immediately, instead, push to a queue
// and flush the queue in a future time
addUpdateToUpdateQueue(update: Uint8Array) {
async addUpdateToUpdateQueue(db: SqliteConnection, update: Uint8Array) {
this.updateQueue.push(update);
this.debouncedFlush();
await this.debouncedFlush(db);
}
flushUpdateQueue() {
async flushUpdateQueue(db = this.db) {
if (!db) {
return; // skip if db is not connected
}
logger.debug(
'flushUpdateQueue',
this.workspaceId,
@@ -62,9 +64,8 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
);
const updates = [...this.updateQueue];
this.updateQueue = [];
this.connect();
this.addUpdateToSQLite(updates);
this.close();
await db.connect();
await this.addUpdateToSQLite(db, updates);
}
// flush after 5s, but will not wait for more than 10s
@@ -75,29 +76,31 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
runCounter = 0;
// wrap the fn with connect and close
// it only works for sync functions
run = <T extends (...args: any[]) => any>(fn: T) => {
async run<T extends (...args: any[]) => any>(
fn: T
): Promise<
(T extends (...args: any[]) => infer U ? Awaited<U> : unknown) | undefined
> {
try {
if (this.runCounter === 0) {
this.connect();
}
await this.connectIfNeeded();
this.runCounter++;
return fn();
return await fn();
} catch (err) {
logger.error(err);
} finally {
this.runCounter--;
if (this.runCounter === 0) {
this.close();
await super.destroy();
}
}
};
}
setupAndListen() {
if (this.firstConnected) {
return;
}
this.firstConnected = true;
const { db } = this;
const onUpstreamUpdate = (update: Uint8Array, origin: YOrigin) => {
if (origin === 'renderer') {
@@ -109,7 +112,7 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
const onSelfUpdate = (update: Uint8Array, origin: YOrigin) => {
// for self update from upstream, we need to push it to external DB
if (origin === 'upstream') {
this.addUpdateToUpdateQueue(update);
this.addUpdateToUpdateQueue(db!, update);
}
if (origin === 'self') {
@@ -126,13 +129,11 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
this.yDoc.off('update', onSelfUpdate);
});
this.run(() => {
this.run(async () => {
// apply all updates from upstream
const upstreamUpdate = this.upstream.getDocAsUpdates();
// to initialize the yDoc, we need to apply all updates from the db
this.applyUpdate(upstreamUpdate, 'upstream');
this.pull();
});
}
@@ -141,17 +142,17 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
};
// TODO: have a better solution to handle blobs
syncBlobs() {
this.run(() => {
async syncBlobs() {
await this.run(async () => {
// pull blobs
const blobsKeys = this.getBlobKeys();
const upstreamBlobsKeys = this.upstream.getBlobKeys();
const blobsKeys = await this.getBlobKeys();
const upstreamBlobsKeys = await this.upstream.getBlobKeys();
// put every missing blob to upstream
for (const key of blobsKeys) {
if (!upstreamBlobsKeys.includes(key)) {
const blob = this.getBlob(key);
const blob = await this.getBlob(key);
if (blob) {
this.upstream.addBlob(key, blob);
await this.upstream.addBlob(key, blob);
logger.debug('syncBlobs', this.workspaceId, key);
}
}
@@ -170,12 +171,17 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
*/
async pull() {
const start = performance.now();
const updates = this.run(() => {
assert(this.upstream.db, 'upstream db should be connected');
const updates = await this.run(async () => {
// TODO: no need to get all updates, just get the latest ones (using a cursor, etc)?
this.syncBlobs();
return this.getUpdates().map(update => update.data);
await this.syncBlobs();
return (await this.getUpdates()).map(update => update.data);
});
if (!updates) {
return;
}
const merged = await mergeUpdateWorker(updates);
this.applyUpdate(merged, 'self');
apps/electron/layers/main/src/db/workspace-db-adapter.ts
+17 -16
View File
@@ -1,4 +1,4 @@
import type { Database } from 'better-sqlite3';
import type { SqliteConnection } from '@affine/native';
import { Subject } from 'rxjs';
import * as Y from 'yjs';
@@ -21,38 +21,38 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
super(path);
}
override destroy() {
this.db?.close();
this.db = null;
override async destroy() {
await super.destroy();
this.yDoc.destroy();
// when db is closed, we can safely remove it from ensure-db list
this.update$.complete();
this.firstConnected = false;
}
getWorkspaceName = () => {
return this.yDoc.getMap('space:meta').get('name') as string;
};
async init(): Promise<Database | undefined> {
const db = super.connect();
async init() {
const db = await super.connectIfNeeded();
if (!this.firstConnected) {
this.yDoc.on('update', (update: Uint8Array, origin: YOrigin) => {
this.yDoc.on('update', async (update: Uint8Array, origin: YOrigin) => {
if (origin === 'renderer') {
this.addUpdateToSQLite([update]);
await this.addUpdateToSQLite(db, [update]);
} else if (origin === 'external') {
this.addUpdateToSQLite([update]);
logger.debug('external update', this.workspaceId);
dbSubjects.externalUpdate.next({
workspaceId: this.workspaceId,
update,
});
await this.addUpdateToSQLite(db, [update]);
logger.debug('external update', this.workspaceId);
}
});
}
const updates = this.getUpdates();
const updates = await this.getUpdates();
const merged = await mergeUpdateWorker(updates.map(update => update.data));
// to initialize the yDoc, we need to apply all updates from the db
@@ -78,19 +78,19 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
Y.applyUpdate(this.yDoc, data, origin);
};
override addBlob(key: string, value: Uint8Array) {
const res = super.addBlob(key, value);
override async addBlob(key: string, value: Uint8Array) {
const res = await super.addBlob(key, value);
this.update$.next();
return res;
}
override deleteBlob(key: string) {
override async deleteBlob(key: string) {
super.deleteBlob(key);
this.update$.next();
}
override addUpdateToSQLite(data: Uint8Array[]) {
super.addUpdateToSQLite(data);
override async addUpdateToSQLite(db: SqliteConnection, data: Uint8Array[]) {
super.addUpdateToSQLite(db, data);
this.update$.next();
}
}
@@ -102,5 +102,6 @@ export async function openWorkspaceDatabase(
const meta = await getWorkspaceMeta(context, workspaceId);
const db = new WorkspaceSQLiteDB(meta.mainDBPath, workspaceId);
await db.init();
logger.info(`openWorkspaceDatabase [${workspaceId}]`);
return db;
}
apps/electron/layers/main/src/dialog/dialog.ts
+6 -3
View File
@@ -7,7 +7,6 @@ import { nanoid } from 'nanoid';
import { appContext } from '../context';
import { ensureSQLiteDB } from '../db/ensure-db';
import { isValidDBFile } from '../db/helper';
import type { WorkspaceSQLiteDB } from '../db/workspace-db-adapter';
import { logger } from '../logger';
import {
@@ -208,7 +207,9 @@ export async function loadDBFile(): Promise<LoadDBFileResult> {
return { error: 'DB_FILE_ALREADY_LOADED' };
}
if (!isValidDBFile(filePath)) {
const { SqliteConnection } = await import('@affine/native');
if (!(await SqliteConnection.validate(filePath))) {
// TODO: report invalid db file error?
return { error: 'DB_FILE_INVALID' }; // invalid db file
}
@@ -305,7 +306,9 @@ export async function moveDBFile(
// remove the old db file, but we don't care if it fails
if (meta.secondaryDBPath) {
fs.remove(meta.secondaryDBPath);
fs.remove(meta.secondaryDBPath).catch(err => {
logger.error(`[moveDBFile] remove ${meta.secondaryDBPath} failed`, err);
});
}
// update meta
apps/electron/layers/main/src/workspace/__tests__/handlers.spec.ts
-34
View File
@@ -172,37 +172,3 @@ test('storeWorkspaceMeta', async () => {
secondaryDBPath: path.join(tmpDir, 'test.db'),
});
});
test('getWorkspaceMeta observable', async () => {
const { storeWorkspaceMeta } = await import('../handlers');
const { getWorkspaceMeta$ } = await import('../index');
const workspaceId = v4();
const workspacePath = path.join(
testAppContext.appDataPath,
'workspaces',
workspaceId
);
const metaChange = vi.fn();
const meta$ = getWorkspaceMeta$(workspaceId);
meta$.subscribe(metaChange);
await new Promise(resolve => setTimeout(resolve, 100));
expect(metaChange).toHaveBeenCalledWith({
id: workspaceId,
mainDBPath: path.join(workspacePath, 'storage.db'),
});
await storeWorkspaceMeta(testAppContext, workspaceId, {
secondaryDBPath: path.join(tmpDir, 'test.db'),
});
expect(metaChange).toHaveBeenCalledWith({
id: workspaceId,
mainDBPath: path.join(workspacePath, 'storage.db'),
secondaryDBPath: path.join(tmpDir, 'test.db'),
});
});
apps/electron/layers/main/src/workspace/handlers.ts
+1 -1
View File
@@ -41,7 +41,7 @@ export async function deleteWorkspace(context: AppContext, id: string) {
);
try {
const db = await ensureSQLiteDB(id);
db.destroy();
await db.destroy();
return await fs.move(basePath, movedPath, {
overwrite: true,
});
apps/electron/layers/main/src/workspace/index.ts
-14
View File
@@ -1,6 +1,3 @@
import { merge } from 'rxjs';
import { filter, map } from 'rxjs/operators';
import { appContext } from '../context';
import type {
MainEventListener,
@@ -31,14 +28,3 @@ export const workspaceHandlers = {
return getWorkspaceMeta(appContext, id);
},
} satisfies NamespaceHandlers;
// used internally. Get a stream of workspace id -> meta
export const getWorkspaceMeta$ = (workspaceId: string) => {
return merge(
getWorkspaceMeta(appContext, workspaceId),
workspaceSubjects.meta.pipe(
map(meta => meta.meta),
filter(meta => meta.id === workspaceId)
)
);
};
apps/electron/package.json
+4 -9
View File
@@ -10,15 +10,13 @@
"description": "AFFiNE App",
"homepage": "https://github.com/toeverything/AFFiNE",
"scripts": {
"dev": "yarn electron-rebuild && yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs",
"watch": "yarn electron-rebuild && yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs --watch",
"prod": "yarn electron-rebuild && yarn node scripts/dev.mjs",
"dev": "yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs",
"watch": "yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs --watch",
"prod": "yarn node scripts/dev.mjs",
"build-layers": "zx scripts/build-layers.mjs",
"generate-assets": "zx scripts/generate-assets.mjs",
"package": "electron-forge package",
"make": "electron-forge make",
"rebuild:for-unit-test": "yarn rebuild better-sqlite3",
"rebuild:for-electron": "yarn electron-rebuild",
"test": "playwright test"
},
"config": {
@@ -36,9 +34,7 @@
"@electron-forge/maker-squirrel": "^6.1.1",
"@electron-forge/maker-zip": "^6.1.1",
"@electron-forge/shared-types": "^6.1.1",
"@electron/rebuild": "^3.2.13",
"@electron/remote": "2.0.9",
"@types/better-sqlite3": "^7.6.4",
"@types/fs-extra": "^11.0.1",
"@types/uuid": "^9.0.1",
"cross-env": "7.0.3",
@@ -55,8 +51,7 @@
"zx": "^7.2.2"
},
"dependencies": {
"better-sqlite3": "^8.4.0",
"chokidar": "^3.5.3",
"cheerio": "^1.0.0-rc.12",
"electron-updater": "^5.3.0",
"lodash-es": "^4.17.21",
"nanoid": "^4.0.2",
apps/electron/scripts/common.mjs
+1 -1
View File
@@ -46,7 +46,7 @@ export const config = () => {
bundle: true,
target: `node${NODE_MAJOR_VERSION}`,
platform: 'node',
external: ['electron', 'yjs', 'better-sqlite3', 'electron-updater'],
external: ['electron', 'yjs', 'electron-updater'],
define: define,
format: 'cjs',
loader: {
packages/native/.env
+1
View File
@@ -0,0 +1 @@
DATABASE_URL="sqlite:affine.db"
packages/native/.gitignore
+1
View File
@@ -1 +1,2 @@
*.fixture
lib
packages/native/Cargo.toml
+23 -2
View File
@@ -7,13 +7,15 @@ version = "0.0.0"
crate-type = ["cdylib"]
[dependencies]
affine_schema = { path = "./schema" }
anyhow = "1"
# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
chrono = "0.4"
napi = { version = "2", default-features = false, features = [
"napi4",
"napi5",
"tokio_rt",
"serde-json",
"error_anyhow",
"chrono_date",
] }
napi-derive = "2"
notify = { version = "6", features = ["serde"] }
@@ -21,6 +23,13 @@ once_cell = "1"
parking_lot = "0.12"
serde = "1"
serde_json = "1"
sqlx = { version = "0.7.0-alpha.3", default-features = false, features = [
"sqlite",
"runtime-tokio",
"tls-rustls",
"chrono",
"macros",
] }
tokio = { version = "1", features = ["full"] }
uuid = { version = "1", default-features = false, features = [
"serde",
@@ -29,4 +38,16 @@ uuid = { version = "1", default-features = false, features = [
] }
[build-dependencies]
affine_schema = { path = "./schema" }
dotenv = "0.15"
napi-build = "2"
sqlx = { version = "0.7.0-alpha.3", default-features = false, features = [
"sqlite",
"runtime-tokio",
"tls-rustls",
"chrono",
"macros",
"migrate",
"json",
] }
tokio = { version = "1", features = ["full"] }
packages/native/affine.db
Symlink
+1
View File
@@ -0,0 +1 @@
../../affine.db
packages/native/build.rs
+19 -2
View File
@@ -1,6 +1,23 @@
extern crate napi_build;
use sqlx::sqlite::SqliteConnectOptions;
#[tokio::main]
async fn main() -> Result<(), std::io::Error> {
dotenv::dotenv().ok();
fn main() -> Result<(), std::io::Error> {
napi_build::setup();
let options = SqliteConnectOptions::new()
.filename("../../affine.db")
.journal_mode(sqlx::sqlite::SqliteJournalMode::Off)
.locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive)
.create_if_missing(true);
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect_with(options)
.await
.unwrap();
sqlx::query(affine_schema::SCHEMA)
.execute(&pool)
.await
.unwrap();
Ok(())
}
packages/native/index.d.ts
Vendored
+24 -1
View File
@@ -7,7 +7,7 @@ export interface WatchOptions {
recursive?: boolean;
}
/** Watcher kind enumeration */
export enum WatcherKind {
export const enum WatcherKind {
/** inotify backend (linux) */
Inotify = 'Inotify',
/** FS-Event backend (mac) */
@@ -23,6 +23,16 @@ export enum WatcherKind {
Unknown = 'Unknown',
}
export function moveFile(src: string, dst: string): Promise<void>;
export interface BlobRow {
key: string;
data: Buffer;
timestamp: Date;
}
export interface UpdateRow {
id: number;
timestamp: Date;
data: Buffer;
}
export class Subscription {
toString(): string;
unsubscribe(): void;
@@ -39,3 +49,16 @@ export class FsWatcher {
static unwatch(p: string): void;
static close(): void;
}
export class SqliteConnection {
constructor(path: string);
connect(): Promise<void>;
addBlob(key: string, blob: Uint8Array): Promise<void>;
getBlob(key: string): Promise<BlobRow | null>;
deleteBlob(key: string): Promise<void>;
getBlobKeys(): Promise<Array<string>>;
getUpdates(): Promise<Array<UpdateRow>>;
insertUpdates(updates: Array<Uint8Array>): Promise<void>;
close(): Promise<void>;
get isClose(): boolean;
static validate(path: string): Promise<boolean>;
}
packages/native/index.js
+3 -1
View File
@@ -263,9 +263,11 @@ if (!nativeBinding) {
throw new Error(`Failed to load native binding`);
}
const { WatcherKind, Subscription, FsWatcher, moveFile } = nativeBinding;
const { WatcherKind, Subscription, FsWatcher, moveFile, SqliteConnection } =
nativeBinding;
module.exports.WatcherKind = WatcherKind;
module.exports.Subscription = Subscription;
module.exports.FsWatcher = FsWatcher;
module.exports.moveFile = moveFile;
module.exports.SqliteConnection = SqliteConnection;
packages/native/schema/Cargo.toml
+4
View File
@@ -0,0 +1,4 @@
[package]
edition = "2021"
name = "affine_schema"
version = "0.0.0"
packages/native/schema/README.md
+1
View File
@@ -0,0 +1 @@
A temporary crate to share the schema between AFFiNE native and `build.rs` in the AFFiNE native.
packages/native/schema/src/lib.rs
+13
View File
@@ -0,0 +1,13 @@
// TODO
// dynamic create it from JavaScript side
// and remove this crate then.
pub const SCHEMA: &str = r#"CREATE TABLE IF NOT EXISTS "updates" (
id INTEGER PRIMARY KEY AUTOINCREMENT,
data BLOB NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
);
CREATE TABLE IF NOT EXISTS "blobs" (
key TEXT PRIMARY KEY NOT NULL,
data BLOB NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
);"#;
packages/native/src/lib.rs
+1
View File
@@ -1 +1,2 @@
pub mod fs;
pub mod sqlite;
packages/native/src/sqlite/mod.rs
+161
View File
@@ -0,0 +1,161 @@
use chrono::NaiveDateTime;
use napi::bindgen_prelude::{Buffer, Uint8Array};
use napi_derive::napi;
use sqlx::{
migrate::MigrateDatabase,
sqlite::{Sqlite, SqliteConnectOptions, SqlitePoolOptions},
Pool, Row,
};
#[napi(object)]
pub struct BlobRow {
pub key: String,
pub data: Buffer,
pub timestamp: NaiveDateTime,
}
#[napi(object)]
pub struct UpdateRow {
pub id: i64,
pub timestamp: NaiveDateTime,
pub data: Buffer,
}
#[napi]
pub struct SqliteConnection {
pool: Pool<Sqlite>,
path: String,
}
#[napi]
impl SqliteConnection {
#[napi(constructor)]
pub fn new(path: String) -> napi::Result<Self> {
let sqlite_options = SqliteConnectOptions::new()
.filename(&path)
.foreign_keys(false)
.journal_mode(sqlx::sqlite::SqliteJournalMode::Off);
let pool = SqlitePoolOptions::new()
.max_connections(4)
.connect_lazy_with(sqlite_options);
Ok(Self { pool, path })
}
#[napi]
pub async fn connect(&self) -> napi::Result<()> {
if !Sqlite::database_exists(&self.path).await.unwrap_or(false) {
Sqlite::create_database(&self.path)
.await
.map_err(anyhow::Error::from)?;
};
let mut connection = self.pool.acquire().await.map_err(anyhow::Error::from)?;
sqlx::query(affine_schema::SCHEMA)
.execute(connection.as_mut())
.await
.map_err(anyhow::Error::from)?;
connection.detach();
Ok(())
}
#[napi]
pub async fn add_blob(&self, key: String, blob: Uint8Array) -> napi::Result<()> {
let blob = blob.as_ref();
sqlx::query_as!(
BlobRow,
"INSERT INTO blobs (key, data) VALUES ($1, $2) ON CONFLICT(key) DO UPDATE SET data = excluded.data",
key,
blob,
)
.execute(&self.pool)
.await
.map_err(anyhow::Error::from)?;
Ok(())
}
#[napi]
pub async fn get_blob(&self, key: String) -> Option<BlobRow> {
sqlx::query_as!(BlobRow, "SELECT * FROM blobs WHERE key = ?", key)
.fetch_one(&self.pool)
.await
.ok()
}
#[napi]
pub async fn delete_blob(&self, key: String) -> napi::Result<()> {
sqlx::query!("DELETE FROM blobs WHERE key = ?", key)
.execute(&self.pool)
.await
.map_err(anyhow::Error::from)?;
Ok(())
}
#[napi]
pub async fn get_blob_keys(&self) -> napi::Result<Vec<String>> {
let keys = sqlx::query!("SELECT key FROM blobs")
.fetch_all(&self.pool)
.await
.map(|rows| rows.into_iter().map(|row| row.key).collect())
.map_err(anyhow::Error::from)?;
Ok(keys)
}
#[napi]
pub async fn get_updates(&self) -> napi::Result<Vec<UpdateRow>> {
let updates = sqlx::query_as!(UpdateRow, "SELECT * FROM updates")
.fetch_all(&self.pool)
.await
.map_err(anyhow::Error::from)?;
Ok(updates)
}
#[napi]
pub async fn insert_updates(&self, updates: Vec<Uint8Array>) -> napi::Result<()> {
let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
for update in updates.into_iter() {
let update = update.as_ref();
sqlx::query_as!(UpdateRow, "INSERT INTO updates (data) VALUES ($1)", update)
.execute(&mut *transaction)
.await
.map_err(anyhow::Error::from)?;
}
transaction.commit().await.map_err(anyhow::Error::from)?;
Ok(())
}
#[napi]
pub async fn close(&self) {
self.pool.close().await;
}
#[napi(getter)]
pub fn is_close(&self) -> bool {
self.pool.is_closed()
}
#[napi]
pub async fn validate(path: String) -> bool {
if let Ok(pool) = SqlitePoolOptions::new()
.max_connections(1)
.connect(&path)
.await
{
if let Ok(res) = sqlx::query("SELECT name FROM sqlite_master WHERE type='table'")
.fetch_all(&pool)
.await
{
let names = res.iter().map(|row| row.get(0));
names.fold(0, |acc, cur: String| {
if cur == "updates" || cur == "blobs" {
acc + 1
} else {
acc
}
}) == 2
} else {
false
}
} else {
false
}
}
}
yarn.lock
+7 -116
View File
@@ -178,13 +178,10 @@ __metadata:
"@electron-forge/maker-squirrel": ^6.1.1
"@electron-forge/maker-zip": ^6.1.1
"@electron-forge/shared-types": ^6.1.1
"@electron/rebuild": ^3.2.13
"@electron/remote": 2.0.9
"@types/better-sqlite3": ^7.6.4
"@types/fs-extra": ^11.0.1
"@types/uuid": ^9.0.1
better-sqlite3: ^8.4.0
chokidar: ^3.5.3
cheerio: ^1.0.0-rc.12
cross-env: 7.0.3
electron: 25.0.1
electron-log: ^5.0.0-beta.24
@@ -3317,7 +3314,7 @@ __metadata:
languageName: node
linkType: hard
"@electron/rebuild@npm:^3.2.10, @electron/rebuild@npm:^3.2.13":
"@electron/rebuild@npm:^3.2.10":
version: 3.2.13
resolution: "@electron/rebuild@npm:3.2.13"
dependencies:
@@ -9233,15 +9230,6 @@ __metadata:
languageName: node
linkType: hard
"@types/better-sqlite3@npm:^7.6.4":
version: 7.6.4
resolution: "@types/better-sqlite3@npm:7.6.4"
dependencies:
"@types/node": "*"
checksum: 75ab00d31b56437cc65fe15ff673cf8d1609edca52628083921bcbab1cbd828d135a2859fb4e68af8ef5a4801705ba99d54b96499f997bce65dd306ade3dbe58
languageName: node
linkType: hard
"@types/body-parser@npm:*":
version: 1.19.2
resolution: "@types/body-parser@npm:1.19.2"
@@ -11689,17 +11677,6 @@ __metadata:
languageName: node
linkType: hard
"better-sqlite3@npm:^8.4.0":
version: 8.4.0
resolution: "better-sqlite3@npm:8.4.0"
dependencies:
bindings: ^1.5.0
node-gyp: latest
prebuild-install: ^7.1.0
checksum: f8b180c26428a2d381482e83b4519d0d81a918e00f92cafd255f42eb9583f49b2fed1015121ad49ebe1f3f790cc8c6f4697d1e805193d65e70dacf2e8abbd6af
languageName: node
linkType: hard
"big-integer@npm:^1.6.44":
version: 1.6.51
resolution: "big-integer@npm:1.6.51"
@@ -11728,15 +11705,6 @@ __metadata:
languageName: node
linkType: hard
"bindings@npm:^1.5.0":
version: 1.5.0
resolution: "bindings@npm:1.5.0"
dependencies:
file-uri-to-path: 1.0.0
checksum: 65b6b48095717c2e6105a021a7da4ea435aa8d3d3cd085cb9e85bcb6e5773cf318c4745c3f7c504412855940b585bdf9b918236612a1c7a7942491de176f1ae7
languageName: node
linkType: hard
"bl@npm:^4.0.3, bl@npm:^4.1.0":
version: 4.1.0
resolution: "bl@npm:4.1.0"
@@ -13708,7 +13676,7 @@ __metadata:
languageName: node
linkType: hard
"detect-libc@npm:^2.0.0, detect-libc@npm:^2.0.1":
"detect-libc@npm:^2.0.1":
version: 2.0.1
resolution: "detect-libc@npm:2.0.1"
checksum: ccb05fcabbb555beb544d48080179c18523a343face9ee4e1a86605a8715b4169f94d663c21a03c310ac824592f2ba9a5270218819bb411ad7be578a527593d7
@@ -15412,13 +15380,6 @@ __metadata:
languageName: node
linkType: hard
"expand-template@npm:^2.0.3":
version: 2.0.3
resolution: "expand-template@npm:2.0.3"
checksum: 588c19847216421ed92befb521767b7018dc88f88b0576df98cb242f20961425e96a92cbece525ef28cc5becceae5d544ae0f5b9b5e2aa05acb13716ca5b3099
languageName: node
linkType: hard
"expand-tilde@npm:^1.2.2":
version: 1.2.2
resolution: "expand-tilde@npm:1.2.2"
@@ -15773,13 +15734,6 @@ __metadata:
languageName: node
linkType: hard
"file-uri-to-path@npm:1.0.0":
version: 1.0.0
resolution: "file-uri-to-path@npm:1.0.0"
checksum: b648580bdd893a008c92c7ecc96c3ee57a5e7b6c4c18a9a09b44fb5d36d79146f8e442578bc0e173dc027adf3987e254ba1dfd6e3ec998b7c282873010502144
languageName: node
linkType: hard
"filelist@npm:^1.0.4":
version: 1.0.4
resolution: "filelist@npm:1.0.4"
@@ -16633,13 +16587,6 @@ __metadata:
languageName: node
linkType: hard
"github-from-package@npm:0.0.0":
version: 0.0.0
resolution: "github-from-package@npm:0.0.0"
checksum: 14e448192a35c1e42efee94c9d01a10f42fe790375891a24b25261246ce9336ab9df5d274585aedd4568f7922246c2a78b8a8cd2571bfe99c693a9718e7dd0e3
languageName: node
linkType: hard
"github-slugger@npm:^1.0.0":
version: 1.5.0
resolution: "github-slugger@npm:1.5.0"
@@ -20892,7 +20839,7 @@ __metadata:
languageName: node
linkType: hard
"minimist@npm:^1.1.1, minimist@npm:^1.1.3, minimist@npm:^1.2.0, minimist@npm:^1.2.3, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7, minimist@npm:^1.2.8, minimist@npm:~1.2.5":
"minimist@npm:^1.1.1, minimist@npm:^1.1.3, minimist@npm:^1.2.0, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7, minimist@npm:^1.2.8, minimist@npm:~1.2.5":
version: 1.2.8
resolution: "minimist@npm:1.2.8"
checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0
@@ -20976,7 +20923,7 @@ __metadata:
languageName: node
linkType: hard
"mkdirp-classic@npm:^0.5.2, mkdirp-classic@npm:^0.5.3":
"mkdirp-classic@npm:^0.5.2":
version: 0.5.3
resolution: "mkdirp-classic@npm:0.5.3"
checksum: 3f4e088208270bbcc148d53b73e9a5bd9eef05ad2cbf3b3d0ff8795278d50dd1d11a8ef1875ff5aea3fa888931f95bfcb2ad5b7c1061cfefd6284d199e6776ac
@@ -21255,13 +21202,6 @@ __metadata:
languageName: node
linkType: hard
"napi-build-utils@npm:^1.0.1":
version: 1.0.2
resolution: "napi-build-utils@npm:1.0.2"
checksum: 06c14271ee966e108d55ae109f340976a9556c8603e888037145d6522726aebe89dd0c861b4b83947feaf6d39e79e08817559e8693deedc2c94e82c5cbd090c7
languageName: node
linkType: hard
"natural-compare-lite@npm:^1.4.0":
version: 1.4.0
resolution: "natural-compare-lite@npm:1.4.0"
@@ -21396,7 +21336,7 @@ __metadata:
languageName: node
linkType: hard
"node-abi@npm:^3.0.0, node-abi@npm:^3.3.0":
"node-abi@npm:^3.0.0":
version: 3.40.0
resolution: "node-abi@npm:3.40.0"
dependencies:
@@ -22767,28 +22707,6 @@ __metadata:
languageName: node
linkType: hard
"prebuild-install@npm:^7.1.0":
version: 7.1.1
resolution: "prebuild-install@npm:7.1.1"
dependencies:
detect-libc: ^2.0.0
expand-template: ^2.0.3
github-from-package: 0.0.0
minimist: ^1.2.3
mkdirp-classic: ^0.5.3
napi-build-utils: ^1.0.1
node-abi: ^3.3.0
pump: ^3.0.0
rc: ^1.2.7
simple-get: ^4.0.0
tar-fs: ^2.0.0
tunnel-agent: ^0.6.0
bin:
prebuild-install: bin.js
checksum: dbf96d0146b6b5827fc8f67f72074d2e19c69628b9a7a0a17d0fad1bf37e9f06922896972e074197fc00a52eae912993e6ef5a0d471652f561df5cb516f3f467
languageName: node
linkType: hard
"precinct@npm:^8.1.0":
version: 8.3.1
resolution: "precinct@npm:8.3.1"
@@ -24884,24 +24802,6 @@ __metadata:
languageName: node
linkType: hard
"simple-concat@npm:^1.0.0":
version: 1.0.1
resolution: "simple-concat@npm:1.0.1"
checksum: 4d211042cc3d73a718c21ac6c4e7d7a0363e184be6a5ad25c8a1502e49df6d0a0253979e3d50dbdd3f60ef6c6c58d756b5d66ac1e05cda9cacd2e9fc59e3876a
languageName: node
linkType: hard
"simple-get@npm:^4.0.0":
version: 4.0.1
resolution: "simple-get@npm:4.0.1"
dependencies:
decompress-response: ^6.0.0
once: ^1.3.1
simple-concat: ^1.0.0
checksum: e4132fd27cf7af230d853fa45c1b8ce900cb430dd0a3c6d3829649fe4f2b26574c803698076c4006450efb0fad2ba8c5455fbb5755d4b0a5ec42d4f12b31d27e
languageName: node
linkType: hard
"simple-git@npm:^3.15.0":
version: 3.19.0
resolution: "simple-git@npm:3.19.0"
@@ -25841,7 +25741,7 @@ __metadata:
languageName: node
linkType: hard
"tar-fs@npm:^2.0.0, tar-fs@npm:^2.1.1":
"tar-fs@npm:^2.1.1":
version: 2.1.1
resolution: "tar-fs@npm:2.1.1"
dependencies:
@@ -26354,15 +26254,6 @@ __metadata:
languageName: node
linkType: hard
"tunnel-agent@npm:^0.6.0":
version: 0.6.0
resolution: "tunnel-agent@npm:0.6.0"
dependencies:
safe-buffer: ^5.0.1
checksum: 05f6510358f8afc62a057b8b692f05d70c1782b70db86d6a1e0d5e28a32389e52fa6e7707b6c5ecccacc031462e4bc35af85ecfe4bbc341767917b7cf6965711
languageName: node
linkType: hard
"turndown@npm:^7.1.1":
version: 7.1.2
resolution: "turndown@npm:7.1.2"