Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-08-22 20:41:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(server): indexer & worker & sync perf (#15504)

Browse Source
This commit is contained in:
DarkSky
2026-08-21 08:11:37 +08:00
committed by GitHub
parent 8c9aad9a9b
commit c57004ea2c
259 changed files with 16530 additions and 17793 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/frontend/admin/src/config.json
+6 -9
View File
@@ -95,6 +95,10 @@
"type": "Boolean",
"desc": "Whether request abuse source facts should trust Cloudflare headers from the origin edge."
},
"signInRateLimit": {
"type": "Object",
"desc": "Limits for sign-in attempts shared through Redis by source IP and email. ttl is measured in milliseconds."
},
"inviteQuotaShadowMode": {
"type": "Boolean",
"desc": "Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions."
@@ -298,13 +302,6 @@
"desc": "Whether allow guest users to create demo workspaces."
}
},
"docService": {
"endpoint": {
"type": "String",
"desc": "The endpoint of the doc service.",
"env": "DOC_SERVICE_ENDPOINT"
}
},
"telemetry": {
"allowedOrigin": {
"type": "Array",
@@ -406,12 +403,12 @@
},
"provider.type": {
"type": "String",
"desc": "Indexer search service provider name",
"desc": "Indexer search provider. Self-hosted uses the embedded provider by default; remote providers require an endpoint.",
"env": "AFFINE_INDEXER_SEARCH_PROVIDER"
},
"provider.endpoint": {
"type": "String",
"desc": "Indexer search service endpoint",
"desc": "Remote indexer endpoint. Not used by the embedded provider.",
"env": "AFFINE_INDEXER_SEARCH_ENDPOINT"
},
"provider.apiKey": {
packages/frontend/admin/src/modules/settings/config.ts
+17
View File
@@ -163,6 +163,23 @@ export const KNOWN_CONFIG_GROUPS = [
},
],
} as ConfigGroup<'copilot'>,
{
name: 'Indexer',
module: 'indexer',
fields: [
{
key: 'provider.type',
type: 'Enum',
options: ['embedded', 'manticoresearch', 'elasticsearch'],
desc: 'Search provider. Embedded keeps external credentials for later reuse.',
},
'provider.endpoint',
'provider.apiKey',
'provider.username',
'provider.password',
'autoIndex.batchSize',
],
} as ConfigGroup<'indexer'>,
];
export const UNKNOWN_CONFIG_GROUPS = ALL_CONFIGURABLE_MODULES.filter(
packages/frontend/admin/src/modules/settings/index.spec.tsx
+90 -1
View File
@@ -24,13 +24,26 @@ vi.mock('../header', () => ({
vi.mock('./config-input-row', () => ({
ConfigRow: ({
field,
defaultValue,
onChange,
onErrorChange,
}: {
field: string;
defaultValue?: unknown;
onChange?: (field: string, value: unknown) => void;
onErrorChange?: (field: string, error?: string) => void;
}) => (
<div data-testid={`field-${field}`}>
<div>{field}</div>
<div>{`${field}:${defaultValue}`}</div>
<button type="button" onClick={() => onChange?.(field, 'embedded')}>
set-embedded-{field}
</button>
<button
type="button"
onClick={() => onChange?.(field, 'manticoresearch')}
>
set-manticoresearch-{field}
</button>
<button
type="button"
onClick={() => {
@@ -65,6 +78,16 @@ vi.mock('./config', () => ({
type: 'Boolean',
},
},
indexer: {
'provider.type': {
desc: 'Provider',
type: 'String',
},
'provider.endpoint': {
desc: 'Endpoint',
type: 'String',
},
},
},
ALL_SETTING_GROUPS: [
{
@@ -77,6 +100,18 @@ vi.mock('./config', () => ({
module: 'auth',
fields: ['allowSignup'],
},
{
name: 'Indexer',
module: 'indexer',
fields: [
{
key: 'provider.type',
type: 'Enum',
options: ['embedded', 'manticoresearch', 'elasticsearch'],
},
'provider.endpoint',
],
},
],
}));
@@ -93,6 +128,13 @@ describe('SettingsPage', () => {
auth: {
allowSignup: true,
},
indexer: {
enabled: false,
provider: {
type: 'elasticsearch',
endpoint: 'http://search.example',
},
},
},
patchedAppConfig: {
server: {
@@ -101,6 +143,13 @@ describe('SettingsPage', () => {
auth: {
allowSignup: true,
},
indexer: {
enabled: false,
provider: {
type: 'elasticsearch',
endpoint: 'http://search.example',
},
},
},
update: vi.fn(),
saveGroup: vi.fn().mockResolvedValue(undefined),
@@ -148,6 +197,46 @@ describe('SettingsPage', () => {
expect(authItem?.dataset.state).toBe('open');
});
test('encodes embedded without replacing external provider settings', () => {
const update = vi.fn();
useAppConfigMock.mockReturnValue({
...useAppConfigMock(),
update,
});
render(
<MemoryRouter initialEntries={['/admin/settings']}>
<Routes>
<Route path="/admin/settings" element={<SettingsPage />} />
</Routes>
</MemoryRouter>
);
fireEvent.click(screen.getAllByRole('button', { name: /Indexer/i })[0]);
expect(screen.getByText('indexer/provider.type:embedded')).toBeTruthy();
expect(screen.queryByTestId('field-indexer/provider.endpoint')).toBeNull();
fireEvent.click(
screen.getByRole('button', {
name: 'set-embedded-indexer/provider.type',
})
);
expect(update).toHaveBeenCalledWith('indexer/enabled', false);
expect(update).not.toHaveBeenCalledWith(
'indexer/provider.type',
'embedded'
);
fireEvent.click(
screen.getByRole('button', {
name: 'set-manticoresearch-indexer/provider.type',
})
);
expect(update).toHaveBeenCalledWith('indexer/enabled', true);
expect(update).toHaveBeenCalledWith(
'indexer/provider.type',
'manticoresearch'
);
});
test('disables save when group has validation errors even if group is dirty', () => {
useAppConfigMock.mockReset();
useAppConfigMock.mockReturnValue({
packages/frontend/admin/src/modules/settings/index.tsx
+33 -5
View File
@@ -173,6 +173,19 @@ const AdminPanel = ({
key={`${module}-${version}`}
>
{fields.map(field => {
const fieldKey =
typeof field === 'string' ? field : String(field.key);
const effectiveIndexerProvider = sourceConfig?.enabled
? sourceConfig?.provider?.type
: 'embedded';
if (
module === 'indexer' &&
effectiveIndexerProvider === 'embedded' &&
fieldKey.startsWith('provider.') &&
fieldKey !== 'provider.type'
) {
return null;
}
let props: ConfigInputProps;
if (typeof field === 'string') {
const descriptor =
@@ -194,11 +207,26 @@ const AdminPanel = ({
type: field.type ?? descriptor.type,
// @ts-expect-error for enum type
options: field.options,
defaultValue: get(
sourceConfig,
field.key + (field.sub ? '.' + field.sub : '')
),
onChange: onUpdate,
defaultValue:
module === 'indexer' &&
field.key === 'provider.type'
? effectiveIndexerProvider
: get(
sourceConfig,
field.key + (field.sub ? '.' + field.sub : '')
),
onChange:
module === 'indexer' &&
field.key === 'provider.type'
? (_path, value) => {
if (value === 'embedded') {
onUpdate('indexer/enabled', false);
} else {
onUpdate('indexer/enabled', true);
onUpdate('indexer/provider.type', value);
}
}
: onUpdate,
};
}
packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/plugin/NbStorePlugin.kt
+151 -60
View File
@@ -7,9 +7,10 @@ import com.getcapacitor.PluginCall
import com.getcapacitor.PluginMethod
import com.getcapacitor.annotation.CapacitorPlugin
import kotlinx.coroutines.Dispatchers
import org.json.JSONObject
import timber.log.Timber
import uniffi.affine_mobile_native.DocRecord
import uniffi.affine_mobile_native.DocIndexedClock
import uniffi.affine_mobile_native.IndexHit
import uniffi.affine_mobile_native.SetBlob
import uniffi.affine_mobile_native.newDocStoragePool
@@ -623,119 +624,209 @@ class NbStorePlugin : Plugin() {
}
@PluginMethod
fun ftsAddDocument(call: PluginCall) {
fun getDocIndexedClock(call: PluginCall) {
launch(Dispatchers.IO) {
try {
val id = call.getStringEnsure("id")
val indexName = call.getStringEnsure("indexName")
val docId = call.getStringEnsure("docId")
val text = call.getStringEnsure("text")
val index = call.getBoolean("index")
?: throw IllegalArgumentException("index is required")
docStoragePool.ftsAddDocument(id, indexName, docId, text, index)
call.resolve()
val clock = docStoragePool.getDocIndexedClock(
call.getStringEnsure("id"),
call.getStringEnsure("docId")
)
call.resolve(clock?.let(::indexedClockJson))
} catch (e: Exception) {
call.reject("Failed to add document to fts: ${e.message}", null, e)
call.reject("Failed to get indexed clock: ${e.message}", null, e)
}
}
}
@PluginMethod
fun ftsDeleteDocument(call: PluginCall) {
fun setDocIndexedClock(call: PluginCall) {
launch(Dispatchers.IO) {
try {
val id = call.getStringEnsure("id")
val indexName = call.getStringEnsure("indexName")
val docId = call.getStringEnsure("docId")
docStoragePool.ftsDeleteDocument(id, indexName, docId)
val clock = DocIndexedClock(
call.getStringEnsure("docId"),
call.getLong("indexedClock") ?: throw IllegalArgumentException("indexedClock is required"),
call.getLong("indexerVersion") ?: throw IllegalArgumentException("indexerVersion is required")
)
docStoragePool.setDocIndexedClock(call.getStringEnsure("id"), clock)
call.resolve()
} catch (e: Exception) {
call.reject("Failed to delete document from fts: ${e.message}", null, e)
call.reject("Failed to set indexed clock: ${e.message}", null, e)
}
}
}
@PluginMethod
fun ftsSearch(call: PluginCall) {
fun setDocIndexedClocks(call: PluginCall) {
launch(Dispatchers.IO) {
try {
val id = call.getStringEnsure("id")
val indexName = call.getStringEnsure("indexName")
val query = call.getStringEnsure("query")
val results = docStoragePool.ftsSearch(id, indexName, query)
val mapped = results.map {
JSObject()
.put("id", it.id)
.put("score", it.score)
.put("terms", JSArray(it.terms))
val values = call.getArray("clocks") ?: throw IllegalArgumentException("clocks is required")
val clocks = (0 until values.length()).map { index ->
val value = values.getJSONObject(index)
DocIndexedClock(
value.getString("docId"),
value.getLong("timestamp"),
value.getLong("indexerVersion")
)
}
docStoragePool.setDocIndexedClocks(call.getStringEnsure("id"), clocks)
call.resolve()
} catch (e: Exception) {
call.reject("Failed to commit indexed clocks: ${e.message}", null, e)
}
}
}
@PluginMethod
fun clearDocIndexedClock(call: PluginCall) {
launch(Dispatchers.IO) {
try {
docStoragePool.clearDocIndexedClock(call.getStringEnsure("id"), call.getStringEnsure("docId"))
call.resolve()
} catch (e: Exception) {
call.reject("Failed to clear indexed clock: ${e.message}", null, e)
}
}
}
@PluginMethod
fun indexUpsert(call: PluginCall) {
launch(Dispatchers.IO) {
try {
val id = call.getStringEnsure("id")
val table = call.getStringEnsure("table")
val document = call.getObject("document")
?: throw IllegalArgumentException("document is required")
docStoragePool.indexUpsert(id, table, document.toString())
call.resolve()
} catch (e: Exception) {
call.reject("Failed to upsert index document: ${e.message}", null, e)
}
}
}
@PluginMethod
fun indexDelete(call: PluginCall) {
launch(Dispatchers.IO) {
try {
val id = call.getStringEnsure("id")
val table = call.getStringEnsure("table")
val docId = call.getStringEnsure("docId")
docStoragePool.indexDelete(id, table, docId)
call.resolve()
} catch (e: Exception) {
call.reject("Failed to delete index document: ${e.message}", null, e)
}
}
}
@PluginMethod
fun indexSearch(call: PluginCall) {
launch(Dispatchers.IO) {
try {
val id = call.getStringEnsure("id")
val table = call.getStringEnsure("table")
val query = call.getObject("query") ?: throw IllegalArgumentException("query is required")
val options = call.getObject("options") ?: throw IllegalArgumentException("options is required")
val result = docStoragePool.indexSearch(id, table, query.toString(), options.toString())
call.resolve(
JSObject().put("results", JSArray(mapped))
JSObject()
.put("total", result.total.toInt())
.put("hits", JSArray(result.hits.map(::indexHitJson)))
)
} catch (e: Exception) {
call.reject("Failed to search fts: ${e.message}", null, e)
call.reject("Failed to search index: ${e.message}", null, e)
}
}
}
@PluginMethod
fun ftsGetDocument(call: PluginCall) {
fun indexAggregate(call: PluginCall) {
launch(Dispatchers.IO) {
try {
val id = call.getStringEnsure("id")
val indexName = call.getStringEnsure("indexName")
val docId = call.getStringEnsure("docId")
val text = docStoragePool.ftsGetDocument(id, indexName, docId)
call.resolve(JSObject().put("text", text ?: JSONObject.NULL))
} catch (e: Exception) {
call.reject("Failed to get fts document: ${e.message}", null, e)
}
}
}
@PluginMethod
fun ftsGetMatches(call: PluginCall) {
launch(Dispatchers.IO) {
try {
val id = call.getStringEnsure("id")
val indexName = call.getStringEnsure("indexName")
val docId = call.getStringEnsure("docId")
val query = call.getStringEnsure("query")
val matches = docStoragePool.ftsGetMatches(id, indexName, docId, query)
val mapped = matches.map {
val table = call.getStringEnsure("table")
val query = call.getObject("query") ?: throw IllegalArgumentException("query is required")
val hits = call.getObject("hits")?.toString()
val result = docStoragePool.indexAggregate(
id,
table,
query.toString(),
call.getStringEnsure("field"),
call.getIntEnsure("limit").toUInt(),
call.getIntEnsure("offset").toUInt(),
hits
)
val buckets = result.buckets.map {
JSObject()
.put("start", it.start.toInt())
.put("end", it.end.toInt())
.put("key", it.key)
.put("count", it.count.toInt())
.put("score", it.score)
.put("hits", JSArray(it.hits.map(::indexHitJson)))
}
call.resolve(JSObject().put("matches", JSArray(mapped)))
call.resolve(JSObject().put("total", result.total.toInt()).put("buckets", JSArray(buckets)))
} catch (e: Exception) {
call.reject("Failed to get fts matches: ${e.message}", null, e)
call.reject("Failed to aggregate index: ${e.message}", null, e)
}
}
}
@PluginMethod
fun ftsFlushIndex(call: PluginCall) {
fun indexDeleteByQuery(call: PluginCall) {
launch(Dispatchers.IO) {
try {
val id = call.getStringEnsure("id")
docStoragePool.ftsFlushIndex(id)
val table = call.getStringEnsure("table")
val query = call.getObject("query") ?: throw IllegalArgumentException("query is required")
val deleted = docStoragePool.indexDeleteByQuery(id, table, query.toString())
call.resolve(JSObject().put("deleted", deleted.toInt()))
} catch (e: Exception) {
call.reject("Failed to delete index documents: ${e.message}", null, e)
}
}
}
@PluginMethod
fun indexFlush(call: PluginCall) {
launch(Dispatchers.IO) {
try {
docStoragePool.indexFlush(call.getStringEnsure("id"))
call.resolve()
} catch (e: Exception) {
call.reject("Failed to flush fts index: ${e.message}", null, e)
call.reject("Failed to flush index: ${e.message}", null, e)
}
}
}
@PluginMethod
fun ftsIndexVersion(call: PluginCall) {
fun indexVersion(call: PluginCall) {
launch(Dispatchers.IO) {
try {
val version = docStoragePool.ftsIndexVersion() + ANDROID_INDEXER_VERSION_OFFSET
val version = docStoragePool.indexVersion() + ANDROID_INDEXER_VERSION_OFFSET
call.resolve(JSObject().put("indexVersion", version))
} catch (e: Exception) {
call.reject("Failed to get fts index version: ${e.message}", null, e)
call.reject("Failed to get index version: ${e.message}", null, e)
}
}
}
}
private fun indexHitJson(hit: IndexHit): JSObject {
val fields = hit.fields.map { JSObject().put("field", it.field).put("values", JSArray(it.values)) }
val highlights = hit.highlights.map { highlight ->
val values = highlight.values.map { value ->
val spans = value.spans.map { JSObject().put("start", it.start.toInt()).put("end", it.end.toInt()) }
JSObject().put("valueIndex", value.valueIndex.toInt()).put("spans", JSArray(spans))
}
JSObject().put("field", highlight.field).put("values", JSArray(values))
}
return JSObject()
.put("id", hit.id)
.put("score", hit.score)
.put("fields", JSArray(fields))
.put("highlights", JSArray(highlights))
}
private fun indexedClockJson(clock: DocIndexedClock): JSObject = JSObject()
.put("docId", clock.docId)
.put("timestamp", clock.timestamp)
.put("indexerVersion", clock.indexerVersion)
packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/plugin/PluginExt.kt
+5 -1
View File
@@ -25,4 +25,8 @@ inline fun <reified T> PluginCall.getListEnsure(key: String): List<T> {
fun PluginCall.getLongEnsure(key: String): Long {
return getLong(key) ?: throw IllegalArgumentException("Missing $key parameter")
}
}
fun PluginCall.getIntEnsure(key: String): Int {
return getInt(key) ?: throw IllegalArgumentException("Missing $key parameter")
}
packages/frontend/apps/android/App/app/src/main/java/uniffi/affine_mobile_native/affine_mobile_native.kt
+880 -337
View File
File diff suppressed because it is too large Load Diff
packages/frontend/apps/android/src/plugins/nbstore/definitions.ts
+46 -24
View File
@@ -1,4 +1,22 @@
import type { CrawlResult, DocIndexedClock } from '@affine/nbstore';
import type {
NativeIndexField,
NativeIndexHit,
NativeIndexQuery,
NativeIndexSearchOptions,
NativeIndexSearchResult,
} from '@affine/nbstore/sqlite';
type NativeIndexDocument = { id: string; fields: NativeIndexField[] };
type NativeIndexAggregateResult = {
total: number;
buckets: {
key: string;
count: number;
score: number;
hits: NativeIndexHit[];
}[];
};
export interface Blob {
key: string;
@@ -157,38 +175,38 @@ export interface NbStorePlugin {
id: string;
docId: string;
}) => Promise<CrawlResult>;
ftsAddDocument: (options: {
indexUpsert: (options: {
id: string;
indexName: string;
docId: string;
text: string;
index: boolean;
table: string;
document: NativeIndexDocument;
}) => Promise<void>;
ftsDeleteDocument: (options: {
indexDelete: (options: {
id: string;
indexName: string;
table: string;
docId: string;
}) => Promise<void>;
ftsSearch: (options: {
indexSearch: (options: {
id: string;
indexName: string;
query: string;
}) => Promise<{
results: { id: string; score: number; terms: Array<string> }[];
}>;
ftsGetDocument: (options: {
table: string;
query: NativeIndexQuery;
options: NativeIndexSearchOptions;
}) => Promise<NativeIndexSearchResult>;
indexAggregate: (options: {
id: string;
indexName: string;
docId: string;
}) => Promise<{ text?: string | null }>;
ftsGetMatches: (options: {
table: string;
query: NativeIndexQuery;
field: string;
limit: number;
offset: number;
hits?: NativeIndexSearchOptions;
}) => Promise<NativeIndexAggregateResult>;
indexDeleteByQuery: (options: {
id: string;
indexName: string;
docId: string;
query: string;
}) => Promise<{ matches: { start: number; end: number }[] }>;
ftsFlushIndex: (options: { id: string }) => Promise<void>;
ftsIndexVersion: () => Promise<{ indexVersion: number }>;
table: string;
query: NativeIndexQuery;
}) => Promise<{ deleted: number }>;
indexFlush: (options: { id: string }) => Promise<void>;
indexVersion: () => Promise<{ indexVersion: number }>;
getDocIndexedClock: (options: {
id: string;
docId: string;
@@ -199,6 +217,10 @@ export interface NbStorePlugin {
indexedClock: number;
indexerVersion: number;
}) => Promise<void>;
setDocIndexedClocks: (options: {
id: string;
clocks: Array<{ docId: string; timestamp: number; indexerVersion: number }>;
}) => Promise<void>;
clearDocIndexedClock: (options: {
id: string;
docId: string;
packages/frontend/apps/android/src/plugins/nbstore/index.ts
+58 -53
View File
@@ -2,7 +2,6 @@ import {
base64ToUint8Array,
uint8ArrayToBase64,
} from '@affine/core/modules/workspace-engine';
import { normalizeNativeOptional } from '@affine/mobile-shared/nbstore/optional';
import {
decodePayload,
MOBILE_BLOB_FILE_PREFIX,
@@ -358,77 +357,74 @@ export const NbStoreNativeDBApis: NativeDBApis = {
): Promise<CrawlResult> {
return await NbStore.crawlDocData({ id, docId });
},
ftsAddDocument: async function (
indexUpsert: async function (
id: string,
indexName: string,
docId: string,
text: string,
index: boolean
table: string,
document
): Promise<void> {
await NbStore.ftsAddDocument({
await NbStore.indexUpsert({
id,
indexName,
docId,
text,
index,
table,
document,
});
},
ftsDeleteDocument: async function (
indexDelete: async function (
id: string,
indexName: string,
table: string,
docId: string
): Promise<void> {
await NbStore.ftsDeleteDocument({
await NbStore.indexDelete({
id,
indexName,
table,
docId,
});
},
ftsSearch: async function (
id: string,
indexName: string,
query: string
): Promise<{ id: string; score: number; terms: Array<string> }[]> {
const { results } = await NbStore.ftsSearch({
indexSearch: async function (id: string, table: string, query, options) {
return await NbStore.indexSearch({
id,
indexName,
table,
query,
options,
});
},
indexAggregate: async function (
id: string,
table: string,
query,
field: string,
limit: number,
offset: number,
hits
) {
return await NbStore.indexAggregate({
id,
table,
query,
field,
limit,
offset,
hits,
});
},
indexDeleteByQuery: async function (
id: string,
table: string,
query
): Promise<number> {
const { deleted } = await NbStore.indexDeleteByQuery({
id,
table,
query,
});
return results ?? [];
return deleted;
},
ftsGetDocument: async function (
id: string,
indexName: string,
docId: string
): Promise<string | null> {
const result = await NbStore.ftsGetDocument({
id,
indexName,
docId,
});
return normalizeNativeOptional(result.text);
},
ftsGetMatches: async function (
id: string,
indexName: string,
docId: string,
query: string
): Promise<{ start: number; end: number }[]> {
const { matches } = await NbStore.ftsGetMatches({
id,
indexName,
docId,
query,
});
return matches ?? [];
},
ftsFlushIndex: async function (id: string): Promise<void> {
await NbStore.ftsFlushIndex({
indexFlush: async function (id: string): Promise<void> {
await NbStore.indexFlush({
id,
});
},
ftsIndexVersion: function (): Promise<number> {
return NbStore.ftsIndexVersion().then(res => res.indexVersion);
indexVersion: function (): Promise<number> {
return NbStore.indexVersion().then(res => res.indexVersion);
},
getDocIndexedClock: function (
id: string,
@@ -451,6 +447,15 @@ export const NbStoreNativeDBApis: NativeDBApis = {
indexerVersion,
});
},
setDocIndexedClocks: function (id, clocks): Promise<void> {
return NbStore.setDocIndexedClocks({
id,
clocks: clocks.map(clock => ({
...clock,
timestamp: clock.timestamp.getTime(),
})),
});
},
clearDocIndexedClock: function (id: string, docId: string): Promise<void> {
return NbStore.clearDocIndexedClock({
id,
packages/frontend/apps/electron/src/helper/nbstore/handlers.ts
+8 -7
View File
@@ -51,11 +51,12 @@ export const nbstoreHandlers: NativeDBApis = {
setBlobUploadedAt: POOL.setBlobUploadedAt.bind(POOL),
getBlobUploadedAt: POOL.getBlobUploadedAt.bind(POOL),
crawlDocData: POOL.crawlDocData.bind(POOL),
ftsAddDocument: POOL.ftsAddDocument.bind(POOL),
ftsDeleteDocument: POOL.ftsDeleteDocument.bind(POOL),
ftsSearch: POOL.ftsSearch.bind(POOL),
ftsGetDocument: POOL.ftsGetDocument.bind(POOL),
ftsGetMatches: POOL.ftsGetMatches.bind(POOL),
ftsFlushIndex: POOL.ftsFlushIndex.bind(POOL),
ftsIndexVersion: POOL.ftsIndexVersion.bind(POOL),
setDocIndexedClocks: POOL.setDocIndexedClocks.bind(POOL),
indexUpsert: POOL.indexUpsert.bind(POOL),
indexDelete: POOL.indexDelete.bind(POOL),
indexSearch: POOL.indexSearch.bind(POOL),
indexAggregate: POOL.indexAggregate.bind(POOL),
indexDeleteByQuery: POOL.indexDeleteByQuery.bind(POOL),
indexFlush: POOL.indexFlush.bind(POOL),
indexVersion: POOL.indexVersion.bind(POOL),
};
packages/frontend/apps/ios/App/App/JSValueContainerExt.swift
+14
View File
@@ -32,6 +32,20 @@ public extension JSValueContainer {
return doub
}
func getInt64Ensure(_ key: String) throws -> Int64 {
guard let value = getDouble(key), let integer = Int64(exactly: value) else {
throw RequestParamError.request(key: key)
}
return integer
}
func getUInt32Ensure(_ key: String) throws -> UInt32 {
guard let value = getDouble(key), let integer = UInt32(exactly: value) else {
throw RequestParamError.request(key: key)
}
return integer
}
func getBoolEnsure(_ key: String) throws -> Bool {
guard let bool = getBool(key) else {
throw RequestParamError.request(key: key)
packages/frontend/apps/ios/App/App/Plugins/NBStore/NBStorePlugin.swift
+174 -83
View File
@@ -37,13 +37,17 @@ public class NbStorePlugin: CAPPlugin, CAPBridgedPlugin {
CAPPluginMethod(name: "getBlobUploadedAt", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "setBlobUploadedAt", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "crawlDocData", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "ftsAddDocument", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "ftsDeleteDocument", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "ftsSearch", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "ftsGetDocument", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "ftsGetMatches", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "ftsFlushIndex", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "ftsIndexVersion", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "getDocIndexedClock", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "setDocIndexedClock", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "setDocIndexedClocks", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "clearDocIndexedClock", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "indexUpsert", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "indexDelete", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "indexSearch", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "indexAggregate", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "indexDeleteByQuery", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "indexFlush", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "indexVersion", returnType: CAPPluginReturnPromise),
]
@objc func connect(_ call: CAPPluginCall) {
@@ -585,138 +589,225 @@ public class NbStorePlugin: CAPPlugin, CAPBridgedPlugin {
}
}
@objc func ftsAddDocument(_ call: CAPPluginCall) {
@objc func getDocIndexedClock(_ call: CAPPluginCall) {
Task {
do {
let id = try call.getStringEnsure("id")
let indexName = try call.getStringEnsure("indexName")
let docId = try call.getStringEnsure("docId")
let text = try call.getStringEnsure("text")
guard let index = call.getBool("index") else {
call.reject("index is required", nil, nil)
let clock = try await docStoragePool.getDocIndexedClock(
universalId: try call.getStringEnsure("id"),
docId: try call.getStringEnsure("docId")
)
guard let clock else {
call.resolve()
return
}
try await docStoragePool.ftsAddDocument(
universalId: id,
indexName: indexName,
docId: docId,
text: text,
index: index
)
call.resolve()
call.resolve(indexedClockJson(clock))
} catch {
call.reject("Failed to add document to fts, \(error)", nil, error)
call.reject("Failed to get indexed clock, \(error)", nil, error)
}
}
}
@objc func ftsDeleteDocument(_ call: CAPPluginCall) {
@objc func setDocIndexedClock(_ call: CAPPluginCall) {
Task {
do {
let clock = DocIndexedClock(
docId: try call.getStringEnsure("docId"),
timestamp: try call.getInt64Ensure("indexedClock"),
indexerVersion: try call.getInt64Ensure("indexerVersion")
)
try await docStoragePool.setDocIndexedClock(universalId: try call.getStringEnsure("id"), clock: clock)
call.resolve()
} catch {
call.reject("Failed to set indexed clock, \(error)", nil, error)
}
}
}
@objc func setDocIndexedClocks(_ call: CAPPluginCall) {
Task {
do {
let clocks = try call.getArrayEnsure("clocks", JSObject.self).map { value in
guard
let docId = value["docId"] as? String,
let timestamp = value["timestamp"] as? Double,
let timestamp = Int64(exactly: timestamp),
let indexerVersion = value["indexerVersion"] as? Double,
let indexerVersion = Int64(exactly: indexerVersion)
else {
throw RequestParamError.request(key: "clocks")
}
return DocIndexedClock(
docId: docId,
timestamp: timestamp,
indexerVersion: indexerVersion
)
}
try await docStoragePool.setDocIndexedClocks(universalId: try call.getStringEnsure("id"), clocks: clocks)
call.resolve()
} catch {
call.reject("Failed to commit indexed clocks, \(error)", nil, error)
}
}
}
@objc func clearDocIndexedClock(_ call: CAPPluginCall) {
Task {
do {
try await docStoragePool.clearDocIndexedClock(
universalId: try call.getStringEnsure("id"),
docId: try call.getStringEnsure("docId")
)
call.resolve()
} catch {
call.reject("Failed to clear indexed clock, \(error)", nil, error)
}
}
}
@objc func indexUpsert(_ call: CAPPluginCall) {
Task {
do {
let id = try call.getStringEnsure("id")
let indexName = try call.getStringEnsure("indexName")
let docId = try call.getStringEnsure("docId")
try await docStoragePool.ftsDeleteDocument(
let table = try call.getStringEnsure("table")
let document = try jsonString(call, "document")
try await docStoragePool.indexUpsert(
universalId: id,
indexName: indexName,
table: table,
document: document
)
call.resolve()
} catch {
call.reject("Failed to upsert index document, \(error)", nil, error)
}
}
}
@objc func indexDelete(_ call: CAPPluginCall) {
Task {
do {
let id = try call.getStringEnsure("id")
let table = try call.getStringEnsure("table")
let docId = try call.getStringEnsure("docId")
try await docStoragePool.indexDelete(
universalId: id,
table: table,
docId: docId
)
call.resolve()
} catch {
call.reject("Failed to delete document from fts, \(error)", nil, error)
call.reject("Failed to delete index document, \(error)", nil, error)
}
}
}
@objc func ftsSearch(_ call: CAPPluginCall) {
@objc func indexSearch(_ call: CAPPluginCall) {
Task {
do {
let id = try call.getStringEnsure("id")
let indexName = try call.getStringEnsure("indexName")
let query = try call.getStringEnsure("query")
let results = try await docStoragePool.ftsSearch(
let table = try call.getStringEnsure("table")
let result = try await docStoragePool.indexSearch(
universalId: id,
indexName: indexName,
query: query
table: table,
query: try jsonString(call, "query"),
options: try jsonString(call, "options")
)
let mapped = results.map {
[
"id": $0.id,
"score": $0.score,
"terms": $0.terms,
] as [String: Any]
call.resolve(["total": result.total, "hits": result.hits.map(indexHitJson)])
} catch {
call.reject("Failed to search index, \(error)", nil, error)
}
}
}
@objc func indexAggregate(_ call: CAPPluginCall) {
Task {
do {
let id = try call.getStringEnsure("id")
let table = try call.getStringEnsure("table")
let result = try await docStoragePool.indexAggregate(
universalId: id,
table: table,
query: try jsonString(call, "query"),
field: try call.getStringEnsure("field"),
limit: try call.getUInt32Ensure("limit"),
offset: try call.getUInt32Ensure("offset"),
hits: try optionalJsonString(call, "hits")
)
let buckets = result.buckets.map { bucket in
["key": bucket.key, "count": bucket.count, "score": bucket.score, "hits": bucket.hits.map(indexHitJson)]
}
call.resolve(["results": mapped])
call.resolve(["total": result.total, "buckets": buckets])
} catch {
call.reject("Failed to search fts, \(error)", nil, error)
call.reject("Failed to aggregate index, \(error)", nil, error)
}
}
}
@objc func ftsGetDocument(_ call: CAPPluginCall) {
@objc func indexDeleteByQuery(_ call: CAPPluginCall) {
Task {
do {
let id = try call.getStringEnsure("id")
let indexName = try call.getStringEnsure("indexName")
let docId = try call.getStringEnsure("docId")
let text = try await docStoragePool.ftsGetDocument(
let deleted = try await docStoragePool.indexDeleteByQuery(
universalId: id,
indexName: indexName,
docId: docId
table: try call.getStringEnsure("table"),
query: try jsonString(call, "query")
)
call.resolve(["text": text ?? NSNull()])
call.resolve(["deleted": deleted])
} catch {
call.reject("Failed to get fts document, \(error)", nil, error)
call.reject("Failed to delete index documents, \(error)", nil, error)
}
}
}
@objc func ftsGetMatches(_ call: CAPPluginCall) {
@objc func indexFlush(_ call: CAPPluginCall) {
Task {
do {
let id = try call.getStringEnsure("id")
let indexName = try call.getStringEnsure("indexName")
let docId = try call.getStringEnsure("docId")
let query = try call.getStringEnsure("query")
let matches = try await docStoragePool.ftsGetMatches(
universalId: id,
indexName: indexName,
docId: docId,
query: query
)
let mapped = matches.map {
[
"start": $0.start,
"end": $0.end,
]
}
call.resolve(["matches": mapped])
} catch {
call.reject("Failed to get fts matches, \(error)", nil, error)
}
}
}
@objc func ftsFlushIndex(_ call: CAPPluginCall) {
Task {
do {
let id = try call.getStringEnsure("id")
try await docStoragePool.ftsFlushIndex(universalId: id)
try await docStoragePool.indexFlush(universalId: id)
call.resolve()
} catch {
call.reject("Failed to flush fts index, \(error)", nil, error)
call.reject("Failed to flush index, \(error)", nil, error)
}
}
}
@objc func ftsIndexVersion(_ call: CAPPluginCall) {
@objc func indexVersion(_ call: CAPPluginCall) {
Task {
do {
let version = try await docStoragePool.ftsIndexVersion()
let version = try await docStoragePool.indexVersion()
call.resolve(["indexVersion": version])
} catch {
call.reject("Failed to get fts index version, \(error)", nil, error)
call.reject("Failed to get index version, \(error)", nil, error)
}
}
}
}
private func jsonString(_ call: CAPPluginCall, _ key: String) throws -> String {
guard let value = call.getObject(key) else { throw RequestParamError.request(key: key) }
return String(data: try JSONSerialization.data(withJSONObject: value), encoding: .utf8)!
}
private func optionalJsonString(_ call: CAPPluginCall, _ key: String) throws -> String? {
guard call.getObject(key) != nil else { return nil }
return try jsonString(call, key)
}
private func indexHitJson(_ hit: IndexHit) -> [String: Any] {
[
"id": hit.id,
"score": hit.score,
"fields": hit.fields.map { ["field": $0.field, "values": $0.values] },
"highlights": hit.highlights.map { highlight in
[
"field": highlight.field,
"values": highlight.values.map { value in
["valueIndex": value.valueIndex, "spans": value.spans.map { ["start": $0.start, "end": $0.end] }]
},
]
},
]
}
private func indexedClockJson(_ clock: DocIndexedClock) -> [String: Any] {
["docId": clock.docId, "timestamp": clock.timestamp, "indexerVersion": clock.indexerVersion]
}
packages/frontend/apps/ios/App/App/uniffi/affine_mobile_native.swift
+1111 -352
View File
File diff suppressed because it is too large Load Diff
packages/frontend/apps/ios/App/App/uniffi/affine_mobile_nativeFFI.h
+121 -77
View File
@@ -266,6 +266,11 @@ void uniffi_affine_mobile_native_fn_free_docstoragepool(void*_Nonnull ptr, RustC
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_clear_clocks(void*_Nonnull ptr, RustBuffer universal_id
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_CLEAR_DOC_INDEXED_CLOCK
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_CLEAR_DOC_INDEXED_CLOCK
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_clear_doc_indexed_clock(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer doc_id
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_CONNECT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_CONNECT
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_connect(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer path
@@ -291,41 +296,6 @@ uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_delete_doc(void*_N
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_disconnect(void*_Nonnull ptr, RustBuffer universal_id
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_ADD_DOCUMENT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_ADD_DOCUMENT
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_fts_add_document(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer index_name, RustBuffer doc_id, RustBuffer text, int8_t index
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_DELETE_DOCUMENT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_DELETE_DOCUMENT
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_fts_delete_document(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer index_name, RustBuffer doc_id
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_FLUSH_INDEX
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_FLUSH_INDEX
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_fts_flush_index(void*_Nonnull ptr, RustBuffer universal_id
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_GET_DOCUMENT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_GET_DOCUMENT
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_fts_get_document(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer index_name, RustBuffer doc_id
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_GET_MATCHES
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_GET_MATCHES
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_fts_get_matches(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer index_name, RustBuffer doc_id, RustBuffer query
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_INDEX_VERSION
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_INDEX_VERSION
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_fts_index_version(void*_Nonnull ptr
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_SEARCH
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_FTS_SEARCH
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_fts_search(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer index_name, RustBuffer query
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_GET_BLOB
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_GET_BLOB
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_blob(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer key
@@ -346,6 +316,11 @@ uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_clock(void
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_clocks(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer after
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_GET_DOC_INDEXED_CLOCK
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_GET_DOC_INDEXED_CLOCK
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_indexed_clock(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer doc_id
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_GET_DOC_SNAPSHOT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_GET_DOC_SNAPSHOT
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_snapshot(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer doc_id
@@ -386,6 +361,41 @@ uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_remote_cl
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_remote_clocks(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer peer
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_AGGREGATE
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_AGGREGATE
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_index_aggregate(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer table, RustBuffer query, RustBuffer field, uint32_t limit, uint32_t offset, RustBuffer hits
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_DELETE
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_DELETE
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_index_delete(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer table, RustBuffer doc_id
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_DELETE_BY_QUERY
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_DELETE_BY_QUERY
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_index_delete_by_query(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer table, RustBuffer query
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_FLUSH
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_FLUSH
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_index_flush(void*_Nonnull ptr, RustBuffer universal_id
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_SEARCH
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_SEARCH
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_index_search(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer table, RustBuffer query, RustBuffer options
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_UPSERT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_UPSERT
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_index_upsert(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer table, RustBuffer document
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_VERSION
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_INDEX_VERSION
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_index_version(void*_Nonnull ptr
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_LIST_BLOBS
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_LIST_BLOBS
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_list_blobs(void*_Nonnull ptr, RustBuffer universal_id
@@ -416,6 +426,16 @@ uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_set_blob(void*_Non
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_set_blob_uploaded_at(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer peer, RustBuffer blob_id, RustBuffer uploaded_at
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_SET_DOC_INDEXED_CLOCK
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_SET_DOC_INDEXED_CLOCK
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_set_doc_indexed_clock(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer clock
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_SET_DOC_INDEXED_CLOCKS
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_SET_DOC_INDEXED_CLOCKS
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_set_doc_indexed_clocks(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer clocks
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_SET_DOC_SNAPSHOT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_SET_DOC_SNAPSHOT
uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_set_doc_snapshot(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer snapshot
@@ -770,6 +790,12 @@ uint16_t uniffi_affine_mobile_native_checksum_func_render_typst_preview_svg(void
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_CLEAR_CLOCKS
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_clear_clocks(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_CLEAR_DOC_INDEXED_CLOCK
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_CLEAR_DOC_INDEXED_CLOCK
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_clear_doc_indexed_clock(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_CONNECT
@@ -800,48 +826,6 @@ uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_delete_doc(v
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_DISCONNECT
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_disconnect(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_ADD_DOCUMENT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_ADD_DOCUMENT
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_fts_add_document(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_DELETE_DOCUMENT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_DELETE_DOCUMENT
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_fts_delete_document(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_FLUSH_INDEX
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_FLUSH_INDEX
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_fts_flush_index(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_GET_DOCUMENT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_GET_DOCUMENT
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_fts_get_document(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_GET_MATCHES
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_GET_MATCHES
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_fts_get_matches(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_INDEX_VERSION
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_INDEX_VERSION
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_fts_index_version(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_SEARCH
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_FTS_SEARCH
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_fts_search(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_GET_BLOB
@@ -866,6 +850,12 @@ uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_cloc
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_GET_DOC_CLOCKS
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_clocks(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_GET_DOC_INDEXED_CLOCK
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_GET_DOC_INDEXED_CLOCK
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_indexed_clock(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_GET_DOC_SNAPSHOT
@@ -914,6 +904,48 @@ uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_rem
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_GET_PEER_REMOTE_CLOCKS
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_remote_clocks(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_AGGREGATE
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_AGGREGATE
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_index_aggregate(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_DELETE
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_DELETE
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_index_delete(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_DELETE_BY_QUERY
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_DELETE_BY_QUERY
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_index_delete_by_query(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_FLUSH
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_FLUSH
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_index_flush(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_SEARCH
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_SEARCH
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_index_search(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_UPSERT
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_UPSERT
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_index_upsert(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_VERSION
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_INDEX_VERSION
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_index_version(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_LIST_BLOBS
@@ -950,6 +982,18 @@ uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_set_blob(voi
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_SET_BLOB_UPLOADED_AT
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_set_blob_uploaded_at(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_SET_DOC_INDEXED_CLOCK
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_SET_DOC_INDEXED_CLOCK
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_set_doc_indexed_clock(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_SET_DOC_INDEXED_CLOCKS
#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_SET_DOC_INDEXED_CLOCKS
uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_set_doc_indexed_clocks(void
);
#endif
#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_SET_DOC_SNAPSHOT
packages/frontend/apps/ios/src/plugins/nbstore/definitions.ts
+46 -24
View File
@@ -1,4 +1,22 @@
import type { CrawlResult, DocIndexedClock } from '@affine/nbstore';
import type {
NativeIndexField,
NativeIndexHit,
NativeIndexQuery,
NativeIndexSearchOptions,
NativeIndexSearchResult,
} from '@affine/nbstore/sqlite';
type NativeIndexDocument = { id: string; fields: NativeIndexField[] };
type NativeIndexAggregateResult = {
total: number;
buckets: {
key: string;
count: number;
score: number;
hits: NativeIndexHit[];
}[];
};
export interface Blob {
key: string;
@@ -157,38 +175,38 @@ export interface NbStorePlugin {
id: string;
docId: string;
}) => Promise<CrawlResult>;
ftsAddDocument: (options: {
indexUpsert: (options: {
id: string;
indexName: string;
docId: string;
text: string;
index: boolean;
table: string;
document: NativeIndexDocument;
}) => Promise<void>;
ftsDeleteDocument: (options: {
indexDelete: (options: {
id: string;
indexName: string;
table: string;
docId: string;
}) => Promise<void>;
ftsSearch: (options: {
indexSearch: (options: {
id: string;
indexName: string;
query: string;
}) => Promise<{
results: { id: string; score: number; terms: Array<string> }[];
}>;
ftsGetDocument: (options: {
table: string;
query: NativeIndexQuery;
options: NativeIndexSearchOptions;
}) => Promise<NativeIndexSearchResult>;
indexAggregate: (options: {
id: string;
indexName: string;
docId: string;
}) => Promise<{ text?: string | null }>;
ftsGetMatches: (options: {
table: string;
query: NativeIndexQuery;
field: string;
limit: number;
offset: number;
hits?: NativeIndexSearchOptions;
}) => Promise<NativeIndexAggregateResult>;
indexDeleteByQuery: (options: {
id: string;
indexName: string;
docId: string;
query: string;
}) => Promise<{ matches: { start: number; end: number }[] }>;
ftsFlushIndex: (options: { id: string }) => Promise<void>;
ftsIndexVersion: () => Promise<{ indexVersion: number }>;
table: string;
query: NativeIndexQuery;
}) => Promise<{ deleted: number }>;
indexFlush: (options: { id: string }) => Promise<void>;
indexVersion: () => Promise<{ indexVersion: number }>;
getDocIndexedClock: (options: {
id: string;
docId: string;
@@ -199,6 +217,10 @@ export interface NbStorePlugin {
indexedClock: number;
indexerVersion: number;
}) => Promise<void>;
setDocIndexedClocks: (options: {
id: string;
clocks: Array<{ docId: string; timestamp: number; indexerVersion: number }>;
}) => Promise<void>;
clearDocIndexedClock: (options: {
id: string;
docId: string;
packages/frontend/apps/ios/src/plugins/nbstore/index.ts
+58 -53
View File
@@ -2,7 +2,6 @@ import {
base64ToUint8Array,
uint8ArrayToBase64,
} from '@affine/core/modules/workspace-engine';
import { normalizeNativeOptional } from '@affine/mobile-shared/nbstore/optional';
import {
decodePayload,
MOBILE_BLOB_FILE_PREFIX,
@@ -358,77 +357,74 @@ export const NbStoreNativeDBApis: NativeDBApis = {
): Promise<CrawlResult> {
return await NbStore.crawlDocData({ id, docId });
},
ftsAddDocument: async function (
indexUpsert: async function (
id: string,
indexName: string,
docId: string,
text: string,
index: boolean
table: string,
document
): Promise<void> {
await NbStore.ftsAddDocument({
await NbStore.indexUpsert({
id,
indexName,
docId,
text,
index,
table,
document,
});
},
ftsDeleteDocument: async function (
indexDelete: async function (
id: string,
indexName: string,
table: string,
docId: string
): Promise<void> {
await NbStore.ftsDeleteDocument({
await NbStore.indexDelete({
id,
indexName,
table,
docId,
});
},
ftsSearch: async function (
id: string,
indexName: string,
query: string
): Promise<{ id: string; score: number; terms: Array<string> }[]> {
const { results } = await NbStore.ftsSearch({
indexSearch: async function (id: string, table: string, query, options) {
return await NbStore.indexSearch({
id,
indexName,
table,
query,
options,
});
},
indexAggregate: async function (
id: string,
table: string,
query,
field: string,
limit: number,
offset: number,
hits
) {
return await NbStore.indexAggregate({
id,
table,
query,
field,
limit,
offset,
hits,
});
},
indexDeleteByQuery: async function (
id: string,
table: string,
query
): Promise<number> {
const { deleted } = await NbStore.indexDeleteByQuery({
id,
table,
query,
});
return results ?? [];
return deleted;
},
ftsGetDocument: async function (
id: string,
indexName: string,
docId: string
): Promise<string | null> {
const result = await NbStore.ftsGetDocument({
id,
indexName,
docId,
});
return normalizeNativeOptional(result.text);
},
ftsGetMatches: async function (
id: string,
indexName: string,
docId: string,
query: string
): Promise<{ start: number; end: number }[]> {
const { matches } = await NbStore.ftsGetMatches({
id,
indexName,
docId,
query,
});
return matches ?? [];
},
ftsFlushIndex: async function (id: string): Promise<void> {
await NbStore.ftsFlushIndex({
indexFlush: async function (id: string): Promise<void> {
await NbStore.indexFlush({
id,
});
},
ftsIndexVersion: function (): Promise<number> {
return NbStore.ftsIndexVersion().then(res => res.indexVersion);
indexVersion: function (): Promise<number> {
return NbStore.indexVersion().then(res => res.indexVersion);
},
getDocIndexedClock: function (
id: string,
@@ -451,6 +447,15 @@ export const NbStoreNativeDBApis: NativeDBApis = {
indexerVersion,
});
},
setDocIndexedClocks: function (id, clocks): Promise<void> {
return NbStore.setDocIndexedClocks({
id,
clocks: clocks.map(clock => ({
...clock,
timestamp: clock.timestamp.getTime(),
})),
});
},
clearDocIndexedClock: function (id: string, docId: string): Promise<void> {
return NbStore.clearDocIndexedClock({
id,
packages/frontend/core/src/modules/cloud/stores/server-config.spec.ts
+17
View File
@@ -3,10 +3,27 @@ import { describe, expect, test } from 'vitest';
import {
assertSupportedServerVersion,
getSyncProtocol,
isBatchSyncServerVersion,
MIN_SUPPORTED_SERVER_VERSION,
} from './server-config';
describe('server config version guard', () => {
test('selects batch sync from server version', () => {
expect(isBatchSyncServerVersion('0.27.4')).toBe(false);
expect(isBatchSyncServerVersion('0.27.5')).toBe(true);
expect(isBatchSyncServerVersion('0.27.5-beta.1')).toBe(true);
expect(isBatchSyncServerVersion('2026.8.20-canary.15')).toBe(true);
expect(isBatchSyncServerVersion('0.28.0')).toBe(true);
});
test('does not select a route before server version is verified', () => {
expect(() => getSyncProtocol()).toThrow(UserFriendlyError);
expect(() => getSyncProtocol('0.26.9')).toThrow(UserFriendlyError);
expect(getSyncProtocol('0.27.4')).toBe('legacy');
expect(getSyncProtocol('0.27.5')).toBe('batch');
});
test('accepts supported server versions', () => {
expect(() => assertSupportedServerVersion('0.27.0')).not.toThrow();
expect(() => assertSupportedServerVersion('0.27.0-beta.5')).not.toThrow();
packages/frontend/core/src/modules/cloud/stores/server-config.ts
+16
View File
@@ -14,6 +14,7 @@ export type ServerConfigType = ServerConfigQuery['serverConfig'] &
OauthProvidersQuery['serverConfig'];
export const MIN_SUPPORTED_SERVER_VERSION = '0.27.0';
export const BATCH_SYNC_SERVER_VERSION = '0.27.5';
const NETWORK_ERROR_PATTERNS = [
/failed to fetch/i,
@@ -63,6 +64,21 @@ export function assertSupportedServerVersion(version?: string | null) {
}
}
export function isBatchSyncServerVersion(version?: string | null) {
const normalized = version && semver.valid(version, { loose: true });
return (
!!normalized &&
semver.gte(normalized, `${BATCH_SYNC_SERVER_VERSION}-0`, {
loose: true,
})
);
}
export function getSyncProtocol(version?: string | null) {
assertSupportedServerVersion(version);
return isBatchSyncServerVersion(version) ? 'batch' : 'legacy';
}
function mapServerConfigError(error: unknown) {
const userFriendlyError = UserFriendlyError.fromAny(error);
if (
packages/frontend/core/src/modules/notification/services/count.spec.ts
+77
View File
@@ -0,0 +1,77 @@
import { Framework, LiveData } from '@toeverything/infra';
import { Subject } from 'rxjs';
import { describe, expect, test, vi } from 'vitest';
import { AuthService } from '../../cloud/services/auth';
import { NbstoreService } from '../../storage/services/nbstore';
import { NotificationStore } from '../stores/notification';
import { NotificationCountService } from './count';
function createCountService() {
const events$ = new Subject<{ type: 'ready' } | { count: number }>();
const request = vi.fn().mockResolvedValue({ count: 1 });
const cache = new LiveData(0);
const setNotificationCountCache = vi.fn((count: number) =>
cache.setValue(count)
);
const store = {
watchNotificationCountCache: () => cache,
setNotificationCountCache,
} as unknown as NotificationStore;
const auth = {
session: {
status$: new LiveData<'authenticated' | 'unauthenticated'>(
'authenticated'
),
},
} as unknown as AuthService;
const nbstore = {
realtime: {
request,
subscribe: () => events$,
},
} as unknown as NbstoreService;
const framework = new Framework();
framework.service(AuthService, auth);
framework.store(NotificationStore, store);
framework.service(NbstoreService, nbstore);
framework.service(NotificationCountService, [
NotificationStore,
AuthService,
NbstoreService,
]);
return {
events$,
request,
service: framework.provider().get(NotificationCountService),
setNotificationCountCache,
};
}
describe('NotificationCountService', () => {
test('uses snapshots for reconnects and applies realtime count changes', async () => {
const { events$, request, service, setNotificationCountCache } =
createCountService();
expect(service.loggedIn$.value).toBe(true);
service.handleServerStarted();
events$.next({ type: 'ready' });
await vi.waitFor(() => expect(request).toHaveBeenCalled());
await vi.waitFor(() => expect(service.count$.value).toBe(1));
events$.next({ count: 3 });
expect(service.count$.value).toBe(3);
expect(setNotificationCountCache).toHaveBeenLastCalledWith(3);
const requestsBeforeFocus = request.mock.calls.length;
service.handleApplicationFocused();
events$.next({ type: 'ready' });
await vi.waitFor(() =>
expect(request.mock.calls.length).toBeGreaterThan(requestsBeforeFocus)
);
await vi.waitFor(() => expect(service.count$.value).toBe(1));
service.dispose();
});
});
packages/frontend/core/src/modules/notification/services/count.ts
+6 -2
View File
@@ -1,8 +1,9 @@
import { LiveData, OnEvent, Service } from '@toeverything/infra';
import { AccountChanged, type AuthService } from '../../cloud';
import { AccountChanged } from '../../cloud/events/account-changed';
import { ServerStarted } from '../../cloud/events/server-started';
import { RealtimeLiveQuery } from '../../cloud/realtime/live-query';
import type { AuthService } from '../../cloud/services/auth';
import { ApplicationFocused } from '../../lifecycle';
import type { NbstoreService } from '../../storage';
import type { NotificationStore } from '../stores/notification';
@@ -19,7 +20,10 @@ export class NotificationCountService extends Service {
super();
}
loggedIn$ = this.authService.session.status$.map(v => v === 'authenticated');
loggedIn$ = LiveData.from(
this.authService.session.status$.map(v => v === 'authenticated'),
this.authService.session.status$.value === 'authenticated'
);
readonly count$ = LiveData.from(this.store.watchNotificationCountCache(), 0);
readonly isLoading$ = new LiveData(false);
packages/frontend/core/src/modules/userspace/entities/user-db-engine.ts
+4
View File
@@ -8,6 +8,7 @@ import type { StoreClient } from '@affine/nbstore/worker/client';
import { Entity } from '@toeverything/infra';
import type { ServerService } from '../../cloud';
import { getSyncProtocol } from '../../cloud/stores/server-config';
import type { NbstoreService } from '../../storage';
export class UserDBEngine extends Entity<{
@@ -64,6 +65,9 @@ export class UserDBEngine extends Entity<{
opts: {
id: this.userId,
serverBaseUrl: serverService.server.baseUrl,
syncProtocol: getSyncProtocol(
serverService.server.config$.value.version
),
type: 'userspace',
isSelfHosted:
serverService.server.config$.value.type ===
packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts
+3
View File
@@ -66,6 +66,7 @@ import {
GraphQLService,
WorkspaceServerService,
} from '../../cloud';
import { getSyncProtocol } from '../../cloud/stores/server-config';
import { type GlobalState, NbstoreService } from '../../storage';
import type {
Workspace,
@@ -519,6 +520,7 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider {
type: 'workspace',
id: workspaceId,
serverBaseUrl: this.server.serverMetadata.baseUrl,
syncProtocol: getSyncProtocol(this.server.config$.value.version),
isSelfHosted:
this.server.config$.value.type ===
ServerDeploymentType.Selfhosted,
@@ -537,6 +539,7 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider {
type: 'workspace',
id: workspaceId,
serverBaseUrl: this.server.serverMetadata.baseUrl,
syncProtocol: getSyncProtocol(this.server.config$.value.version),
isSelfHosted:
this.server.config$.value.type ===
ServerDeploymentType.Selfhosted,
packages/frontend/mobile-native/Cargo.toml
+1
View File
@@ -22,6 +22,7 @@ affine_nbstore = { workspace = true }
anyhow = { workspace = true }
base64-simd = { workspace = true }
chrono = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread"] }
uniffi = { workspace = true, features = ["cli", "tokio"] }
packages/frontend/mobile-native/src/error.rs
+6
View File
@@ -16,4 +16,10 @@ impl From<NbStoreError> for UniffiError {
}
}
impl From<serde_json::Error> for UniffiError {
fn from(err: serde_json::Error) -> Self {
Self::Err(err.to_string())
}
}
pub(crate) type Result<T> = std::result::Result<T, UniffiError>;
packages/frontend/mobile-native/src/ffi_types.rs
+145 -15
View File
@@ -1,7 +1,10 @@
use affine_nbstore::{
Blob as NbBlob, Data, DocClock as NbDocClock, DocRecord as NbDocRecord, DocUpdate as NbDocUpdate,
ListedBlob as NbListedBlob, SetBlob as NbSetBlob,
indexer::{NativeBlockInfo, NativeCrawlResult, NativeMatch, NativeSearchHit},
Blob as NbBlob, Data, DocClock as NbDocClock, DocIndexedClock as NbDocIndexedClock, DocRecord as NbDocRecord,
DocUpdate as NbDocUpdate, ListedBlob as NbListedBlob, SetBlob as NbSetBlob,
indexer::{
NativeBlockInfo, NativeCrawlResult, NativeIndexAggregateResult, NativeIndexBucket, NativeIndexField,
NativeIndexHighlight, NativeIndexHighlightValue, NativeIndexHit, NativeIndexSearchResult, NativeIndexSpan,
},
};
use chrono::{DateTime, Utc};
@@ -80,6 +83,37 @@ pub struct DocClock {
pub timestamp: i64,
}
#[derive(uniffi::Record)]
pub struct DocIndexedClock {
pub doc_id: String,
pub timestamp: i64,
pub indexer_version: i64,
}
impl From<NbDocIndexedClock> for DocIndexedClock {
fn from(clock: NbDocIndexedClock) -> Self {
Self {
doc_id: clock.doc_id,
timestamp: clock.timestamp.and_utc().timestamp_millis(),
indexer_version: clock.indexer_version,
}
}
}
impl TryFrom<DocIndexedClock> for NbDocIndexedClock {
type Error = UniffiError;
fn try_from(clock: DocIndexedClock) -> Result<Self> {
Ok(Self {
doc_id: clock.doc_id,
timestamp: DateTime::<Utc>::from_timestamp_millis(clock.timestamp)
.ok_or(UniffiError::TimestampDecodingError)?
.naive_utc(),
indexer_version: clock.indexer_version,
})
}
}
impl From<NbDocClock> for DocClock {
fn from(clock: NbDocClock) -> Self {
Self {
@@ -211,33 +245,129 @@ impl From<NativeCrawlResult> for CrawlResult {
}
#[derive(uniffi::Record)]
pub struct SearchHit {
pub id: String,
pub score: f64,
pub terms: Vec<String>,
pub struct IndexField {
pub field: String,
pub values: Vec<String>,
}
impl From<NativeSearchHit> for SearchHit {
fn from(value: NativeSearchHit) -> Self {
impl From<NativeIndexField> for IndexField {
fn from(value: NativeIndexField) -> Self {
Self {
id: value.id,
score: value.score,
terms: value.terms,
field: value.field,
values: value.values,
}
}
}
#[derive(uniffi::Record)]
pub struct MatchRange {
pub struct IndexSpan {
pub start: u32,
pub end: u32,
}
impl From<NativeMatch> for MatchRange {
fn from(value: NativeMatch) -> Self {
impl From<NativeIndexSpan> for IndexSpan {
fn from(value: NativeIndexSpan) -> Self {
Self {
start: value.start,
end: value.end,
}
}
}
#[derive(uniffi::Record)]
pub struct IndexHighlightValue {
pub value_index: u32,
pub spans: Vec<IndexSpan>,
}
impl From<NativeIndexHighlightValue> for IndexHighlightValue {
fn from(value: NativeIndexHighlightValue) -> Self {
Self {
value_index: value.value_index,
spans: value.spans.into_iter().map(Into::into).collect(),
}
}
}
#[derive(uniffi::Record)]
pub struct IndexHighlight {
pub field: String,
pub values: Vec<IndexHighlightValue>,
}
impl From<NativeIndexHighlight> for IndexHighlight {
fn from(value: NativeIndexHighlight) -> Self {
Self {
field: value.field,
values: value.values.into_iter().map(Into::into).collect(),
}
}
}
#[derive(uniffi::Record)]
pub struct IndexHit {
pub id: String,
pub score: f64,
pub fields: Vec<IndexField>,
pub highlights: Vec<IndexHighlight>,
}
impl From<NativeIndexHit> for IndexHit {
fn from(value: NativeIndexHit) -> Self {
Self {
id: value.id,
score: value.score,
fields: value.fields.into_iter().map(Into::into).collect(),
highlights: value.highlights.into_iter().map(Into::into).collect(),
}
}
}
#[derive(uniffi::Record)]
pub struct IndexSearchResult {
pub total: u32,
pub hits: Vec<IndexHit>,
}
impl From<NativeIndexSearchResult> for IndexSearchResult {
fn from(value: NativeIndexSearchResult) -> Self {
Self {
total: value.total,
hits: value.hits.into_iter().map(Into::into).collect(),
}
}
}
#[derive(uniffi::Record)]
pub struct IndexBucket {
pub key: String,
pub count: u32,
pub score: f64,
pub hits: Vec<IndexHit>,
}
impl From<NativeIndexBucket> for IndexBucket {
fn from(value: NativeIndexBucket) -> Self {
Self {
key: value.key,
count: value.count,
score: value.score,
hits: value.hits.into_iter().map(Into::into).collect(),
}
}
}
#[derive(uniffi::Record)]
pub struct IndexAggregateResult {
pub total: u32,
pub buckets: Vec<IndexBucket>,
}
impl From<NativeIndexAggregateResult> for IndexAggregateResult {
fn from(value: NativeIndexAggregateResult) -> Self {
Self {
total: value.total,
buckets: value.buckets.into_iter().map(Into::into).collect(),
}
}
}
packages/frontend/mobile-native/src/lib.rs
+2 -1
View File
@@ -14,7 +14,8 @@ use affine_common::hashcash::Stamp;
pub(crate) use error::Result;
pub use error::UniffiError;
pub use ffi_types::{
Blob, BlockInfo, CrawlResult, DocClock, DocRecord, DocUpdate, ListedBlob, MatchRange, SearchHit, SetBlob,
Blob, BlockInfo, CrawlResult, DocClock, DocIndexedClock, DocRecord, DocUpdate, IndexAggregateResult, IndexBucket,
IndexField, IndexHighlight, IndexHighlightValue, IndexHit, IndexSearchResult, IndexSpan, ListedBlob, SetBlob,
};
#[cfg(any(target_os = "android", target_os = "ios"))]
pub use preview::{render_mermaid_preview_svg, render_typst_preview_svg};
packages/frontend/mobile-native/src/storage/indexer.rs
+100 -52
View File
@@ -1,3 +1,5 @@
use affine_nbstore::indexer::{NativeIndexDocument, NativeIndexQuery, NativeIndexSearchOptions};
use super::*;
#[uniffi::export(async_runtime = "tokio")]
@@ -12,89 +14,135 @@ impl DocStoragePool {
Ok(result.into())
}
pub async fn fts_add_document(
&self,
universal_id: String,
index_name: String,
doc_id: String,
text: String,
index: bool,
) -> Result<()> {
pub async fn index_upsert(&self, universal_id: String, table: String, document: String) -> Result<()> {
let document: NativeIndexDocument = serde_json::from_str(&document)?;
self
.inner
.get(universal_id)
.await?
.fts_add(&index_name, &doc_id, &text, index)
.index_upsert(&table, document)
.await?;
Ok(())
}
pub async fn fts_delete_document(&self, universal_id: String, index_name: String, doc_id: String) -> Result<()> {
pub async fn index_delete(&self, universal_id: String, table: String, doc_id: String) -> Result<()> {
self
.inner
.get(universal_id)
.await?
.fts_delete(&index_name, &doc_id)
.index_delete(&table, &doc_id)
.await?;
Ok(())
}
pub async fn fts_get_document(
pub async fn index_search(
&self,
universal_id: String,
index_name: String,
doc_id: String,
) -> Result<Option<String>> {
table: String,
query: String,
options: String,
) -> Result<IndexSearchResult> {
let query: NativeIndexQuery = serde_json::from_str(&query)?;
let options: NativeIndexSearchOptions = serde_json::from_str(&options)?;
Ok(
self
.inner
.get(universal_id)
.await?
.fts_get(&index_name, &doc_id)
.index_search(&table, query, options)
.await?
.into(),
)
}
#[allow(clippy::too_many_arguments)]
pub async fn index_aggregate(
&self,
universal_id: String,
table: String,
query: String,
field: String,
limit: u32,
offset: u32,
hits: Option<String>,
) -> Result<IndexAggregateResult> {
let query: NativeIndexQuery = serde_json::from_str(&query)?;
let hits = hits
.map(|value| serde_json::from_str::<NativeIndexSearchOptions>(&value))
.transpose()?;
Ok(
self
.inner
.get(universal_id)
.await?
.index_aggregate(&table, query, &field, limit, offset, hits)
.await?
.into(),
)
}
pub async fn index_delete_by_query(&self, universal_id: String, table: String, query: String) -> Result<u32> {
let query: NativeIndexQuery = serde_json::from_str(&query)?;
Ok(
self
.inner
.get(universal_id)
.await?
.index_delete_by_query(&table, query)
.await?,
)
}
pub async fn fts_search(&self, universal_id: String, index_name: String, query: String) -> Result<Vec<SearchHit>> {
Ok(
self
.inner
.get(universal_id)
.await?
.fts_search(&index_name, &query)
.await?
.into_iter()
.map(Into::into)
.collect(),
)
}
pub async fn fts_get_matches(
&self,
universal_id: String,
index_name: String,
doc_id: String,
query: String,
) -> Result<Vec<MatchRange>> {
Ok(
self
.inner
.get(universal_id)
.await?
.fts_get_matches(&index_name, &doc_id, &query)
.await?
.into_iter()
.map(Into::into)
.collect(),
)
}
pub async fn fts_flush_index(&self, universal_id: String) -> Result<()> {
pub async fn index_flush(&self, universal_id: String) -> Result<()> {
self.inner.get(universal_id).await?.flush_index().await?;
Ok(())
}
pub async fn fts_index_version(&self) -> Result<u32> {
pub async fn index_version(&self) -> Result<u32> {
Ok(SqliteDocStorage::index_version())
}
pub async fn set_doc_indexed_clocks(&self, universal_id: String, clocks: Vec<DocIndexedClock>) -> Result<()> {
let clocks = clocks.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?;
self
.inner
.get(universal_id)
.await?
.commit_indexed_clocks(&clocks)
.await?;
Ok(())
}
pub async fn get_doc_indexed_clock(&self, universal_id: String, doc_id: String) -> Result<Option<DocIndexedClock>> {
Ok(
self
.inner
.get(universal_id)
.await?
.get_doc_indexed_clock(doc_id)
.await?
.map(Into::into),
)
}
pub async fn set_doc_indexed_clock(&self, universal_id: String, clock: DocIndexedClock) -> Result<()> {
let clock = clock.try_into()?;
self
.inner
.get(universal_id)
.await?
.commit_indexed_clocks(&[clock])
.await?;
Ok(())
}
pub async fn clear_doc_indexed_clock(&self, universal_id: String, doc_id: String) -> Result<()> {
self
.inner
.get(universal_id)
.await?
.clear_doc_indexed_clock(doc_id)
.await?;
Ok(())
}
}
packages/frontend/mobile-native/src/storage/mod.rs
+2 -1
View File
@@ -15,7 +15,8 @@ use chrono::{DateTime, NaiveDateTime, Utc};
#[cfg(any(target_os = "android", target_os = "ios", test))]
use crate::cache::{MobileBlobCache, is_mobile_binary_file_token, should_cache_payload_as_file};
use crate::{
Blob, CrawlResult, DocClock, DocRecord, DocUpdate, ListedBlob, MatchRange, Result, SearchHit, SetBlob, UniffiError,
Blob, CrawlResult, DocClock, DocIndexedClock, DocRecord, DocUpdate, IndexAggregateResult, IndexSearchResult,
ListedBlob, Result, SetBlob, UniffiError,
payload_codec::{decode_base64_data, encode_base64_data},
};
packages/frontend/native/index.d.ts
Vendored
+67 -12
View File
@@ -154,6 +154,7 @@ export declare class DocStoragePool {
getDocClock(universalId: string, docId: string): Promise<DocClock | null>
getDocIndexedClock(universalId: string, docId: string): Promise<DocIndexedClock | null>
setDocIndexedClock(universalId: string, docId: string, indexedClock: Date, indexerVersion: number): Promise<void>
setDocIndexedClocks(universalId: string, clocks: Array<DocIndexedClock>): Promise<void>
clearDocIndexedClock(universalId: string, docId: string): Promise<void>
getBlob(universalId: string, key: string): Promise<Blob | null>
setBlob(universalId: string, blob: SetBlob): Promise<void>
@@ -172,13 +173,13 @@ export declare class DocStoragePool {
clearClocks(universalId: string): Promise<void>
setBlobUploadedAt(universalId: string, peer: string, blobId: string, uploadedAt?: Date | undefined | null): Promise<void>
getBlobUploadedAt(universalId: string, peer: string, blobId: string): Promise<Date | null>
ftsAddDocument(id: string, indexName: string, docId: string, text: string, index: boolean): Promise<void>
ftsFlushIndex(id: string): Promise<void>
ftsIndexVersion(): Promise<number>
ftsDeleteDocument(id: string, indexName: string, docId: string): Promise<void>
ftsGetDocument(id: string, indexName: string, docId: string): Promise<string | null>
ftsSearch(id: string, indexName: string, query: string): Promise<Array<NativeSearchHit>>
ftsGetMatches(id: string, indexName: string, docId: string, query: string): Promise<Array<NativeMatch>>
indexUpsert(id: string, table: string, document: NativeIndexDocument): Promise<void>
indexFlush(id: string): Promise<void>
indexVersion(): Promise<number>
indexDelete(id: string, table: string, docId: string): Promise<void>
indexSearch(id: string, table: string, query: NativeIndexQuery, options: NativeIndexSearchOptions): Promise<NativeIndexSearchResult>
indexAggregate(id: string, table: string, query: NativeIndexQuery, field: string, limit: number, offset: number, hits?: NativeIndexSearchOptions | undefined | null): Promise<NativeIndexAggregateResult>
indexDeleteByQuery(id: string, table: string, query: NativeIndexQuery): Promise<number>
}
export interface Blob {
@@ -237,15 +238,69 @@ export interface NativeCrawlResult {
summary: string
}
export interface NativeMatch {
start: number
end: number
export interface NativeIndexAggregateResult {
total: number
buckets: Array<NativeIndexBucket>
}
export interface NativeSearchHit {
export interface NativeIndexBucket {
key: string
count: number
score: number
hits: Array<NativeIndexHit>
}
export interface NativeIndexDocument {
id: string
fields: Array<NativeIndexField>
}
export interface NativeIndexField {
field: string
values: Array<string>
}
export interface NativeIndexHighlight {
field: string
values: Array<NativeIndexHighlightValue>
}
export interface NativeIndexHighlightValue {
valueIndex: number
spans: Array<NativeIndexSpan>
}
export interface NativeIndexHit {
id: string
score: number
terms: Array<string>
fields: Array<NativeIndexField>
highlights: Array<NativeIndexHighlight>
}
export interface NativeIndexQuery {
kind: string
field?: string
value?: string
occur?: string
clauses?: Array<NativeIndexQuery>
boost?: number
}
export interface NativeIndexSearchOptions {
limit: number
offset: number
fields: Array<string>
highlights: Array<string>
}
export interface NativeIndexSearchResult {
total: number
hits: Array<NativeIndexHit>
}
export interface NativeIndexSpan {
start: number
end: number
}
export interface SetBlob {
packages/frontend/native/nbstore/src/error.rs
+4
View File
@@ -12,8 +12,12 @@ pub enum Error {
ConnectionInProgress,
#[error("Invalid operation")]
InvalidOperation,
#[error("Index is rebuilding")]
IndexNotReady,
#[error("Serialization Error: {0}")]
Serialization(String),
#[error(transparent)]
Indexer(#[from] memory_indexer::Error),
#[error(transparent)]
Parse(#[from] ParseError),
}
packages/frontend/native/nbstore/src/indexer.rs
-353
View File
@@ -1,353 +0,0 @@
use affine_doc_loader::{BlockInfo, CrawlResult, ParseError, parse_doc_from_binary};
use memory_indexer::{SearchHit, SnapshotData};
use napi_derive::napi;
use serde::Serialize;
use sqlx::Row;
use y_octo::merge_updates_v1;
// Increment this whenever there is a breaking change in the index format or how
// updates are applied
const NBSTORE_INDEXER_VERSION: u32 = 1;
use super::{
error::{Error, Result},
storage::SqliteDocStorage,
};
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeBlockInfo {
pub block_id: String,
pub flavour: String,
pub content: Option<Vec<String>>,
pub blob: Option<Vec<String>>,
pub ref_doc_id: Option<Vec<String>>,
pub ref_info: Option<Vec<String>>,
pub parent_flavour: Option<String>,
pub parent_block_id: Option<String>,
pub additional: Option<String>,
}
impl From<BlockInfo> for NativeBlockInfo {
fn from(value: BlockInfo) -> Self {
Self {
block_id: value.block_id,
flavour: value.flavour,
content: value.content,
blob: value.blob,
ref_doc_id: value.ref_doc_id,
ref_info: value.ref_info,
parent_flavour: value.parent_flavour,
parent_block_id: value.parent_block_id,
additional: value.additional,
}
}
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeCrawlResult {
pub blocks: Vec<NativeBlockInfo>,
pub title: String,
pub summary: String,
}
impl From<CrawlResult> for NativeCrawlResult {
fn from(value: CrawlResult) -> Self {
Self {
blocks: value.blocks.into_iter().map(Into::into).collect(),
title: value.title,
summary: value.summary,
}
}
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeSearchHit {
pub id: String,
pub score: f64,
pub terms: Vec<String>,
}
impl From<SearchHit> for NativeSearchHit {
fn from(value: SearchHit) -> Self {
Self {
id: value.doc_id,
score: value.score,
terms: value.matched_terms.into_iter().map(|t| t.term).collect(),
}
}
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeMatch {
pub start: u32,
pub end: u32,
}
impl From<(u32, u32)> for NativeMatch {
fn from(value: (u32, u32)) -> Self {
Self {
start: value.0,
end: value.1,
}
}
}
impl SqliteDocStorage {
pub async fn crawl_doc_data(&self, doc_id: &str) -> Result<NativeCrawlResult> {
let doc_bin = self.load_doc_binary(doc_id).await?.ok_or(ParseError::DocNotFound)?;
let result = parse_doc_from_binary(doc_bin, doc_id.to_string())?;
Ok(result.into())
}
async fn load_doc_binary(&self, doc_id: &str) -> Result<Option<Vec<u8>>> {
let snapshot = self.get_doc_snapshot(doc_id.to_string()).await?;
let mut updates = self.get_doc_updates(doc_id.to_string()).await?;
if snapshot.is_none() && updates.is_empty() {
return Ok(None);
}
updates.sort_by_key(|a| a.timestamp);
let mut segments = Vec::with_capacity(snapshot.as_ref().map(|_| 1).unwrap_or(0) + updates.len());
if let Some(record) = snapshot {
segments.push(record.bin.to_vec());
}
segments.extend(updates.into_iter().map(|update| update.bin.to_vec()));
merge_updates(segments).map(Some)
}
pub async fn init_index(&self) -> Result<()> {
let snapshots = sqlx::query("SELECT index_name, data FROM idx_snapshots")
.fetch_all(&self.pool)
.await?;
{
let mut index = self.index.write().await;
let config = bincode::config::standard();
for row in snapshots {
let index_name: String = row.get("index_name");
let data: Vec<u8> = row.get("data");
if let Ok(decompressed) = zstd::stream::decode_all(std::io::Cursor::new(&data))
&& let Ok((snapshot, _)) = bincode::serde::decode_from_slice::<SnapshotData, _>(&decompressed, config)
{
index.load_snapshot(&index_name, snapshot);
}
}
}
Ok(())
}
async fn compact_index(&self, index_name: &str) -> Result<()> {
let snapshot_data = {
let index = self.index.read().await;
index.get_snapshot_data(index_name)
};
if let Some(data) = snapshot_data {
let blob = bincode::serde::encode_to_vec(&data, bincode::config::standard())
.map_err(|e| Error::Serialization(e.to_string()))?;
let compressed =
zstd::stream::encode_all(std::io::Cursor::new(&blob), 4).map_err(|e| Error::Serialization(e.to_string()))?;
let mut tx = self.pool.begin().await?;
sqlx::query("INSERT OR REPLACE INTO idx_snapshots (index_name, data) VALUES (?, ?)")
.bind(index_name)
.bind(compressed)
.execute(&mut *tx)
.await?;
tx.commit().await?;
}
Ok(())
}
pub async fn flush_index(&self) -> Result<()> {
let (dirty_docs, deleted_docs) = {
let mut index = self.index.write().await;
index.take_dirty_and_deleted()
};
if dirty_docs.is_empty() && deleted_docs.is_empty() {
return Ok(());
}
let mut modified_indices = std::collections::HashSet::new();
for index_name in deleted_docs.keys() {
modified_indices.insert(index_name.clone());
}
for (index_name, _, _, _) in &dirty_docs {
modified_indices.insert(index_name.clone());
}
for index_name in modified_indices {
self.compact_index(&index_name).await?;
}
Ok(())
}
pub fn index_version() -> u32 {
memory_indexer::InMemoryIndex::snapshot_version() + NBSTORE_INDEXER_VERSION
}
pub async fn fts_add(&self, index_name: &str, doc_id: &str, text: &str, index: bool) -> Result<()> {
let mut idx = self.index.write().await;
idx.add_doc(index_name, doc_id, text, index);
Ok(())
}
pub async fn fts_delete(&self, index_name: &str, doc_id: &str) -> Result<()> {
let mut idx = self.index.write().await;
idx.remove_doc(index_name, doc_id);
Ok(())
}
pub async fn fts_get(&self, index_name: &str, doc_id: &str) -> Result<Option<String>> {
let idx = self.index.read().await;
Ok(idx.get_doc(index_name, doc_id))
}
pub async fn fts_search(&self, index_name: &str, query: &str) -> Result<Vec<NativeSearchHit>> {
let idx = self.index.read().await;
Ok(idx.search_hits(index_name, query).into_iter().map(Into::into).collect())
}
pub async fn fts_get_matches(&self, index_name: &str, doc_id: &str, query: &str) -> Result<Vec<NativeMatch>> {
let idx = self.index.read().await;
Ok(
idx
.get_matches(index_name, doc_id, query)
.into_iter()
.map(Into::into)
.collect(),
)
}
pub async fn fts_get_matches_for_terms(
&self,
index_name: &str,
doc_id: &str,
terms: Vec<String>,
) -> Result<Vec<NativeMatch>> {
let idx = self.index.read().await;
Ok(
idx
.get_matches_for_terms(index_name, doc_id, &terms)
.into_iter()
.map(Into::into)
.collect(),
)
}
}
fn merge_updates(mut segments: Vec<Vec<u8>>) -> Result<Vec<u8>> {
if segments.is_empty() {
return Err(ParseError::DocNotFound.into());
}
if segments.len() == 1 {
return segments.pop().ok_or(ParseError::DocNotFound.into());
}
let update = merge_updates_v1(segments).map_err(|_| ParseError::InvalidBinary)?;
let buffer = update
.encode_v1()
.map_err(|err| ParseError::ParserError(err.to_string()))?;
Ok(buffer)
}
#[cfg(test)]
mod tests {
use std::path::{Path, PathBuf};
use affine_doc_loader::ParseError;
use assert_json_diff::assert_json_eq;
use chrono::Utc;
use serde_json::Value;
use tokio::fs;
use uuid::Uuid;
use super::{super::error::Error, *};
const DEMO_BIN: &[u8] = include_bytes!("../../../../common/native/fixtures/demo.ydoc");
const DEMO_JSON: &[u8] = include_bytes!("../../../../common/native/fixtures/demo.ydoc.json");
fn temp_workspace_dir() -> PathBuf {
std::env::temp_dir().join(format!("affine-native-{}", Uuid::new_v4()))
}
async fn init_db(path: &Path) -> SqliteDocStorage {
fs::create_dir_all(path.parent().unwrap()).await.unwrap();
let storage = SqliteDocStorage::new(path.to_string_lossy().into_owned());
storage.connect().await.unwrap();
storage
}
async fn cleanup(path: &Path) {
let _ = fs::remove_dir_all(path.parent().unwrap()).await;
}
#[tokio::test]
async fn parse_demo_snapshot_matches_fixture() {
let base = temp_workspace_dir();
fs::create_dir_all(&base).await.unwrap();
let db_path = base.join("storage.db");
let storage = init_db(&db_path).await;
sqlx::query(r#"INSERT INTO snapshots (doc_id, data, updated_at) VALUES (?, ?, ?)"#)
.bind("demo-doc")
.bind(DEMO_BIN)
.bind(Utc::now().naive_utc())
.execute(&storage.pool)
.await
.unwrap();
sqlx::query(r#"INSERT INTO updates (doc_id, data, created_at) VALUES (?, ?, ?)"#)
.bind("demo-doc")
.bind(&[0, 0][..])
.bind(Utc::now().naive_utc())
.execute(&storage.pool)
.await
.unwrap();
let result = storage.crawl_doc_data("demo-doc").await.unwrap();
let mut expected: Value = serde_json::from_slice(DEMO_JSON).unwrap();
let mut actual = serde_json::to_value(&result).unwrap();
for document in [&mut expected, &mut actual] {
for block in document["blocks"].as_array_mut().unwrap() {
if let Some(additional) = block["additional"].as_str() {
block["additional"] = serde_json::from_str(additional).unwrap();
}
}
}
assert_json_eq!(expected, actual);
storage.close().await;
cleanup(&db_path).await;
}
#[tokio::test]
async fn missing_doc_returns_error() {
let base = temp_workspace_dir();
fs::create_dir_all(&base).await.unwrap();
let db_path = base.join("storage.db");
let storage = init_db(&db_path).await;
let err = storage.crawl_doc_data("absent-doc").await.unwrap_err();
assert!(matches!(err, Error::Parse(ParseError::DocNotFound)));
storage.close().await;
cleanup(&db_path).await;
}
}
packages/frontend/native/nbstore/src/indexer/crawl.rs
+39
View File
@@ -0,0 +1,39 @@
use affine_doc_loader::{ParseError, parse_doc_from_binary};
use y_octo::merge_updates_v1;
use super::{NativeCrawlResult, SqliteDocStorage, error::Result};
impl SqliteDocStorage {
pub async fn crawl_doc_data(&self, doc_id: &str) -> Result<NativeCrawlResult> {
let doc_bin = self.load_doc_binary(doc_id).await?.ok_or(ParseError::DocNotFound)?;
Ok(parse_doc_from_binary(doc_bin, doc_id.to_string())?.into())
}
async fn load_doc_binary(&self, doc_id: &str) -> Result<Option<Vec<u8>>> {
let snapshot = self.get_doc_snapshot(doc_id.to_string()).await?;
let mut updates = self.get_doc_updates(doc_id.to_string()).await?;
if snapshot.is_none() && updates.is_empty() {
return Ok(None);
}
updates.sort_by_key(|update| update.timestamp);
let mut segments = Vec::with_capacity(snapshot.as_ref().map(|_| 1).unwrap_or(0) + updates.len());
if let Some(record) = snapshot {
segments.push(record.bin.to_vec());
}
segments.extend(updates.into_iter().map(|update| update.bin.to_vec()));
merge_updates(segments).map(Some)
}
}
fn merge_updates(mut segments: Vec<Vec<u8>>) -> Result<Vec<u8>> {
if segments.is_empty() {
return Err(ParseError::DocNotFound.into());
}
if segments.len() == 1 {
return segments.pop().ok_or(ParseError::DocNotFound.into());
}
let update = merge_updates_v1(segments).map_err(|_| ParseError::InvalidBinary)?;
update
.encode_v1()
.map_err(|error| ParseError::ParserError(error.to_string()).into())
}
packages/frontend/native/nbstore/src/indexer/dto.rs
+142
View File
@@ -0,0 +1,142 @@
use affine_doc_loader::{BlockInfo, CrawlResult};
use napi_derive::napi;
use serde::{Deserialize, Serialize};
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeBlockInfo {
pub block_id: String,
pub flavour: String,
pub content: Option<Vec<String>>,
pub blob: Option<Vec<String>>,
pub ref_doc_id: Option<Vec<String>>,
pub ref_info: Option<Vec<String>>,
pub parent_flavour: Option<String>,
pub parent_block_id: Option<String>,
pub additional: Option<String>,
}
impl From<BlockInfo> for NativeBlockInfo {
fn from(value: BlockInfo) -> Self {
Self {
block_id: value.block_id,
flavour: value.flavour,
content: value.content,
blob: value.blob,
ref_doc_id: value.ref_doc_id,
ref_info: value.ref_info,
parent_flavour: value.parent_flavour,
parent_block_id: value.parent_block_id,
additional: value.additional,
}
}
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeCrawlResult {
pub blocks: Vec<NativeBlockInfo>,
pub title: String,
pub summary: String,
}
impl From<CrawlResult> for NativeCrawlResult {
fn from(value: CrawlResult) -> Self {
Self {
blocks: value.blocks.into_iter().map(Into::into).collect(),
title: value.title,
summary: value.summary,
}
}
}
#[napi(object)]
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct NativeIndexField {
pub field: String,
pub values: Vec<String>,
}
#[napi(object)]
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct NativeIndexDocument {
pub id: String,
pub fields: Vec<NativeIndexField>,
}
#[napi(object)]
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct NativeIndexQuery {
pub kind: String,
pub field: Option<String>,
pub value: Option<String>,
pub occur: Option<String>,
pub clauses: Option<Vec<NativeIndexQuery>>,
pub boost: Option<f64>,
}
#[napi(object)]
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct NativeIndexSearchOptions {
pub limit: u32,
pub offset: u32,
pub fields: Vec<String>,
pub highlights: Vec<String>,
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeIndexSpan {
pub start: u32,
pub end: u32,
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeIndexHighlightValue {
pub value_index: u32,
pub spans: Vec<NativeIndexSpan>,
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeIndexHighlight {
pub field: String,
pub values: Vec<NativeIndexHighlightValue>,
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeIndexHit {
pub id: String,
pub score: f64,
pub fields: Vec<NativeIndexField>,
pub highlights: Vec<NativeIndexHighlight>,
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeIndexSearchResult {
pub total: u32,
pub hits: Vec<NativeIndexHit>,
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeIndexBucket {
pub key: String,
pub count: u32,
pub score: f64,
pub hits: Vec<NativeIndexHit>,
}
#[napi(object)]
#[derive(Debug, Serialize)]
pub struct NativeIndexAggregateResult {
pub total: u32,
pub buckets: Vec<NativeIndexBucket>,
}
packages/frontend/native/nbstore/src/indexer/mod.rs
+71
View File
@@ -0,0 +1,71 @@
mod crawl;
mod dto;
mod persistence;
mod table;
use std::sync::atomic::{AtomicBool, Ordering};
pub use dto::{
NativeBlockInfo, NativeCrawlResult, NativeIndexAggregateResult, NativeIndexBucket, NativeIndexDocument,
NativeIndexField, NativeIndexHighlight, NativeIndexHighlightValue, NativeIndexHit, NativeIndexQuery,
NativeIndexSearchOptions, NativeIndexSearchResult, NativeIndexSpan,
};
use error::{Error, Result};
use memory_indexer::MemoryIndex;
pub(super) use table::{TableIndex, string_values};
pub(super) use super::{DocIndexedClock, error, storage::SqliteDocStorage};
const NBSTORE_INDEXER_VERSION: u32 = 7;
pub struct IndexManager {
pub(super) doc: TableIndex,
pub(super) block: TableIndex,
pub(super) ready: AtomicBool,
}
impl IndexManager {
pub fn new() -> Self {
Self {
doc: TableIndex::doc(),
block: TableIndex::block(),
ready: AtomicBool::new(true),
}
}
pub(super) fn table(&self, table: &str) -> Result<&TableIndex> {
match table {
"doc" => Ok(&self.doc),
"block" => Ok(&self.block),
_ => Err(Error::Serialization(format!("unknown index table {table}"))),
}
}
pub(super) fn tables(&self) -> [(&'static str, &TableIndex); 2] {
[("doc", &self.doc), ("block", &self.block)]
}
pub(super) fn ensure_ready(&self) -> Result<()> {
self
.ready
.load(Ordering::Acquire)
.then_some(())
.ok_or(Error::IndexNotReady)
}
pub(super) async fn reset(&self) {
for (_, table) in self.tables() {
*table.index.write().await = MemoryIndex::new(table.schema.clone());
}
self.ready.store(false, Ordering::Release);
}
}
impl Default for IndexManager {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests;
packages/frontend/native/nbstore/src/indexer/persistence.rs
+193
View File
@@ -0,0 +1,193 @@
use std::sync::atomic::Ordering;
use memory_indexer::{Document, MemoryIndex, TermsAggregation, Value};
use sqlx::Row;
use super::{
DocIndexedClock, NBSTORE_INDEXER_VERSION, NativeIndexAggregateResult, NativeIndexBucket, NativeIndexDocument,
NativeIndexQuery, NativeIndexSearchOptions, NativeIndexSearchResult, SqliteDocStorage, error::Result, string_values,
};
impl SqliteDocStorage {
pub async fn init_index(&self) -> Result<()> {
let snapshots = sqlx::query("SELECT index_name, data FROM idx_snapshots WHERE index_name IN ('doc', 'block')")
.fetch_all(&self.pool)
.await?;
let mut corrupted = false;
for row in snapshots {
let name: String = row.get("index_name");
let data: Vec<u8> = row.get("data");
let table = self.indexes.table(&name)?;
if let Ok(index) = MemoryIndex::from_checkpoint(table.schema.clone(), &data) {
*table.index.write().await = index;
} else {
corrupted = true;
}
}
if corrupted {
self.indexes.reset().await;
let mut tx = self.pool.begin().await?;
sqlx::query("DELETE FROM idx_snapshots").execute(&mut *tx).await?;
sqlx::query("DELETE FROM indexer_sync").execute(&mut *tx).await?;
tx.commit().await?;
}
sqlx::query("DELETE FROM idx_snapshots WHERE index_name NOT IN ('doc', 'block')")
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn flush_index(&self) -> Result<()> {
let mut checkpoints = Vec::new();
for (name, table) in self.indexes.tables() {
let index = table.index.read().await;
if index.has_unpersisted_changes() {
checkpoints.push((name, index.checkpoint()?));
}
}
if checkpoints.is_empty() {
return Ok(());
}
let mut tx = self.pool.begin().await?;
for (name, checkpoint) in &checkpoints {
sqlx::query("INSERT OR REPLACE INTO idx_snapshots (index_name, data) VALUES (?, ?)")
.bind(name)
.bind(&checkpoint.bytes)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
for (name, checkpoint) in checkpoints {
self
.indexes
.table(name)?
.index
.write()
.await
.mark_checkpoint_persisted(checkpoint.sequence)?;
}
Ok(())
}
pub async fn commit_indexed_clocks(&self, clocks: &[DocIndexedClock]) -> Result<()> {
let mut checkpoints = Vec::new();
for (name, table) in self.indexes.tables() {
checkpoints.push((name, table.index.read().await.checkpoint()?));
}
let mut tx = self.pool.begin().await?;
for (name, checkpoint) in &checkpoints {
sqlx::query("INSERT OR REPLACE INTO idx_snapshots (index_name, data) VALUES (?, ?)")
.bind(name)
.bind(&checkpoint.bytes)
.execute(&mut *tx)
.await?;
}
for clock in clocks {
sqlx::query(
r#"INSERT INTO indexer_sync (doc_id, indexed_clock, indexer_version)
VALUES ($1, $2, $3)
ON CONFLICT(doc_id)
DO UPDATE SET indexed_clock=$2, indexer_version=$3"#,
)
.bind(&clock.doc_id)
.bind(clock.timestamp)
.bind(clock.indexer_version)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
for (name, checkpoint) in checkpoints {
self
.indexes
.table(name)?
.index
.write()
.await
.mark_checkpoint_persisted(checkpoint.sequence)?;
}
self.indexes.ready.store(true, Ordering::Release);
Ok(())
}
pub fn index_version() -> u32 {
NBSTORE_INDEXER_VERSION
}
pub async fn index_upsert(&self, table: &str, document: NativeIndexDocument) -> Result<()> {
let table = self.indexes.table(table)?;
let mut index_document = Document::new(document.id);
for field in document.fields {
let field_id = table.field(&field.field)?;
index_document.add_values(field_id, field.values.into_iter().map(Value::String));
}
table.index.write().await.upsert(index_document)?;
Ok(())
}
pub async fn index_delete(&self, table: &str, id: &str) -> Result<()> {
self.indexes.table(table)?.index.write().await.delete(id);
Ok(())
}
pub async fn index_search(
&self,
table: &str,
query: NativeIndexQuery,
options: NativeIndexSearchOptions,
) -> Result<NativeIndexSearchResult> {
self.indexes.ensure_ready()?;
let table = self.indexes.table(table)?;
let query = table.compile_query(query)?;
let result = table
.index
.read()
.await
.search(&query, table.compile_options(options)?)?;
Ok(NativeIndexSearchResult {
total: result.total as u32,
hits: result.hits.into_iter().map(|hit| table.hit(hit)).collect(),
})
}
pub async fn index_aggregate(
&self,
table: &str,
query: NativeIndexQuery,
field: &str,
limit: u32,
offset: u32,
hits: Option<NativeIndexSearchOptions>,
) -> Result<NativeIndexAggregateResult> {
self.indexes.ensure_ready()?;
let table = self.indexes.table(table)?;
let query = table.compile_query(query)?;
let result = table.index.read().await.aggregate(
&query,
TermsAggregation {
field: table.field(field)?,
limit: limit as usize,
offset: offset as usize,
top_hits: hits.map(|options| table.compile_options(options)).transpose()?,
},
)?;
Ok(NativeIndexAggregateResult {
total: result.total as u32,
buckets: result
.buckets
.into_iter()
.map(|bucket| NativeIndexBucket {
key: string_values(vec![bucket.key]).pop().unwrap_or_default(),
count: bucket.count as u32,
score: bucket.max_score as f64,
hits: bucket.hits.into_iter().map(|hit| table.hit(hit)).collect(),
})
.collect(),
})
}
pub async fn index_delete_by_query(&self, table: &str, query: NativeIndexQuery) -> Result<u32> {
let table = self.indexes.table(table)?;
let query = table.compile_query(query)?;
Ok(table.index.write().await.delete_by_query(&query)? as u32)
}
}
packages/frontend/native/nbstore/src/indexer/table.rs
+223
View File
@@ -0,0 +1,223 @@
use std::collections::HashMap;
use memory_indexer::{
FieldId, FieldOptions, FieldType, Highlight, MemoryIndex, PositionEncoding, Query, Schema, SearchMode, SearchOptions,
TextOptions, Value,
};
use tokio::sync::RwLock;
use super::{
NativeIndexField, NativeIndexHighlight, NativeIndexHighlightValue, NativeIndexHit, NativeIndexQuery,
NativeIndexSearchOptions, NativeIndexSpan,
error::{Error, Result},
};
pub(crate) struct TableIndex {
pub(super) schema: Schema,
fields: HashMap<String, FieldId>,
pub(super) index: RwLock<MemoryIndex>,
}
impl TableIndex {
pub(super) fn doc() -> Self {
let mut schema = Schema::builder().position_encoding(PositionEncoding::Utf16);
let fields = HashMap::from([
("docId".into(), schema.keyword("docId", FieldOptions::indexed_stored())),
(
"title".into(),
schema.text("title", text_options(), FieldOptions::indexed_stored()),
),
(
"summary".into(),
schema.keyword("summary", FieldOptions::new().stored()),
),
]);
Self::from_schema(schema, fields)
}
pub(super) fn block() -> Self {
let mut schema = Schema::builder().position_encoding(PositionEncoding::Utf16);
let fields = HashMap::from([
("docId".into(), schema.keyword("docId", keyword_options())),
("blockId".into(), schema.keyword("blockId", keyword_options())),
(
"content".into(),
schema.text("content", text_options(), FieldOptions::indexed_stored().multi_value()),
),
("flavour".into(), schema.keyword("flavour", keyword_options())),
("blob".into(), schema.keyword("blob", keyword_options())),
("refDocId".into(), schema.keyword("refDocId", keyword_options())),
(
"ref".into(),
schema.keyword("ref", FieldOptions::new().stored().multi_value()),
),
(
"parentFlavour".into(),
schema.keyword("parentFlavour", keyword_options()),
),
(
"parentBlockId".into(),
schema.keyword("parentBlockId", keyword_options()),
),
(
"additional".into(),
schema.keyword("additional", FieldOptions::new().stored().multi_value()),
),
(
"markdownPreview".into(),
schema.keyword("markdownPreview", FieldOptions::new().stored().multi_value()),
),
]);
Self::from_schema(schema, fields)
}
fn from_schema(builder: memory_indexer::SchemaBuilder, fields: HashMap<String, FieldId>) -> Self {
let schema = builder.build().expect("static nbstore index schema must be valid");
Self {
index: RwLock::new(MemoryIndex::new(schema.clone())),
schema,
fields,
}
}
pub(super) fn field(&self, name: &str) -> Result<FieldId> {
self
.fields
.get(name)
.copied()
.ok_or_else(|| Error::Serialization(format!("unknown index field {name}")))
}
pub(super) fn compile_query(&self, query: NativeIndexQuery) -> Result<Query> {
match query.kind.as_str() {
"match" => {
let field = self.field(query.field.as_deref().unwrap_or_default())?;
let value = query.value.unwrap_or_default();
match self.schema.field(field).map(|field| &field.field_type) {
Some(FieldType::Text(_)) => Ok(Query::text(field, value, SearchMode::Auto)),
Some(FieldType::Keyword) => Ok(Query::term(field, value)),
_ => Err(Error::Serialization("match requires Text or Keyword field".into())),
}
}
"exists" => Ok(Query::Exists(self.field(query.field.as_deref().unwrap_or_default())?)),
"all" => Ok(Query::All),
"boost" => {
let clause = query
.clauses
.unwrap_or_default()
.pop()
.ok_or_else(|| Error::Serialization("boost requires one clause".into()))?;
Ok(Query::Boost {
query: Box::new(self.compile_query(clause)?),
factor: query.boost.unwrap_or(1.0) as f32,
})
}
"boolean" => {
let clauses = query
.clauses
.unwrap_or_default()
.into_iter()
.map(|query| self.compile_query(query))
.collect::<Result<Vec<_>>>()?;
let (must, should, must_not) = match query.occur.as_deref() {
Some("must") => (clauses, vec![], vec![]),
Some("should") => (vec![], clauses, vec![]),
Some("must_not") => (vec![], vec![], clauses),
_ => return Err(Error::Serialization("invalid boolean occurrence".into())),
};
Ok(Query::boolean(must, should, must_not))
}
kind => Err(Error::Serialization(format!("unknown query kind {kind}"))),
}
}
pub(super) fn compile_options(&self, options: NativeIndexSearchOptions) -> Result<SearchOptions> {
Ok(SearchOptions {
limit: options.limit as usize,
offset: options.offset as usize,
after: None,
sort: vec![],
stored_fields: options
.fields
.iter()
.map(|field| self.field(field))
.collect::<Result<Vec<_>>>()?,
highlight_fields: options
.highlights
.iter()
.map(|field| self.field(field))
.collect::<Result<Vec<_>>>()?,
})
}
pub(super) fn hit(&self, hit: memory_indexer::SearchHit) -> NativeIndexHit {
NativeIndexHit {
id: hit.id,
score: hit.score as f64,
fields: hit
.fields
.into_iter()
.map(|(field, values)| NativeIndexField {
field: self.field_name(field),
values: string_values(values),
})
.collect(),
highlights: group_highlights(self, hit.highlights),
}
}
fn field_name(&self, field: FieldId) -> String {
self
.schema
.field(field)
.expect("result field belongs to schema")
.name
.clone()
}
}
pub(crate) fn string_values(values: Vec<Value>) -> Vec<String> {
values
.into_iter()
.filter_map(|value| match value {
Value::String(value) => Some(value),
Value::I64(_) | Value::Bool(_) => None,
})
.collect()
}
fn keyword_options() -> FieldOptions {
FieldOptions::indexed_stored().multi_value()
}
fn text_options() -> TextOptions {
TextOptions::multilingual()
.with_pinyin()
.with_prefix()
.with_fuzzy()
.with_positions()
}
fn group_highlights(table: &TableIndex, highlights: Vec<Highlight>) -> Vec<NativeIndexHighlight> {
let mut grouped: Vec<NativeIndexHighlight> = Vec::new();
for highlight in highlights {
let field = table.field_name(highlight.field);
let values = NativeIndexHighlightValue {
value_index: highlight.value_index as u32,
spans: highlight
.spans
.into_iter()
.map(|(start, end)| NativeIndexSpan { start, end })
.collect(),
};
if let Some(existing) = grouped.iter_mut().find(|item| item.field == field) {
existing.values.push(values);
} else {
grouped.push(NativeIndexHighlight {
field,
values: vec![values],
});
}
}
grouped
}
packages/frontend/native/nbstore/src/indexer/tests.rs
+318
View File
@@ -0,0 +1,318 @@
use std::path::{Path, PathBuf};
use affine_doc_loader::ParseError;
use assert_json_diff::assert_json_eq;
use chrono::Utc;
use serde_json::Value;
use tokio::fs;
use uuid::Uuid;
use super::{
super::{DocIndexedClock, error::Error, storage::SqliteDocStorage},
NativeIndexDocument, NativeIndexField, NativeIndexQuery, NativeIndexSearchOptions,
};
const DEMO_BIN: &[u8] = include_bytes!("../../../../../common/native/fixtures/demo.ydoc");
const DEMO_JSON: &[u8] = include_bytes!("../../../../../common/native/fixtures/demo.ydoc.json");
fn temp_workspace_dir() -> PathBuf {
std::env::temp_dir().join(format!("affine-native-{}", Uuid::new_v4()))
}
async fn init_db(path: &Path) -> SqliteDocStorage {
fs::create_dir_all(path.parent().unwrap()).await.unwrap();
let storage = SqliteDocStorage::new(path.to_string_lossy().into_owned());
storage.connect().await.unwrap();
storage
}
async fn cleanup(path: &Path) {
let _ = fs::remove_dir_all(path.parent().unwrap()).await;
}
fn query(kind: &str, field: Option<&str>, value: Option<&str>) -> NativeIndexQuery {
NativeIndexQuery {
kind: kind.into(),
field: field.map(Into::into),
value: value.map(Into::into),
occur: None,
clauses: None,
boost: None,
}
}
fn options(fields: &[&str], highlights: &[&str]) -> NativeIndexSearchOptions {
NativeIndexSearchOptions {
limit: 10,
offset: 0,
fields: fields.iter().map(|value| (*value).into()).collect(),
highlights: highlights.iter().map(|value| (*value).into()).collect(),
}
}
fn document(id: &str, fields: &[(&str, &[&str])]) -> NativeIndexDocument {
NativeIndexDocument {
id: id.into(),
fields: fields
.iter()
.map(|(field, values)| NativeIndexField {
field: (*field).into(),
values: values.iter().map(|value| (*value).into()).collect(),
})
.collect(),
}
}
#[tokio::test]
async fn parse_demo_snapshot_matches_fixture() {
let base = temp_workspace_dir();
fs::create_dir_all(&base).await.unwrap();
let db_path = base.join("storage.db");
let storage = init_db(&db_path).await;
sqlx::query(r#"INSERT INTO snapshots (doc_id, data, updated_at) VALUES (?, ?, ?)"#)
.bind("demo-doc")
.bind(DEMO_BIN)
.bind(Utc::now().naive_utc())
.execute(&storage.pool)
.await
.unwrap();
sqlx::query(r#"INSERT INTO updates (doc_id, data, created_at) VALUES (?, ?, ?)"#)
.bind("demo-doc")
.bind(&[0, 0][..])
.bind(Utc::now().naive_utc())
.execute(&storage.pool)
.await
.unwrap();
let result = storage.crawl_doc_data("demo-doc").await.unwrap();
let mut expected: Value = serde_json::from_slice(DEMO_JSON).unwrap();
let mut actual = serde_json::to_value(&result).unwrap();
for document in [&mut expected, &mut actual] {
for block in document["blocks"].as_array_mut().unwrap() {
if let Some(additional) = block["additional"].as_str() {
block["additional"] = serde_json::from_str(additional).unwrap();
}
}
}
assert_json_eq!(expected, actual);
storage.close().await;
cleanup(&db_path).await;
}
#[tokio::test]
async fn missing_doc_returns_error() {
let db_path = temp_workspace_dir().join("storage.db");
let storage = init_db(&db_path).await;
let error = storage.crawl_doc_data("absent-doc").await.unwrap_err();
assert!(matches!(error, Error::Parse(ParseError::DocNotFound)));
storage.close().await;
cleanup(&db_path).await;
}
#[tokio::test]
async fn index_tables_support_terminal_queries_and_restart() {
let db_path = temp_workspace_dir().join("storage.db");
let storage = init_db(&db_path).await;
storage
.index_upsert(
"doc",
document(
"doc-1",
&[
("docId", &["doc-1"]),
("title", &["Rust 搜索"]),
("summary", &["stored summary"]),
],
),
)
.await
.unwrap();
storage
.index_upsert(
"block",
document(
"block-1",
&[
("docId", &["doc-1"]),
("blockId", &["block-1"]),
("content", &["hello world", "你好搜索"]),
("flavour", &["affine:paragraph"]),
],
),
)
.await
.unwrap();
storage
.index_upsert(
"block",
document(
"block-2",
&[
("docId", &["doc-10"]),
("blockId", &["block-2"]),
("content", &["hello unrelated"]),
("flavour", &["affine:code"]),
],
),
)
.await
.unwrap();
let text = storage
.index_search(
"block",
query("match", Some("content"), Some("搜索")),
options(&["content"], &["content"]),
)
.await
.unwrap();
assert_eq!(text.total, 1);
assert_eq!(text.hits[0].id, "block-1");
assert!(!text.hits[0].highlights[0].values[0].spans.is_empty());
let exact = storage
.index_search(
"block",
query("match", Some("docId"), Some("doc-1")),
options(&["docId"], &[]),
)
.await
.unwrap();
assert_eq!(exact.total, 1);
assert_eq!(exact.hits[0].fields[0].values, ["doc-1"]);
let aggregate = storage
.index_aggregate(
"block",
query("all", None, None),
"flavour",
10,
0,
Some(options(&["blockId"], &[])),
)
.await
.unwrap();
assert_eq!(aggregate.total, 2);
assert_eq!(aggregate.buckets.len(), 2);
let clock = DocIndexedClock {
doc_id: "doc-1".into(),
timestamp: Utc::now().naive_utc(),
indexer_version: SqliteDocStorage::index_version() as i64,
};
storage.commit_indexed_clocks(&[clock]).await.unwrap();
storage.close().await;
let restored = init_db(&db_path).await;
let result = restored
.index_search(
"block",
query("match", Some("content"), Some("hello")),
options(&[], &[]),
)
.await
.unwrap();
assert_eq!(result.total, 2);
assert_eq!(
restored
.index_delete_by_query("block", query("match", Some("docId"), Some("doc-1")))
.await
.unwrap(),
1
);
restored.close().await;
cleanup(&db_path).await;
}
#[tokio::test]
async fn corrupt_checkpoint_requires_rebuild_and_clears_clocks() {
let db_path = temp_workspace_dir().join("storage.db");
let storage = init_db(&db_path).await;
storage
.index_upsert(
"doc",
document("doc-1", &[("docId", &["doc-1"]), ("title", &["title"])]),
)
.await
.unwrap();
let clock = DocIndexedClock {
doc_id: "doc-1".into(),
timestamp: Utc::now().naive_utc(),
indexer_version: SqliteDocStorage::index_version() as i64,
};
storage.commit_indexed_clocks(&[clock]).await.unwrap();
sqlx::query("UPDATE idx_snapshots SET data = x'00' WHERE index_name = 'doc'")
.execute(&storage.pool)
.await
.unwrap();
storage.close().await;
let restored = init_db(&db_path).await;
assert!(matches!(
restored
.index_search("doc", query("all", None, None), options(&[], &[]))
.await,
Err(Error::IndexNotReady)
));
assert!(restored.get_doc_indexed_clock("doc-1".into()).await.unwrap().is_none());
restored
.index_upsert(
"doc",
document("doc-1", &[("docId", &["doc-1"]), ("title", &["rebuilt"])]),
)
.await
.unwrap();
restored
.commit_indexed_clocks(&[DocIndexedClock {
doc_id: "doc-1".into(),
timestamp: Utc::now().naive_utc(),
indexer_version: SqliteDocStorage::index_version() as i64,
}])
.await
.unwrap();
assert_eq!(
restored
.index_search("doc", query("all", None, None), options(&[], &[]))
.await
.unwrap()
.total,
1
);
restored.close().await;
cleanup(&db_path).await;
}
#[tokio::test]
async fn failed_atomic_commit_keeps_index_dirty_and_clock_unadvanced() {
let db_path = temp_workspace_dir().join("storage.db");
let storage = init_db(&db_path).await;
storage
.index_upsert(
"doc",
document("doc-1", &[("docId", &["doc-1"]), ("title", &["title"])]),
)
.await
.unwrap();
sqlx::query("DROP TABLE idx_snapshots")
.execute(&storage.pool)
.await
.unwrap();
let clock = DocIndexedClock {
doc_id: "doc-1".into(),
timestamp: Utc::now().naive_utc(),
indexer_version: SqliteDocStorage::index_version() as i64,
};
assert!(storage.commit_indexed_clocks(&[clock]).await.is_err());
assert!(
storage
.indexes
.table("doc")
.unwrap()
.index
.read()
.await
.has_unpersisted_changes()
);
assert!(storage.get_doc_indexed_clock("doc-1".into()).await.unwrap().is_none());
storage.close().await;
cleanup(&db_path).await;
}
packages/frontend/native/nbstore/src/lib.rs
+39 -26
View File
@@ -220,6 +220,12 @@ impl DocStoragePool {
Ok(())
}
#[napi]
pub async fn set_doc_indexed_clocks(&self, universal_id: String, clocks: Vec<DocIndexedClock>) -> Result<()> {
self.get(universal_id).await?.commit_indexed_clocks(&clocks).await?;
Ok(())
}
#[napi]
pub async fn clear_doc_indexed_clock(&self, universal_id: String, doc_id: String) -> Result<()> {
self.get(universal_id).await?.clear_doc_indexed_clock(doc_id).await?;
@@ -410,65 +416,72 @@ impl DocStoragePool {
}
#[napi]
pub async fn fts_add_document(
&self,
id: String,
index_name: String,
doc_id: String,
text: String,
index: bool,
) -> Result<()> {
pub async fn index_upsert(&self, id: String, table: String, document: indexer::NativeIndexDocument) -> Result<()> {
let storage = self.pool.get(id).await?;
storage.fts_add(&index_name, &doc_id, &text, index).await?;
storage.index_upsert(&table, document).await?;
Ok(())
}
#[napi]
pub async fn fts_flush_index(&self, id: String) -> Result<()> {
pub async fn index_flush(&self, id: String) -> Result<()> {
let storage = self.pool.get(id).await?;
storage.flush_index().await?;
Ok(())
}
#[napi]
pub async fn fts_index_version(&self) -> Result<u32> {
pub async fn index_version(&self) -> Result<u32> {
Ok(SqliteDocStorage::index_version())
}
#[napi]
pub async fn fts_delete_document(&self, id: String, index_name: String, doc_id: String) -> Result<()> {
pub async fn index_delete(&self, id: String, table: String, doc_id: String) -> Result<()> {
let storage = self.pool.get(id).await?;
storage.fts_delete(&index_name, &doc_id).await?;
storage.index_delete(&table, &doc_id).await?;
Ok(())
}
#[napi]
pub async fn fts_get_document(&self, id: String, index_name: String, doc_id: String) -> Result<Option<String>> {
pub async fn index_search(
&self,
id: String,
table: String,
query: indexer::NativeIndexQuery,
options: indexer::NativeIndexSearchOptions,
) -> Result<indexer::NativeIndexSearchResult> {
let storage = self.pool.get(id).await?;
Ok(storage.fts_get(&index_name, &doc_id).await?)
Ok(storage.index_search(&table, query, options).await?)
}
#[napi]
pub async fn fts_search(
#[allow(clippy::too_many_arguments)]
pub async fn index_aggregate(
&self,
id: String,
index_name: String,
query: String,
) -> Result<Vec<indexer::NativeSearchHit>> {
table: String,
query: indexer::NativeIndexQuery,
field: String,
limit: u32,
offset: u32,
hits: Option<indexer::NativeIndexSearchOptions>,
) -> Result<indexer::NativeIndexAggregateResult> {
let storage = self.pool.get(id).await?;
Ok(storage.fts_search(&index_name, &query).await?)
Ok(
storage
.index_aggregate(&table, query, &field, limit, offset, hits)
.await?,
)
}
#[napi]
pub async fn fts_get_matches(
pub async fn index_delete_by_query(
&self,
id: String,
index_name: String,
doc_id: String,
query: String,
) -> Result<Vec<indexer::NativeMatch>> {
table: String,
query: indexer::NativeIndexQuery,
) -> Result<u32> {
let storage = self.pool.get(id).await?;
Ok(storage.fts_get_matches(&index_name, &doc_id, &query).await?)
Ok(storage.index_delete_by_query(&table, query).await?)
}
}
packages/frontend/native/nbstore/src/storage.rs
+5 -7
View File
@@ -4,20 +4,18 @@ use affine_schema::{
get_migrator,
import_validation::{V2_IMPORT_SCHEMA_RULES, validate_import_schema, validate_required_schema},
};
use memory_indexer::InMemoryIndex;
use sqlx::{
Pool, Row,
migrate::{MigrateDatabase, Migration, Migrator},
sqlite::{Sqlite, SqliteConnectOptions, SqlitePoolOptions},
};
use tokio::sync::RwLock;
use super::error::Result;
use super::{error::Result, indexer::IndexManager};
pub struct SqliteDocStorage {
pub pool: Pool<Sqlite>,
path: String,
pub index: Arc<RwLock<InMemoryIndex>>,
pub indexes: Arc<IndexManager>,
}
impl SqliteDocStorage {
@@ -26,7 +24,7 @@ impl SqliteDocStorage {
let mut pool_options = SqlitePoolOptions::new();
let index = Arc::new(RwLock::new(InMemoryIndex::default()));
let indexes = Arc::new(IndexManager::new());
if path == ":memory:" {
pool_options = pool_options
@@ -38,7 +36,7 @@ impl SqliteDocStorage {
Self {
pool: pool_options.connect_lazy_with(sqlite_options),
path,
index,
indexes,
}
} else {
Self {
@@ -46,7 +44,7 @@ impl SqliteDocStorage {
.max_connections(4)
.connect_lazy_with(sqlite_options.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)),
path,
index,
indexes,
}
}
}