mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-21 03:56:23 +08:00
refactor: jwt internal version migration
This commit is contained in:
DarkSky
@@ -0,0 +1,66 @@
|
||||
import { Array as YArray, Map as YMap } from 'yjs';
|
||||
|
||||
import { RemoteKvService } from '@toeverything/datasource/remote-kv';
|
||||
|
||||
export class YjsRemoteBinaries {
|
||||
private readonly _binaries: YMap<YArray<ArrayBuffer>>; // binary instance
|
||||
private readonly _remoteStorage?: RemoteKvService;
|
||||
|
||||
constructor(binaries: YMap<YArray<ArrayBuffer>>, remote_token?: string) {
|
||||
this._binaries = binaries;
|
||||
if (remote_token) {
|
||||
this._remoteStorage = new RemoteKvService(remote_token);
|
||||
} else {
|
||||
console.warn(`Remote storage is not ready`);
|
||||
}
|
||||
}
|
||||
|
||||
has(name: string): boolean {
|
||||
return this._binaries.has(name);
|
||||
}
|
||||
|
||||
async get(name: string): Promise<YArray<ArrayBuffer> | undefined> {
|
||||
if (this._binaries.has(name)) {
|
||||
return this._binaries.get(name);
|
||||
} else {
|
||||
// TODO: Remote Load
|
||||
try {
|
||||
const file = await this._remoteStorage?.instance?.getBuffData(
|
||||
name
|
||||
);
|
||||
console.log(file);
|
||||
// return file;
|
||||
} catch (e) {
|
||||
throw new Error(`Binary ${name} not found`);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async set(name: string, binary: YArray<ArrayBuffer>) {
|
||||
if (!this._binaries.has(name)) {
|
||||
console.log(name, 'name');
|
||||
if (binary.length === 1) {
|
||||
this._binaries.set(name, binary);
|
||||
if (this._remoteStorage) {
|
||||
// TODO: Remote Save, if there is an object with the same name remotely, the upload is skipped, because the file name is the hash of the file content
|
||||
const has_file = this._remoteStorage.instance?.exist(name);
|
||||
if (!has_file) {
|
||||
const upload_file = new File(binary.toArray(), name);
|
||||
await this._remoteStorage.instance
|
||||
?.upload(upload_file)
|
||||
.catch(err => {
|
||||
throw new Error(`${err} upload error`);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
console.warn(`Remote storage is not ready`);
|
||||
}
|
||||
return;
|
||||
} else {
|
||||
console.log('err');
|
||||
}
|
||||
throw new Error(`Binary ${name} is invalid`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
import {
|
||||
AbstractType as YAbstractType,
|
||||
Array as YArray,
|
||||
Map as YMap,
|
||||
transact,
|
||||
} from 'yjs';
|
||||
|
||||
import { BlockItem, BlockTypes } from '../types';
|
||||
import { BlockInstance, BlockListener, HistoryManager } from './types';
|
||||
|
||||
import { YjsHistoryManager } from './history';
|
||||
import { ChildrenListenerHandler, ContentListenerHandler } from './listener';
|
||||
import { YjsContentOperation } from './operation';
|
||||
|
||||
const GET_BLOCK_ITEM = Symbol('GET_BLOCK_ITEM');
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
const getMapFromYArray = (array: YArray<string>) =>
|
||||
new Map(array.map((child, index) => [child, index]));
|
||||
|
||||
type YjsBlockInstanceProps = {
|
||||
id: string;
|
||||
block: YMap<unknown>;
|
||||
binary?: YArray<ArrayBuffer>;
|
||||
setBlock: (
|
||||
id: string,
|
||||
block: BlockItem<YjsContentOperation>
|
||||
) => Promise<void>;
|
||||
getUpdated: (id: string) => number | undefined;
|
||||
getCreator: (id: string) => string | undefined;
|
||||
getBlockInstance: (id: string) => YjsBlockInstance | undefined;
|
||||
};
|
||||
|
||||
export class YjsBlockInstance implements BlockInstance<YjsContentOperation> {
|
||||
private readonly _id: string;
|
||||
private readonly _block: YMap<unknown>;
|
||||
private readonly _binary: YArray<ArrayBuffer> | undefined;
|
||||
private readonly _children: YArray<string>;
|
||||
private readonly _setBlock: (
|
||||
id: string,
|
||||
block: BlockItem<YjsContentOperation>
|
||||
) => Promise<void>;
|
||||
private readonly _getUpdated: (id: string) => number | undefined;
|
||||
private readonly _getCreator: (id: string) => string | undefined;
|
||||
private readonly _getBlockInstance: (
|
||||
id: string
|
||||
) => YjsBlockInstance | undefined;
|
||||
private readonly _childrenListeners: Map<string, BlockListener>;
|
||||
private readonly _contentListeners: Map<string, BlockListener>;
|
||||
|
||||
private _childrenMap: Map<string, number>;
|
||||
|
||||
constructor(props: YjsBlockInstanceProps) {
|
||||
this._id = props.id;
|
||||
this._block = props.block;
|
||||
this._binary = props.binary;
|
||||
|
||||
this._children = props.block.get('children') as YArray<string>;
|
||||
this._childrenMap = getMapFromYArray(this._children);
|
||||
this._setBlock = props.setBlock;
|
||||
this._getUpdated = props.getUpdated;
|
||||
this._getCreator = props.getCreator;
|
||||
this._getBlockInstance = props.getBlockInstance;
|
||||
|
||||
this._childrenListeners = new Map();
|
||||
this._contentListeners = new Map();
|
||||
|
||||
const content = this._block.get('content') as YMap<unknown>;
|
||||
|
||||
this._children.observe(event =>
|
||||
ChildrenListenerHandler(this._childrenListeners, event)
|
||||
);
|
||||
content?.observeDeep(events =>
|
||||
ContentListenerHandler(this._contentListeners, events)
|
||||
);
|
||||
// TODO: flavor needs optimization
|
||||
this._block.observeDeep(events =>
|
||||
ContentListenerHandler(this._contentListeners, events)
|
||||
);
|
||||
}
|
||||
|
||||
on(
|
||||
key: 'children' | 'content',
|
||||
name: string,
|
||||
listener: BlockListener
|
||||
): void {
|
||||
if (key === 'children') {
|
||||
this.addChildrenListener(name, listener);
|
||||
} else if (key === 'content') {
|
||||
this.addContentListener(name, listener);
|
||||
}
|
||||
}
|
||||
|
||||
off(key: 'children' | 'content', name: string): void {
|
||||
if (key === 'children') {
|
||||
this.removeChildrenListener(name);
|
||||
} else if (key === 'content') {
|
||||
this.removeContentListener(name);
|
||||
}
|
||||
}
|
||||
|
||||
addChildrenListener(name: string, listener: BlockListener): void {
|
||||
this._childrenListeners.set(name, listener);
|
||||
}
|
||||
|
||||
removeChildrenListener(name: string): void {
|
||||
this._childrenListeners.delete(name);
|
||||
}
|
||||
|
||||
addContentListener(name: string, listener: BlockListener): void {
|
||||
this._contentListeners.set(name, listener);
|
||||
}
|
||||
|
||||
removeContentListener(name: string): void {
|
||||
this._contentListeners.delete(name);
|
||||
}
|
||||
|
||||
get id() {
|
||||
return this._id;
|
||||
}
|
||||
|
||||
get content(): YjsContentOperation {
|
||||
if (this.type === BlockTypes.block) {
|
||||
const content = this._block.get('content');
|
||||
if (content instanceof YAbstractType) {
|
||||
return new YjsContentOperation(content);
|
||||
} else {
|
||||
throw new Error(`Invalid content type: ${typeof content}`);
|
||||
}
|
||||
} else if (this.type === BlockTypes.binary && this._binary) {
|
||||
return new YjsContentOperation(this._binary);
|
||||
}
|
||||
throw new Error(
|
||||
`Invalid content type: ${this.type}, ${this._block.get(
|
||||
'content'
|
||||
)}, ${this._binary}`
|
||||
);
|
||||
}
|
||||
|
||||
get type(): BlockItem<YjsContentOperation>['type'] {
|
||||
return this._block.get(
|
||||
'type'
|
||||
) as BlockItem<YjsContentOperation>['type'];
|
||||
}
|
||||
|
||||
get flavor(): BlockItem<YjsContentOperation>['flavor'] {
|
||||
return this._block.get(
|
||||
'flavor'
|
||||
) as BlockItem<YjsContentOperation>['flavor'];
|
||||
}
|
||||
|
||||
// TODO: bad case. Need to optimize.
|
||||
setFlavor(flavor: BlockItem<YjsContentOperation>['flavor']) {
|
||||
this._block.set('flavor', flavor);
|
||||
}
|
||||
|
||||
get created(): BlockItem<YjsContentOperation>['created'] {
|
||||
return this._block.get(
|
||||
'created'
|
||||
) as BlockItem<YjsContentOperation>['created'];
|
||||
}
|
||||
|
||||
get updated(): number {
|
||||
return this._getUpdated(this._id) || this.created;
|
||||
}
|
||||
|
||||
get creator(): string | undefined {
|
||||
return this._getCreator(this._id);
|
||||
}
|
||||
|
||||
get children(): string[] {
|
||||
return this._children.toArray();
|
||||
}
|
||||
|
||||
getChildren(ids?: (string | undefined)[]): YjsBlockInstance[] {
|
||||
const query_ids = ids?.filter((id): id is string => !!id) || [];
|
||||
const exists_ids = this._children.map(id => id);
|
||||
const filter_ids = query_ids.length ? query_ids : exists_ids;
|
||||
return exists_ids
|
||||
.filter(id => filter_ids.includes(id))
|
||||
.map(id => this._getBlockInstance(id))
|
||||
.filter((v): v is YjsBlockInstance => !!v);
|
||||
}
|
||||
|
||||
hasChildren(id: string): boolean {
|
||||
if (this.children.includes(id)) {
|
||||
return true;
|
||||
}
|
||||
return this.getChildren().some(block => block.hasChildren(id));
|
||||
}
|
||||
|
||||
private position_calculator(
|
||||
max_pos: number,
|
||||
position?: { pos?: number; before?: string; after?: string }
|
||||
) {
|
||||
const { pos, before, after } = position || {};
|
||||
if (typeof pos === 'number' && Number.isInteger(pos)) {
|
||||
if (pos >= 0 && pos < max_pos) {
|
||||
return pos;
|
||||
}
|
||||
} else if (before) {
|
||||
const current_pos = this._childrenMap.get(before || '');
|
||||
if (
|
||||
typeof current_pos === 'number' &&
|
||||
Number.isInteger(current_pos)
|
||||
) {
|
||||
const prev_pos = current_pos;
|
||||
if (prev_pos >= 0 && prev_pos < max_pos) {
|
||||
return prev_pos;
|
||||
}
|
||||
}
|
||||
} else if (after) {
|
||||
const current_pos = this._childrenMap.get(after || '');
|
||||
if (
|
||||
typeof current_pos === 'number' &&
|
||||
Number.isInteger(current_pos)
|
||||
) {
|
||||
const next_pos = current_pos + 1;
|
||||
if (next_pos >= 0 && next_pos < max_pos) {
|
||||
return next_pos;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async insertChildren(
|
||||
block: YjsBlockInstance,
|
||||
pos?: { pos?: number; before?: string; after?: string }
|
||||
): Promise<void> {
|
||||
const content = block[GET_BLOCK_ITEM]();
|
||||
if (content) {
|
||||
const lastIndex = this._childrenMap.get(block.id);
|
||||
if (typeof lastIndex === 'number') {
|
||||
this._children.delete(lastIndex);
|
||||
this._childrenMap = getMapFromYArray(this._children);
|
||||
}
|
||||
|
||||
const position = this.position_calculator(
|
||||
this._childrenMap.size,
|
||||
pos
|
||||
);
|
||||
if (typeof position === 'number') {
|
||||
this._children.insert(position, [block.id]);
|
||||
} else {
|
||||
this._children.push([block.id]);
|
||||
}
|
||||
await this._setBlock(block.id, content);
|
||||
this._childrenMap = getMapFromYArray(this._children);
|
||||
}
|
||||
}
|
||||
|
||||
removeChildren(ids: (string | undefined)[]): Promise<string[]> {
|
||||
return new Promise(resolve => {
|
||||
if (this._children.doc) {
|
||||
transact(this._children.doc, () => {
|
||||
const failed = [];
|
||||
for (const id of ids) {
|
||||
let idx = -1;
|
||||
for (const block_id of this._children) {
|
||||
idx += 1;
|
||||
if (block_id === id) {
|
||||
this._children.delete(idx);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (id) {
|
||||
failed.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
this._childrenMap = getMapFromYArray(this._children);
|
||||
resolve(failed);
|
||||
});
|
||||
} else {
|
||||
resolve(ids.filter((id): id is string => !!id));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public scopedHistory(scope: any[]): HistoryManager {
|
||||
return new YjsHistoryManager(this._block, scope);
|
||||
}
|
||||
|
||||
[GET_BLOCK_ITEM]() {
|
||||
// check null & undefined
|
||||
if (this.content != null) {
|
||||
return {
|
||||
type: this.type,
|
||||
flavor: this.flavor,
|
||||
children: this._children.slice(),
|
||||
created: this.created,
|
||||
content: this.content,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { Map as YMap } from 'yjs';
|
||||
|
||||
export class GateKeeper {
|
||||
private readonly _userId: string;
|
||||
private readonly _creators: YMap<string>;
|
||||
private readonly _common: YMap<string>;
|
||||
|
||||
constructor(userId: string, creators: YMap<string>, common: YMap<string>) {
|
||||
this._userId = userId;
|
||||
this._creators = creators;
|
||||
this._common = common;
|
||||
}
|
||||
|
||||
getCreator(block_id: string): string | undefined {
|
||||
return this._creators.get(block_id) || this._common.get(block_id);
|
||||
}
|
||||
|
||||
setCreator(block_id: string) {
|
||||
if (!this._creators.get(block_id)) {
|
||||
this._creators.set(block_id, this._userId);
|
||||
}
|
||||
}
|
||||
|
||||
setCommon(block_id: string) {
|
||||
if (!this._creators.get(block_id) && !this._common.get(block_id)) {
|
||||
this._common.set(block_id, this._userId);
|
||||
}
|
||||
}
|
||||
|
||||
private check_delete(block_id: string): boolean {
|
||||
const creator = this._creators.get(block_id);
|
||||
return creator === this._userId || !!this._common.get(block_id);
|
||||
}
|
||||
|
||||
checkDeleteLists(block_ids: string[]): [string[], string[]] {
|
||||
const success = [];
|
||||
const fail = [];
|
||||
for (const block_id of block_ids) {
|
||||
if (this.check_delete(block_id)) {
|
||||
success.push(block_id);
|
||||
} else {
|
||||
fail.push(block_id);
|
||||
}
|
||||
}
|
||||
return [success, fail];
|
||||
}
|
||||
|
||||
clear() {
|
||||
this._creators.clear();
|
||||
this._common.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { Map as YMap, UndoManager } from 'yjs';
|
||||
|
||||
import { HistoryCallback, HistoryManager } from './types';
|
||||
|
||||
type StackItem = UndoManager['undoStack'][0];
|
||||
|
||||
export class YjsHistoryManager implements HistoryManager {
|
||||
// @ts-ignore
|
||||
private readonly _blocks: YMap<any>;
|
||||
private readonly _historyManager: UndoManager;
|
||||
private readonly _pushListeners: Map<string, HistoryCallback<any>>;
|
||||
private readonly _popListeners: Map<string, HistoryCallback<any>>;
|
||||
|
||||
constructor(scope: YMap<any>, tracker?: any[]) {
|
||||
this._blocks = scope;
|
||||
this._historyManager = new UndoManager(scope, {
|
||||
trackedOrigins: tracker ? new Set(tracker) : undefined,
|
||||
});
|
||||
|
||||
this._pushListeners = new Map();
|
||||
this._historyManager.on(
|
||||
'stack-item-added',
|
||||
(event: { stackItem: StackItem }) => {
|
||||
const meta = event.stackItem.meta;
|
||||
for (const listener of this._pushListeners.values()) {
|
||||
listener(meta);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
this._popListeners = new Map();
|
||||
this._historyManager.on(
|
||||
'stack-item-popped',
|
||||
(event: { stackItem: StackItem }) => {
|
||||
const meta = event.stackItem.meta;
|
||||
for (const listener of this._popListeners.values()) {
|
||||
listener(new Map(meta));
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
onPush<T = unknown>(name: string, callback: HistoryCallback<T>): void {
|
||||
this._pushListeners.set(name, callback);
|
||||
}
|
||||
|
||||
offPush(name: string): boolean {
|
||||
return this._pushListeners.delete(name);
|
||||
}
|
||||
|
||||
onPop<T = unknown>(name: string, callback: HistoryCallback<T>): void {
|
||||
this._popListeners.set(name, callback);
|
||||
}
|
||||
|
||||
offPop(name: string): boolean {
|
||||
return this._popListeners.delete(name);
|
||||
}
|
||||
|
||||
break(): void {
|
||||
// this._historyManager.
|
||||
}
|
||||
|
||||
undo<T = unknown>(): Map<string, T> | undefined {
|
||||
return this._historyManager.undo()?.meta;
|
||||
}
|
||||
|
||||
redo<T = unknown>(): Map<string, T> | undefined {
|
||||
return this._historyManager.redo()?.meta;
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
return this._historyManager.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,640 @@
|
||||
/* eslint-disable max-lines */
|
||||
/* eslint-disable @typescript-eslint/no-unused-vars */
|
||||
/// <reference types="wicg-file-system-access" />
|
||||
import { Buffer } from 'buffer';
|
||||
import { saveAs } from 'file-saver';
|
||||
import { fromEvent } from 'file-selector';
|
||||
import LRUCache from 'lru-cache';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { debounce } from 'ts-debounce';
|
||||
import { Awareness } from 'y-protocols/awareness.js';
|
||||
import {
|
||||
applyUpdate,
|
||||
Array as YArray,
|
||||
Doc,
|
||||
encodeStateAsUpdate,
|
||||
Map as YMap,
|
||||
snapshot,
|
||||
transact,
|
||||
} from 'yjs';
|
||||
|
||||
import { BlockItem, BlockTypes } from '../types';
|
||||
import { getLogger, sha3, sleep } from '../utils';
|
||||
import {
|
||||
AsyncDatabaseAdapter,
|
||||
BlockListener,
|
||||
ChangedStateKeys,
|
||||
Connectivity,
|
||||
HistoryManager,
|
||||
} from './types';
|
||||
|
||||
import { YjsRemoteBinaries } from './binary';
|
||||
import { YjsBlockInstance } from './block';
|
||||
import { GateKeeper } from './gatekeeper';
|
||||
import { YjsHistoryManager } from './history';
|
||||
import { EmitEvents, Suspend } from './listener';
|
||||
import {
|
||||
DO_NOT_USE_THIS_OR_YOU_WILL_BE_FIRED_SYMBOL_INTO_INNER as INTO_INNER,
|
||||
YjsContentOperation,
|
||||
} from './operation';
|
||||
import { YjsProvider } from './provider';
|
||||
|
||||
declare const JWT_DEV: boolean;
|
||||
// @ts-ignore
|
||||
const logger = getLogger('BlockDB:yjs');
|
||||
|
||||
type ConnectivityListener = (
|
||||
workspace: string,
|
||||
connectivity: Connectivity
|
||||
) => void;
|
||||
|
||||
type YjsProviders = {
|
||||
awareness: Awareness;
|
||||
binaries: Doc;
|
||||
doc: Doc;
|
||||
gatekeeper: GateKeeper;
|
||||
connListener: { listeners?: ConnectivityListener };
|
||||
userId: string;
|
||||
remoteToken: string | undefined; // remote storage token
|
||||
};
|
||||
|
||||
const _yjsDatabaseInstance = new Map<string, YjsProviders>();
|
||||
|
||||
const _asyncInitLoading = new Set<string>();
|
||||
const _waitLoading = async (workspace: string) => {
|
||||
while (_asyncInitLoading.has(workspace)) {
|
||||
await sleep();
|
||||
}
|
||||
};
|
||||
|
||||
async function _initYjsDatabase(
|
||||
workspace: string,
|
||||
options: {
|
||||
userId: string;
|
||||
token?: string | undefined;
|
||||
provider?: Record<string, YjsProvider> | undefined;
|
||||
}
|
||||
): Promise<YjsProviders> {
|
||||
if (_asyncInitLoading.has(workspace)) {
|
||||
await _waitLoading(workspace);
|
||||
}
|
||||
const instance = _yjsDatabaseInstance.get(workspace);
|
||||
// TODO: temporarily handle this
|
||||
if (
|
||||
instance &&
|
||||
(instance.userId === options.userId || options.userId === 'default')
|
||||
) {
|
||||
return instance;
|
||||
}
|
||||
// if (instance) return instance;
|
||||
_asyncInitLoading.add(workspace);
|
||||
const { userId, token } = options;
|
||||
|
||||
const doc = new Doc({ autoLoad: true, shouldLoad: true });
|
||||
// const idb = await new IndexedDBProvider(workspace, doc).whenSynced;
|
||||
|
||||
const binaries = new Doc({ autoLoad: true, shouldLoad: true });
|
||||
|
||||
const awareness = new Awareness(doc);
|
||||
|
||||
const gateKeeperData = doc.getMap<YMap<string>>('gatekeeper');
|
||||
|
||||
const gatekeeper = new GateKeeper(
|
||||
userId,
|
||||
gateKeeperData.get('creators') ||
|
||||
gateKeeperData.set('creators', new YMap()),
|
||||
gateKeeperData.get('common') || gateKeeperData.set('common', new YMap())
|
||||
);
|
||||
|
||||
const connListener: { listeners?: ConnectivityListener } = {};
|
||||
if (options.provider) {
|
||||
const emitState = (c: Connectivity) =>
|
||||
connListener.listeners?.(workspace, c);
|
||||
await Promise.all(
|
||||
Object.entries(options.provider).flatMap(([, p]) => [
|
||||
p({ awareness, doc, token, workspace, emitState }),
|
||||
p({
|
||||
awareness,
|
||||
doc: binaries,
|
||||
token,
|
||||
workspace: `${workspace}_binaries`,
|
||||
emitState,
|
||||
}),
|
||||
])
|
||||
);
|
||||
}
|
||||
const newInstance = {
|
||||
awareness,
|
||||
binaries,
|
||||
doc,
|
||||
gatekeeper,
|
||||
|
||||
connListener,
|
||||
userId,
|
||||
remoteToken: token,
|
||||
};
|
||||
|
||||
_yjsDatabaseInstance.set(workspace, newInstance);
|
||||
|
||||
_asyncInitLoading.delete(workspace);
|
||||
|
||||
return newInstance;
|
||||
}
|
||||
|
||||
export type { YjsBlockInstance } from './block';
|
||||
export type { YjsContentOperation } from './operation';
|
||||
export { getYjsProviders } from './provider';
|
||||
export type { YjsProviderOptions } from './provider';
|
||||
|
||||
export type YjsInitOptions = {
|
||||
userId?: string;
|
||||
token?: string;
|
||||
provider?: Record<string, YjsProvider>;
|
||||
};
|
||||
|
||||
export class YjsAdapter implements AsyncDatabaseAdapter<YjsContentOperation> {
|
||||
private readonly _provider: YjsProviders;
|
||||
private readonly _doc: Doc; // doc instance
|
||||
private readonly _awareness: Awareness; // lightweight state synchronization
|
||||
private readonly _gatekeeper: GateKeeper; // Simple access control
|
||||
private readonly _history!: YjsHistoryManager;
|
||||
|
||||
// Block Collection
|
||||
// key is a randomly generated global id
|
||||
private readonly _blocks!: YMap<YMap<unknown>>;
|
||||
private readonly _blockUpdated!: YMap<number>;
|
||||
// Maximum cache Block 1024, ttl 10 minutes
|
||||
private readonly _blockCaches!: LRUCache<string, YjsBlockInstance>;
|
||||
|
||||
private readonly _binaries!: YjsRemoteBinaries;
|
||||
|
||||
private readonly _listener: Map<string, BlockListener<any>>;
|
||||
|
||||
private readonly _reload: () => void;
|
||||
|
||||
static async init(
|
||||
workspace: string,
|
||||
options: YjsInitOptions
|
||||
): Promise<YjsAdapter> {
|
||||
const { userId = 'default', token, provider } = options;
|
||||
const providers = await _initYjsDatabase(workspace, {
|
||||
userId,
|
||||
token,
|
||||
provider,
|
||||
});
|
||||
return new YjsAdapter(providers);
|
||||
}
|
||||
|
||||
private constructor(providers: YjsProviders) {
|
||||
this._provider = providers;
|
||||
this._doc = providers.doc;
|
||||
this._awareness = providers.awareness;
|
||||
this._gatekeeper = providers.gatekeeper;
|
||||
this._reload = () => {
|
||||
const blocks = this._doc.getMap<YMap<any>>('blocks');
|
||||
// @ts-ignore
|
||||
this._blocks =
|
||||
blocks.get('content') || blocks.set('content', new YMap());
|
||||
// @ts-ignore
|
||||
this._blockUpdated =
|
||||
blocks.get('updated') || blocks.set('updated', new YMap());
|
||||
// @ts-ignore
|
||||
this._blockCaches = new LRUCache({
|
||||
max: 1024,
|
||||
ttl: 1000 * 60 * 10,
|
||||
});
|
||||
// @ts-ignore
|
||||
this._binaries = new YjsRemoteBinaries(
|
||||
providers.binaries.getMap(),
|
||||
providers.remoteToken
|
||||
);
|
||||
// @ts-ignore
|
||||
this._history = new YjsHistoryManager(this._blocks);
|
||||
};
|
||||
this._reload();
|
||||
|
||||
this._listener = new Map();
|
||||
|
||||
providers.connListener.listeners = (
|
||||
workspace: string,
|
||||
connectivity: Connectivity
|
||||
) => {
|
||||
this._listener.get('connectivity')?.(
|
||||
new Map([[workspace, connectivity]])
|
||||
);
|
||||
};
|
||||
|
||||
const debounced_editing_notifier = debounce(
|
||||
() => {
|
||||
const listener: BlockListener<Set<string>> | undefined =
|
||||
this._listener.get('editing');
|
||||
if (listener) {
|
||||
const mapping = this._awareness.getStates();
|
||||
const editing_mapping: Record<string, string[]> = {};
|
||||
for (const {
|
||||
userId,
|
||||
editing,
|
||||
updated,
|
||||
} of mapping.values()) {
|
||||
// Only return the status with refresh time within 10 seconds
|
||||
if (
|
||||
userId &&
|
||||
editing &&
|
||||
updated &&
|
||||
typeof updated === 'number' &&
|
||||
updated + 1000 * 10 > Date.now()
|
||||
) {
|
||||
if (!editing_mapping[editing]) {
|
||||
editing_mapping[editing] = [];
|
||||
}
|
||||
editing_mapping[editing]?.push(userId);
|
||||
}
|
||||
}
|
||||
listener(
|
||||
new Map(
|
||||
Object.entries(editing_mapping).map(([k, v]) => [
|
||||
k,
|
||||
new Set(v),
|
||||
])
|
||||
)
|
||||
);
|
||||
}
|
||||
},
|
||||
200,
|
||||
{ maxWait: 1000 }
|
||||
);
|
||||
|
||||
this._awareness.setLocalStateField('userId', providers.userId);
|
||||
|
||||
this._awareness.on('update', debounced_editing_notifier);
|
||||
|
||||
this._blocks.observeDeep(events => {
|
||||
const now = Date.now();
|
||||
|
||||
const keys = events.flatMap(e => {
|
||||
if ((e.path?.length | 0) > 0) {
|
||||
return [
|
||||
[e.path[0], 'update'] as [string, ChangedStateKeys],
|
||||
];
|
||||
} else {
|
||||
return Array.from(e.changes.keys.entries()).map(
|
||||
([k, { action }]) =>
|
||||
[k, action] as [string, ChangedStateKeys]
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
EmitEvents(keys, this._listener.get('updated'));
|
||||
|
||||
transact(this._doc, () => {
|
||||
for (const [key, action] of keys) {
|
||||
if (action === 'delete') {
|
||||
this._blockUpdated.delete(key);
|
||||
} else {
|
||||
this._blockUpdated.set(key, now);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
reload() {
|
||||
this._reload();
|
||||
}
|
||||
|
||||
getUserId(): string {
|
||||
return this._provider.userId;
|
||||
}
|
||||
|
||||
inspector() {
|
||||
const resolve_block = (blocks: Record<string, any>, id: string) => {
|
||||
const block = blocks[id];
|
||||
if (block) {
|
||||
return {
|
||||
...block,
|
||||
children: block.children.map((id: string) =>
|
||||
resolve_block(blocks, id)
|
||||
),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
save: () => {
|
||||
const binary = encodeStateAsUpdate(this._doc);
|
||||
saveAs(
|
||||
new Blob([binary]),
|
||||
`affine_workspace_${new Date().toDateString()}.apk`
|
||||
);
|
||||
},
|
||||
load: async () => {
|
||||
const handles = await window.showOpenFilePicker({
|
||||
types: [
|
||||
{
|
||||
description: 'AFFiNE Package',
|
||||
accept: {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
'application/affine': ['.apk'],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
const [file] = (await fromEvent(handles)) as File[];
|
||||
const binary = await file?.arrayBuffer();
|
||||
// await this._provider.idb.clearData();
|
||||
const doc = new Doc({ autoLoad: true, shouldLoad: true });
|
||||
let updated = 0;
|
||||
let isUpdated = false;
|
||||
doc.on('update', () => {
|
||||
isUpdated = true;
|
||||
updated += 1;
|
||||
});
|
||||
setInterval(() => {
|
||||
if (updated > 0) {
|
||||
updated -= 1;
|
||||
}
|
||||
}, 500);
|
||||
|
||||
const update_check = new Promise<void>(resolve => {
|
||||
const check = async () => {
|
||||
while (!isUpdated || updated > 0) {
|
||||
await sleep();
|
||||
}
|
||||
resolve();
|
||||
};
|
||||
check();
|
||||
});
|
||||
// await new IndexedDBProvider(this._provider.idb.name, doc)
|
||||
// .whenSynced;
|
||||
if (binary) {
|
||||
applyUpdate(doc, new Uint8Array(binary));
|
||||
await update_check;
|
||||
}
|
||||
console.log('load success');
|
||||
},
|
||||
parse: () => this._doc.toJSON(),
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
parse_page: (page_id: string) => {
|
||||
const blocks = this._blocks.toJSON();
|
||||
return resolve_block(blocks, page_id);
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
parse_pages: (resolve = false) => {
|
||||
const blocks = this._blocks.toJSON();
|
||||
return Object.fromEntries(
|
||||
Object.entries(blocks)
|
||||
.filter(([, block]) => block.flavor === 'page')
|
||||
.map(([key, block]) => {
|
||||
if (resolve) {
|
||||
return resolve_block(blocks, key);
|
||||
} else {
|
||||
return [key, block];
|
||||
}
|
||||
})
|
||||
);
|
||||
},
|
||||
clear: () => {
|
||||
this._blocks.clear();
|
||||
this._blockUpdated.clear();
|
||||
this._gatekeeper.clear();
|
||||
this._doc.getMap('blocks').clear();
|
||||
this._doc.getMap('gatekeeper').clear();
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
clear_old: () => {
|
||||
this._doc.getMap('block_updated').clear();
|
||||
this._doc.getMap('blocks').clear();
|
||||
this._doc.getMap('common').clear();
|
||||
this._doc.getMap('creators').clear();
|
||||
},
|
||||
snapshot: () => {
|
||||
return snapshot(this._doc);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async createBlock(
|
||||
options: Pick<BlockItem<YjsContentOperation>, 'type' | 'flavor'> & {
|
||||
uuid: string | undefined;
|
||||
binary: ArrayBufferLike | undefined;
|
||||
}
|
||||
): Promise<YjsBlockInstance> {
|
||||
const uuid = options.uuid || `affine${nanoid(16)}`;
|
||||
if (options.type === BlockTypes.binary) {
|
||||
if (options.binary && options.binary instanceof ArrayBuffer) {
|
||||
const array = new YArray();
|
||||
array.insert(0, [options.binary]);
|
||||
const block = {
|
||||
type: options.type,
|
||||
flavor: options.flavor,
|
||||
children: [] as string[],
|
||||
created: Date.now(),
|
||||
content: new YjsContentOperation(array),
|
||||
hash: sha3(Buffer.from(options.binary)),
|
||||
};
|
||||
await this.set_block(uuid, block);
|
||||
return (await this.getBlock(uuid))!;
|
||||
} else {
|
||||
throw new Error(`Invalid binary type: ${options.binary}`);
|
||||
}
|
||||
} else {
|
||||
const block = {
|
||||
type: options.type,
|
||||
flavor: options.flavor,
|
||||
children: [] as string[],
|
||||
created: Date.now(),
|
||||
content: new YjsContentOperation(new YMap()),
|
||||
};
|
||||
await this.set_block(uuid, block);
|
||||
return (await this.getBlock(uuid))!;
|
||||
}
|
||||
}
|
||||
|
||||
private get_updated(id: string) {
|
||||
return this._blockUpdated.get(id);
|
||||
}
|
||||
|
||||
private get_creator(id: string) {
|
||||
return this._gatekeeper.getCreator(id);
|
||||
}
|
||||
|
||||
private get_block_sync(id: string): YjsBlockInstance | undefined {
|
||||
const cached = this._blockCaches.get(id);
|
||||
if (cached) {
|
||||
// Synchronous read cannot read binary
|
||||
if (cached.type === BlockTypes.block) {
|
||||
return cached;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const block = this._blocks.get(id);
|
||||
|
||||
// Synchronous read cannot read binary
|
||||
if (block && block.get('type') === BlockTypes.block) {
|
||||
return new YjsBlockInstance({
|
||||
id,
|
||||
block,
|
||||
setBlock: this.set_block.bind(this),
|
||||
getUpdated: this.get_updated.bind(this),
|
||||
getCreator: this.get_creator.bind(this),
|
||||
getBlockInstance: this.get_block_sync.bind(this),
|
||||
});
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async getBlock(id: string): Promise<YjsBlockInstance | undefined> {
|
||||
const block_instance = this.get_block_sync(id);
|
||||
if (block_instance) {
|
||||
return block_instance;
|
||||
}
|
||||
const block = this._blocks.get(id);
|
||||
if (block && block.get('type') === BlockTypes.binary) {
|
||||
const binary = await this._binaries.get(
|
||||
block.get('hash') as string
|
||||
);
|
||||
if (binary) {
|
||||
return new YjsBlockInstance({
|
||||
id,
|
||||
block,
|
||||
binary,
|
||||
setBlock: this.set_block.bind(this),
|
||||
getUpdated: this.get_updated.bind(this),
|
||||
getCreator: this.get_creator.bind(this),
|
||||
getBlockInstance: this.get_block_sync.bind(this),
|
||||
});
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async getBlockByFlavor(
|
||||
flavor: BlockItem<YjsContentOperation>['flavor']
|
||||
): Promise<string[]> {
|
||||
const keys: string[] = [];
|
||||
this._blocks.forEach((doc, key) => {
|
||||
if (doc.get('flavor') === flavor) {
|
||||
keys.push(key);
|
||||
}
|
||||
});
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
async getBlockByType(
|
||||
type: BlockItem<YjsContentOperation>['type']
|
||||
): Promise<string[]> {
|
||||
const keys: string[] = [];
|
||||
this._blocks.forEach((doc, key) => {
|
||||
if (doc.get('type') === type) {
|
||||
keys.push(key);
|
||||
}
|
||||
});
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
private async set_block(
|
||||
key: string,
|
||||
item: BlockItem<YjsContentOperation> & { hash?: string }
|
||||
): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const block = this._blocks.get(key) || new YMap();
|
||||
transact(this._doc, () => {
|
||||
// Insert only if the block doesn't exist yet
|
||||
// Other modification operations are done in the block instance
|
||||
let uploaded: Promise<void> | undefined;
|
||||
if (!block.size) {
|
||||
const content = item.content[INTO_INNER]();
|
||||
if (!content) {
|
||||
return reject();
|
||||
}
|
||||
|
||||
const children = new YArray();
|
||||
children.push(item.children);
|
||||
|
||||
block.set('type', item.type);
|
||||
block.set('flavor', item.flavor);
|
||||
block.set('children', children);
|
||||
block.set('created', item.created);
|
||||
if (item.type === BlockTypes.block) {
|
||||
block.set('content', content);
|
||||
} else if (item.type === BlockTypes.binary && item.hash) {
|
||||
if (content instanceof YArray) {
|
||||
block.set('hash', item.hash);
|
||||
if (!this._binaries.has(item.hash)) {
|
||||
uploaded = this._binaries.set(
|
||||
item.hash,
|
||||
content
|
||||
);
|
||||
}
|
||||
} else {
|
||||
throw new Error(
|
||||
'binary content must be an buffer yarray'
|
||||
);
|
||||
}
|
||||
} else {
|
||||
throw new Error('invalid block type: ' + item.type);
|
||||
}
|
||||
|
||||
this._blocks.set(key, block);
|
||||
}
|
||||
|
||||
if (item.flavor === 'page') {
|
||||
this._awareness.setLocalStateField('editing', key);
|
||||
this._awareness.setLocalStateField('updated', Date.now());
|
||||
}
|
||||
// References do not add delete restrictions
|
||||
if (item.flavor === 'reference') {
|
||||
this._gatekeeper.setCommon(key);
|
||||
} else {
|
||||
this._gatekeeper.setCreator(key);
|
||||
}
|
||||
|
||||
if (uploaded) {
|
||||
// TODO: there should be a mechanism to retry the upload
|
||||
uploaded.catch(err => {
|
||||
// undo set on failure
|
||||
console.error('Failed to upload object: ', err);
|
||||
this.deleteBlocks([key]);
|
||||
reject(err);
|
||||
});
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async checkBlocks(keys: string[]): Promise<boolean> {
|
||||
return (
|
||||
keys.filter(key => !!this._blocks.get(key)).length === keys.length
|
||||
);
|
||||
}
|
||||
|
||||
async deleteBlocks(keys: string[]): Promise<string[]> {
|
||||
const [success, fail] = this._gatekeeper.checkDeleteLists(keys);
|
||||
transact(this._doc, () => {
|
||||
for (const key of success) {
|
||||
this._blocks.delete(key);
|
||||
}
|
||||
});
|
||||
return fail;
|
||||
}
|
||||
|
||||
on<S, R>(
|
||||
key: 'editing' | 'updated' | 'connectivity',
|
||||
listener: BlockListener<S, R>
|
||||
): void {
|
||||
this._listener.set(key, listener);
|
||||
}
|
||||
|
||||
suspend(suspend: boolean) {
|
||||
Suspend(suspend);
|
||||
}
|
||||
|
||||
public history(): HistoryManager {
|
||||
return this._history;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { produce } from 'immer';
|
||||
import { debounce } from 'ts-debounce';
|
||||
import { YEvent } from 'yjs';
|
||||
|
||||
import { BlockListener, ChangedStateKeys } from './types';
|
||||
|
||||
let listener_suspend = false;
|
||||
|
||||
let listener_map = new Map<BlockListener, [string, ChangedStateKeys][]>();
|
||||
|
||||
const debounced_suspend_notifier = debounce(
|
||||
(listener?: BlockListener) => {
|
||||
if (listener) {
|
||||
listener_map = produce(listener_map, draft => {
|
||||
const events = draft.get(listener);
|
||||
if (events) {
|
||||
listener(new Map(events));
|
||||
draft.delete(listener);
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
500,
|
||||
{ maxWait: 2000 }
|
||||
);
|
||||
|
||||
/**
|
||||
* Suspend instant update event dispatch, extend to at least 500ms once, and up to 2000ms once when triggered continuously
|
||||
* @param suspend true: suspend monitoring, false: resume monitoring
|
||||
*/
|
||||
export function Suspend(suspend: boolean) {
|
||||
listener_suspend = produce(listener_suspend, draft => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
draft = suspend;
|
||||
});
|
||||
if (!suspend && listener_map.size) {
|
||||
listener_map = produce(listener_map, draft => {
|
||||
for (const [listener, events] of draft) {
|
||||
listener(new Map(events));
|
||||
}
|
||||
draft.clear();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function EmitEvents(
|
||||
events: [string, ChangedStateKeys][],
|
||||
listener?: BlockListener
|
||||
) {
|
||||
if (listener) {
|
||||
if (listener_suspend) {
|
||||
listener_map = produce(listener_map, draft => {
|
||||
const old_events = listener_map.get(listener) || [];
|
||||
draft.set(listener, [...old_events, ...events]);
|
||||
});
|
||||
debounced_suspend_notifier(listener);
|
||||
} else {
|
||||
listener(new Map(events));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function ChildrenListenerHandler(
|
||||
listeners: Map<string, BlockListener>,
|
||||
event: YEvent<any>
|
||||
) {
|
||||
if (listeners.size) {
|
||||
const keys = Array.from(event.keys.entries()).map(
|
||||
([key, { action }]) => [key, action] as [string, ChangedStateKeys]
|
||||
);
|
||||
const deleted = Array.from(event.changes.deleted.values())
|
||||
.flatMap(val => val.content.getContent() as string[])
|
||||
.filter(v => v)
|
||||
.map(k => [k, 'delete'] as [string, ChangedStateKeys]);
|
||||
for (const listener of listeners.values()) {
|
||||
EmitEvents([...keys, ...deleted], listener);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function ContentListenerHandler(
|
||||
listeners: Map<string, BlockListener>,
|
||||
events: YEvent<any>[]
|
||||
) {
|
||||
if (listeners.size) {
|
||||
const keys = events.flatMap(e => {
|
||||
if ((e.path?.length | 0) > 0) {
|
||||
return [[e.path[0], 'update'] as [string, 'update']];
|
||||
} else {
|
||||
return Array.from(e.changes.keys.entries()).map(
|
||||
([k, { action }]) => [k, action] as [string, typeof action]
|
||||
);
|
||||
}
|
||||
});
|
||||
if (keys.length) {
|
||||
for (const listener of listeners.values()) {
|
||||
EmitEvents(keys, listener);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,408 @@
|
||||
/* eslint-disable max-lines */
|
||||
import {
|
||||
AbstractType as YAbstractType,
|
||||
Array as YArray,
|
||||
Map as YMap,
|
||||
Text as YText,
|
||||
} from 'yjs';
|
||||
|
||||
import { ChildrenListenerHandler, ContentListenerHandler } from './listener';
|
||||
import {
|
||||
ArrayOperation,
|
||||
BaseTypes,
|
||||
BlockListener,
|
||||
ContentOperation,
|
||||
ContentTypes,
|
||||
MapOperation,
|
||||
Operable,
|
||||
TextOperation,
|
||||
TextToken,
|
||||
} from './types';
|
||||
|
||||
const INTO_INNER = Symbol('INTO_INNER');
|
||||
|
||||
export const DO_NOT_USE_THIS_OR_YOU_WILL_BE_FIRED_SYMBOL_INTO_INNER: typeof INTO_INNER =
|
||||
INTO_INNER;
|
||||
|
||||
function auto_get(
|
||||
root: ContentOperation,
|
||||
key: string | undefined
|
||||
): unknown | undefined {
|
||||
const array = root.asArray();
|
||||
if (array && !Number.isNaN(Number(key))) {
|
||||
return array.get(Number(key));
|
||||
}
|
||||
const map = root.asMap();
|
||||
if (map && key) {
|
||||
return map.get(key);
|
||||
}
|
||||
const text = root.asText();
|
||||
if (text) {
|
||||
return text.toString();
|
||||
}
|
||||
console.error('auto_get unknown root', root, key);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function auto_set(root: ContentOperation, key: string, data: BaseTypes): void {
|
||||
const array = root.asArray<BaseTypes>();
|
||||
if (array && !Number.isNaN(Number(key))) {
|
||||
return array.insert(Number(key), [data]);
|
||||
}
|
||||
const map = root.asMap<BaseTypes>();
|
||||
if (map) {
|
||||
return map.set(key, data);
|
||||
}
|
||||
const text = root.asText();
|
||||
if (text && !Number.isNaN(Number(key)) && typeof data === 'string') {
|
||||
return text.insert(Number(key), data);
|
||||
}
|
||||
console.error('autoSet unknown root or path', root, key, data);
|
||||
}
|
||||
|
||||
export class YjsContentOperation implements ContentOperation {
|
||||
private readonly _content: YAbstractType<unknown>;
|
||||
|
||||
constructor(content: YAbstractType<any>) {
|
||||
this._content = content;
|
||||
}
|
||||
|
||||
get length(): number {
|
||||
if (this._content instanceof YMap) {
|
||||
return this._content.size;
|
||||
}
|
||||
if (this._content instanceof YArray || this._content instanceof YText) {
|
||||
return this._content.length;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
createText(): YjsTextOperation {
|
||||
return new YjsTextOperation(new YText());
|
||||
}
|
||||
|
||||
createArray<
|
||||
T extends ContentTypes = ContentOperation
|
||||
>(): YjsArrayOperation<T> {
|
||||
return new YjsArrayOperation(new YArray());
|
||||
}
|
||||
|
||||
createMap<T extends ContentTypes = ContentOperation>(): YjsMapOperation<T> {
|
||||
return new YjsMapOperation(new YMap());
|
||||
}
|
||||
|
||||
asText(): YjsTextOperation | undefined {
|
||||
if (this._content instanceof YText) {
|
||||
return new YjsTextOperation(this._content);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
asArray<T extends ContentTypes = ContentOperation>():
|
||||
| YjsArrayOperation<T>
|
||||
| undefined {
|
||||
if (this._content instanceof YArray) {
|
||||
return new YjsArrayOperation(this._content);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
asMap<T extends ContentTypes = ContentOperation>():
|
||||
| YjsMapOperation<T>
|
||||
| undefined {
|
||||
if (this._content instanceof YMap) {
|
||||
return new YjsMapOperation(this._content);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
autoGet(
|
||||
root: ThisType<ContentOperation> | Record<string, unknown>,
|
||||
path: string[]
|
||||
): unknown | undefined {
|
||||
if (root) {
|
||||
if (path.length === 0) {
|
||||
return root;
|
||||
} else if (root instanceof YjsContentOperation) {
|
||||
const [key, ...rest] = path;
|
||||
const new_root = auto_get(root, key);
|
||||
if (new_root) {
|
||||
return this.autoGet(new_root as typeof root, rest);
|
||||
}
|
||||
} else if (typeof root === 'object') {
|
||||
throw new Error(
|
||||
'autoGet must not get a non-value type, this is a deprecated behavior'
|
||||
);
|
||||
}
|
||||
}
|
||||
console.error('autoGet unknown root', root, path);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
autoSet(
|
||||
root: ThisType<ContentOperation>,
|
||||
path: string[],
|
||||
data: BaseTypes,
|
||||
partial?: boolean
|
||||
): void {
|
||||
if (root) {
|
||||
if (path.length === 0) {
|
||||
if (data && typeof data === 'object') {
|
||||
throw new Error(
|
||||
'autoSet must not set a non-value type, this is a deprecated behavior'
|
||||
);
|
||||
} else {
|
||||
console.error('autoSet unknown data', root, path, data);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (root instanceof YjsContentOperation) {
|
||||
if (path.length === 1) {
|
||||
const [key] = path;
|
||||
if (key) {
|
||||
return auto_set(root, key, data);
|
||||
}
|
||||
console.error('autoSet unknown path', root, path, data);
|
||||
return;
|
||||
}
|
||||
const [key, ...rest] = path;
|
||||
const new_root = auto_get(root, key);
|
||||
if (new_root && new_root instanceof YjsContentOperation) {
|
||||
return this.autoSet(new_root, rest, data, partial);
|
||||
} else {
|
||||
throw new Error(
|
||||
'autoSet must not set a non-value type, this is a deprecated behavior'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
console.error('autoSet unknown root', root, path);
|
||||
}
|
||||
|
||||
protected into_inner<T>(content: Operable<T>): T {
|
||||
if (content instanceof YjsContentOperation) {
|
||||
return content[INTO_INNER]() as unknown as T;
|
||||
} else {
|
||||
return content as T;
|
||||
}
|
||||
}
|
||||
|
||||
protected to_operable<T>(content: T): Operable<T> {
|
||||
if (content instanceof YAbstractType) {
|
||||
return new YjsContentOperation(content) as unknown as Operable<T>;
|
||||
}
|
||||
return content as Operable<T>;
|
||||
}
|
||||
|
||||
[INTO_INNER](): YAbstractType<unknown> | undefined {
|
||||
if (this._content instanceof YAbstractType) {
|
||||
return this._content;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
// @ts-ignore
|
||||
private toJSON() {
|
||||
return this._content.toJSON();
|
||||
}
|
||||
}
|
||||
|
||||
class YjsTextOperation extends YjsContentOperation implements TextOperation {
|
||||
private readonly _textContent: YText;
|
||||
|
||||
constructor(content: YText) {
|
||||
super(content);
|
||||
this._textContent = content;
|
||||
}
|
||||
|
||||
insert(
|
||||
index: number,
|
||||
content: string,
|
||||
format?: Record<string, string>
|
||||
): void {
|
||||
this._textContent.insert(index, content, format);
|
||||
}
|
||||
|
||||
format(
|
||||
index: number,
|
||||
length: number,
|
||||
format: Record<string, string>
|
||||
): void {
|
||||
this._textContent.format(index, length, format);
|
||||
}
|
||||
|
||||
delete(index: number, length: number): void {
|
||||
this._textContent.delete(index, length);
|
||||
}
|
||||
|
||||
setAttribute(name: string, value: BaseTypes) {
|
||||
this._textContent.setAttribute(name, value);
|
||||
}
|
||||
|
||||
getAttribute<T extends BaseTypes = string>(name: string): T | undefined {
|
||||
return this._textContent.getAttribute(name);
|
||||
}
|
||||
|
||||
override toString(): TextToken[] {
|
||||
return this._textContent.toDelta();
|
||||
}
|
||||
}
|
||||
|
||||
class YjsArrayOperation<T extends ContentTypes>
|
||||
extends YjsContentOperation
|
||||
implements ArrayOperation<T>
|
||||
{
|
||||
private readonly _arrayContent: YArray<T>;
|
||||
private readonly _listeners: Map<string, BlockListener>;
|
||||
|
||||
constructor(content: YArray<T>) {
|
||||
super(content);
|
||||
this._arrayContent = content;
|
||||
this._listeners = new Map();
|
||||
|
||||
this._arrayContent.observe(event =>
|
||||
ChildrenListenerHandler(this._listeners, event)
|
||||
);
|
||||
}
|
||||
|
||||
on(name: string, listener: BlockListener) {
|
||||
this._listeners.set(name, listener);
|
||||
}
|
||||
|
||||
off(name: string) {
|
||||
this._listeners.delete(name);
|
||||
}
|
||||
|
||||
insert(index: number, content: Array<Operable<T>>): void {
|
||||
this._arrayContent.insert(
|
||||
index,
|
||||
content.map(v => this.into_inner(v))
|
||||
);
|
||||
}
|
||||
|
||||
delete(index: number, length: number): void {
|
||||
this._arrayContent.delete(index, length);
|
||||
}
|
||||
|
||||
push(content: Array<Operable<T>>): void {
|
||||
this._arrayContent.push(content.map(v => this.into_inner(v)));
|
||||
}
|
||||
|
||||
unshift(content: Array<Operable<T>>): void {
|
||||
this._arrayContent.unshift(content.map(v => this.into_inner(v)));
|
||||
}
|
||||
|
||||
get(index: number): Operable<T> | undefined {
|
||||
const content = this._arrayContent.get(index);
|
||||
if (content) {
|
||||
return this.to_operable(content);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private get_internal(index: number): T {
|
||||
return this._arrayContent.get(index);
|
||||
}
|
||||
|
||||
slice(start?: number, end?: number): Operable<T>[] {
|
||||
return this._arrayContent
|
||||
.slice(start, end)
|
||||
.map(v => this.to_operable(v));
|
||||
}
|
||||
|
||||
map<R = unknown>(callback: (value: T, index: number) => R): R[] {
|
||||
return this._arrayContent.map((value, index) => callback(value, index));
|
||||
}
|
||||
|
||||
// Traverse, if callback returns false, stop traversing
|
||||
forEach(callback: (value: T, index: number) => boolean) {
|
||||
for (let i = 0; i < this._arrayContent.length; i++) {
|
||||
const ret = callback(this.get_internal(i), i);
|
||||
if (ret === false) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
find<R = unknown>(
|
||||
callback: (value: T, index: number) => boolean
|
||||
): R | undefined {
|
||||
let result: R | undefined = undefined;
|
||||
this.forEach((value, i) => {
|
||||
const found = callback(value, i);
|
||||
if (found) {
|
||||
result = value as unknown as R;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
findIndex(callback: (value: T, index: number) => boolean): number {
|
||||
let position = -1;
|
||||
this.forEach((value, i) => {
|
||||
const found = callback(value, i);
|
||||
if (found) {
|
||||
position = i;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return position;
|
||||
}
|
||||
}
|
||||
|
||||
class YjsMapOperation<T extends ContentTypes>
|
||||
extends YjsContentOperation
|
||||
implements MapOperation<T>
|
||||
{
|
||||
private readonly _mapContent: YMap<T>;
|
||||
private readonly _listeners: Map<string, BlockListener>;
|
||||
|
||||
constructor(content: YMap<T>) {
|
||||
super(content);
|
||||
this._mapContent = content;
|
||||
this._listeners = new Map();
|
||||
|
||||
content?.observeDeep(events =>
|
||||
ContentListenerHandler(this._listeners, events)
|
||||
);
|
||||
}
|
||||
|
||||
on(name: string, listener: BlockListener) {
|
||||
this._listeners.set(name, listener);
|
||||
}
|
||||
|
||||
off(name: string) {
|
||||
this._listeners.delete(name);
|
||||
}
|
||||
|
||||
set(key: string, value: Operable<T>): void {
|
||||
if (value instanceof YjsContentOperation) {
|
||||
const content = value[INTO_INNER]();
|
||||
if (content) {
|
||||
this._mapContent.set(key, content as unknown as T);
|
||||
}
|
||||
} else {
|
||||
this._mapContent.set(key, value as T);
|
||||
}
|
||||
}
|
||||
|
||||
get(key: string): Operable<T> | undefined {
|
||||
const content = this._mapContent.get(key);
|
||||
if (content) {
|
||||
return this.to_operable(content);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
delete(key: string): void {
|
||||
this._mapContent.delete(key);
|
||||
}
|
||||
|
||||
has(key: string): boolean {
|
||||
return this._mapContent.has(key);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import { Awareness } from 'y-protocols/awareness.js';
|
||||
import { Doc } from 'yjs';
|
||||
|
||||
import {
|
||||
IndexedDBProvider,
|
||||
SQLiteProvider,
|
||||
WebsocketProvider,
|
||||
} from '@toeverything/datasource/jwt-rpc';
|
||||
|
||||
import { BucketBackend } from '../types';
|
||||
import { Connectivity } from './types';
|
||||
|
||||
type YjsDefaultInstances = {
|
||||
awareness: Awareness;
|
||||
doc: Doc;
|
||||
token?: string | undefined;
|
||||
workspace: string;
|
||||
emitState: (connectivity: Connectivity) => void;
|
||||
};
|
||||
|
||||
export type YjsProvider = (instances: YjsDefaultInstances) => Promise<void>;
|
||||
|
||||
type ProviderType = 'idb' | 'sqlite' | 'ws';
|
||||
|
||||
export type YjsProviderOptions = {
|
||||
enabled: ProviderType[];
|
||||
backend: typeof BucketBackend[keyof typeof BucketBackend];
|
||||
params?: Record<string, string>;
|
||||
importData?: () => Promise<Uint8Array> | Uint8Array | undefined;
|
||||
exportData?: (binary: Uint8Array) => Promise<void> | undefined;
|
||||
hasExporter?: () => boolean;
|
||||
};
|
||||
|
||||
export const getYjsProviders = (
|
||||
options: YjsProviderOptions
|
||||
): Record<string, YjsProvider> => {
|
||||
console.log('getYjsProviders', options);
|
||||
return {
|
||||
indexeddb: async (instances: YjsDefaultInstances) => {
|
||||
if (options.enabled.includes('idb')) {
|
||||
await new IndexedDBProvider(instances.workspace, instances.doc)
|
||||
.whenSynced;
|
||||
}
|
||||
},
|
||||
sqlite: async (instances: YjsDefaultInstances) => {
|
||||
if (options.enabled.includes('sqlite')) {
|
||||
const fsHandle = setInterval(async () => {
|
||||
if (options.hasExporter?.()) {
|
||||
clearInterval(fsHandle);
|
||||
const fs = new SQLiteProvider(
|
||||
instances.workspace,
|
||||
instances.doc,
|
||||
await options.importData?.()
|
||||
);
|
||||
if (options.exportData) {
|
||||
fs.registerExporter(options.exportData);
|
||||
}
|
||||
await fs.whenSynced;
|
||||
}
|
||||
}, 500);
|
||||
}
|
||||
},
|
||||
ws: async (instances: YjsDefaultInstances) => {
|
||||
if (options.enabled.includes('ws')) {
|
||||
if (instances.token) {
|
||||
const ws = new WebsocketProvider(
|
||||
instances.token,
|
||||
options.backend,
|
||||
instances.workspace,
|
||||
instances.doc,
|
||||
{
|
||||
awareness: instances.awareness,
|
||||
params: options.params,
|
||||
}
|
||||
) as any; // TODO: type is erased after cascading references
|
||||
|
||||
// Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
// TODO: synced will also be triggered on reconnection after losing sync
|
||||
// There needs to be an event mechanism to emit the synchronization state to the upper layer
|
||||
ws.once('synced', () => resolve());
|
||||
ws.once('lost-connection', () => resolve());
|
||||
ws.once('connection-error', () => reject());
|
||||
ws.on('synced', () => instances.emitState('connected'));
|
||||
ws.on('lost-connection', () =>
|
||||
instances.emitState('retry')
|
||||
);
|
||||
ws.on('connection-error', () =>
|
||||
instances.emitState('retry')
|
||||
);
|
||||
});
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,192 @@
|
||||
import { AbstractType as YAbstractType } from 'yjs';
|
||||
|
||||
import { BlockItem } from '../types';
|
||||
|
||||
export type ChangedStateKeys = 'add' | 'update' | 'delete';
|
||||
export type ChangedStates<S = ChangedStateKeys> = Map<string, S>;
|
||||
|
||||
export type BlockListener<S = ChangedStateKeys, R = unknown> = (
|
||||
states: ChangedStates<S>
|
||||
) => Promise<R> | R;
|
||||
|
||||
export type Connectivity = 'disconnect' | 'connecting' | 'connected' | 'retry';
|
||||
|
||||
export type Operable<T, Base = YAbstractType<any>> = T extends Base
|
||||
? ContentOperation
|
||||
: T;
|
||||
|
||||
export interface InternalPlainObject {}
|
||||
|
||||
export type BaseTypes = string | number | boolean | InternalPlainObject;
|
||||
|
||||
export type ContentTypes = BaseTypes | ContentOperation;
|
||||
|
||||
interface ContentOperation {
|
||||
get length(): number;
|
||||
createText(): TextOperation;
|
||||
createArray<T extends ContentTypes = ContentOperation>(): ArrayOperation<T>;
|
||||
createMap<T extends ContentTypes = ContentOperation>(): MapOperation<T>;
|
||||
asText(): TextOperation | undefined;
|
||||
asArray<T extends ContentTypes = ContentOperation>():
|
||||
| ArrayOperation<T>
|
||||
| undefined;
|
||||
asMap<T extends ContentTypes = ContentOperation>():
|
||||
| MapOperation<T>
|
||||
| undefined;
|
||||
autoGet(
|
||||
root: ThisType<ContentOperation> | Record<string, unknown>,
|
||||
path: string[]
|
||||
): unknown | undefined;
|
||||
autoSet(
|
||||
root: ThisType<ContentOperation>,
|
||||
path: string[],
|
||||
data: unknown,
|
||||
partial?: boolean
|
||||
): void;
|
||||
}
|
||||
|
||||
type TextAttributes = Record<string, string>;
|
||||
|
||||
export type TextToken = {
|
||||
insert: string;
|
||||
attributes?: TextAttributes;
|
||||
};
|
||||
|
||||
export interface TextOperation extends ContentOperation {
|
||||
insert(
|
||||
index: number,
|
||||
content: string,
|
||||
format?: Record<string, string>
|
||||
): void;
|
||||
format(index: number, length: number, format: Record<string, string>): void;
|
||||
delete(index: number, length: number): void;
|
||||
setAttribute(name: string, value: BaseTypes): void;
|
||||
getAttribute<T extends BaseTypes = string>(name: string): T | undefined;
|
||||
toString(): TextToken[];
|
||||
}
|
||||
|
||||
export interface ArrayOperation<T extends ContentTypes = ContentOperation>
|
||||
extends ContentOperation {
|
||||
insert(index: number, content: Array<Operable<T>>): void;
|
||||
delete(index: number, length: number): void;
|
||||
push(content: Array<Operable<T>>): void;
|
||||
unshift(content: Array<Operable<T>>): void;
|
||||
get(index: number): Operable<T> | undefined;
|
||||
slice(start?: number, end?: number): Array<Operable<T>>;
|
||||
map<R = unknown>(callback: (value: T, index: number) => R): Array<R>;
|
||||
forEach(callback: (value: T, index: number) => boolean | void): void;
|
||||
find<R = unknown>(
|
||||
callback: (value: T, index: number) => boolean
|
||||
): R | undefined;
|
||||
findIndex(callback: (value: T, index: number) => boolean): number;
|
||||
}
|
||||
|
||||
export interface MapOperation<T extends ContentTypes = ContentOperation>
|
||||
extends ContentOperation {
|
||||
set(key: string, value: Operable<T>): void;
|
||||
get(key: string): Operable<T> | undefined;
|
||||
delete(key: string): void;
|
||||
has(key: string): boolean;
|
||||
}
|
||||
|
||||
export type HistoryCallback<T = unknown> = (map: Map<string, T>) => void;
|
||||
|
||||
export interface HistoryManager {
|
||||
onPush<T = unknown>(name: string, callback: HistoryCallback<T>): void;
|
||||
offPush(name: string): boolean;
|
||||
onPop<T = unknown>(name: string, callback: HistoryCallback<T>): void;
|
||||
offPop(name: string): boolean;
|
||||
undo<T = unknown>(): Map<string, T> | undefined;
|
||||
redo<T = unknown>(): Map<string, T> | undefined;
|
||||
clear(): void;
|
||||
}
|
||||
|
||||
type BlockPosition = { pos?: number; before?: string; after?: string };
|
||||
|
||||
interface BlockInstance<C extends ContentOperation> {
|
||||
get id(): string;
|
||||
get type(): BlockItem<C>['type'];
|
||||
get flavor(): BlockItem<C>['flavor'];
|
||||
// TODO: flavor needs optimization
|
||||
setFlavor(flavor: BlockItem<C>['flavor']): void;
|
||||
get created(): BlockItem<C>['created'];
|
||||
get updated(): number; // update time, UTC timestamp, read only
|
||||
get creator(): string | undefined; // creator id
|
||||
get children(): string[];
|
||||
getChildren(block_ids: (string | undefined)[]): BlockInstance<C>[];
|
||||
hasChildren(block_id: string): boolean;
|
||||
insertChildren(
|
||||
block: ThisType<BlockInstance<C>>,
|
||||
pos?: BlockPosition
|
||||
): void;
|
||||
removeChildren(block_ids: (string | undefined)[]): void;
|
||||
get content(): BlockItem<C>['content'];
|
||||
on(
|
||||
key: 'children' | 'content',
|
||||
name: string,
|
||||
listener: BlockListener
|
||||
): void;
|
||||
off(key: 'children' | 'content', name: string): void;
|
||||
addChildrenListener(name: string, listener: BlockListener): void;
|
||||
removeChildrenListener(name: string): void;
|
||||
addContentListener(name: string, listener: BlockListener): void;
|
||||
removeContentListener(name: string): void;
|
||||
scopedHistory(scope: any[]): HistoryManager;
|
||||
}
|
||||
|
||||
interface AsyncDatabaseAdapter<C extends ContentOperation> {
|
||||
inspector(): Record<string, any>;
|
||||
reload(): void;
|
||||
createBlock(
|
||||
options: Pick<BlockItem<C>, 'type' | 'flavor'> & {
|
||||
binary: ArrayBuffer | undefined;
|
||||
uuid: string | undefined;
|
||||
}
|
||||
): Promise<BlockInstance<C>>;
|
||||
getBlock(id: string): Promise<BlockInstance<C> | undefined>;
|
||||
getBlockByFlavor(flavor: BlockItem<C>['flavor']): Promise<string[]>;
|
||||
getBlockByType(type: BlockItem<C>['type']): Promise<string[]>;
|
||||
checkBlocks(keys: string[]): Promise<boolean>;
|
||||
deleteBlocks(keys: string[]): Promise<string[]>;
|
||||
on<S, R>(
|
||||
key: 'editing' | 'updated' | 'connectivity',
|
||||
listener: BlockListener<S, R>
|
||||
): void;
|
||||
suspend(suspend: boolean): void;
|
||||
history(): HistoryManager;
|
||||
getUserId(): string;
|
||||
}
|
||||
|
||||
export type DataExporter = (binary: Uint8Array) => Promise<void>;
|
||||
|
||||
export const getDataExporter = () => {
|
||||
let exporter: DataExporter | undefined = undefined;
|
||||
let importer: (() => Uint8Array | undefined) | undefined = undefined;
|
||||
|
||||
const importData = () => importer?.();
|
||||
const exportData = (binary: Uint8Array) => exporter?.(binary);
|
||||
const hasExporter = () => !!exporter;
|
||||
|
||||
const installExporter = (
|
||||
initialData: Uint8Array | undefined,
|
||||
cb: DataExporter
|
||||
) => {
|
||||
return new Promise<void>(resolve => {
|
||||
importer = () => initialData;
|
||||
exporter = async (data: Uint8Array) => {
|
||||
exporter = cb;
|
||||
await cb(data);
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
return { importData, exportData, hasExporter, installExporter };
|
||||
};
|
||||
|
||||
export type {
|
||||
AsyncDatabaseAdapter,
|
||||
BlockPosition,
|
||||
BlockInstance,
|
||||
ContentOperation,
|
||||
};
|
||||
Reference in New Issue
Block a user