refactor(editor): job should not rely on doc collection directly (#9488)

This commit is contained in:
Saul-Mirone
2025-01-02 10:50:15 +00:00
parent f2906bc6d0
commit edb5e1d87a
34 changed files with 565 additions and 337 deletions

View File

@@ -27,5 +27,14 @@ export function createJob(middlewares?: JobMiddleware[]) {
const schema = new Schema().register(AffineSchemas);
const docCollection = new DocCollection({ schema });
docCollection.meta.initialize();
return new Job({ collection: docCollection, middlewares: testMiddlewares });
return new Job({
schema,
blobCRUD: docCollection.blobSync,
middlewares: testMiddlewares,
docCRUD: {
create: (id: string) => docCollection.createDoc({ id }),
get: (id: string) => docCollection.getDoc(id),
delete: (id: string) => docCollection.removeDoc(id),
},
});
}

View File

@@ -45,8 +45,17 @@ const provider = container.provider();
*/
async function exportDoc(doc: Doc) {
const job = new Job({
collection: doc.collection,
middlewares: [docLinkBaseURLMiddleware, titleMiddleware],
schema: doc.schema,
blobCRUD: doc.blobSync,
docCRUD: {
create: (id: string) => doc.collection.createDoc({ id }),
get: (id: string) => doc.collection.getDoc(id),
delete: (id: string) => doc.collection.removeDoc(id),
},
middlewares: [
docLinkBaseURLMiddleware(doc.collection.id),
titleMiddleware(doc.collection.meta.docMetas),
],
});
const snapshot = job.docToSnapshot(doc);
const adapter = new HtmlAdapter(job, provider);
@@ -91,11 +100,17 @@ async function importHTMLToDoc({
fileName,
}: ImportHTMLToDocOptions) {
const job = new Job({
collection,
schema: collection.schema,
blobCRUD: collection.blobSync,
docCRUD: {
create: (id: string) => collection.createDoc({ id }),
get: (id: string) => collection.getDoc(id),
delete: (id: string) => collection.removeDoc(id),
},
middlewares: [
defaultImageProxyMiddleware,
fileNameMiddleware(fileName),
docLinkBaseURLMiddleware,
docLinkBaseURLMiddleware(collection.id),
],
});
const htmlAdapter = new HtmlAdapter(job, provider);
@@ -147,11 +162,17 @@ async function importHTMLZip({ collection, imported }: ImportHTMLZipOptions) {
htmlBlobs.map(async ([fileName, blob]) => {
const fileNameWithoutExt = fileName.replace(/\.[^/.]+$/, '');
const job = new Job({
collection,
schema: collection.schema,
blobCRUD: collection.blobSync,
docCRUD: {
create: (id: string) => collection.createDoc({ id }),
get: (id: string) => collection.getDoc(id),
delete: (id: string) => collection.removeDoc(id),
},
middlewares: [
defaultImageProxyMiddleware,
fileNameMiddleware(fileNameWithoutExt),
docLinkBaseURLMiddleware,
docLinkBaseURLMiddleware(collection.id),
],
});
const assets = job.assets;

View File

@@ -51,8 +51,17 @@ type ImportMarkdownZipOptions = {
*/
async function exportDoc(doc: Doc) {
const job = new Job({
collection: doc.collection,
middlewares: [docLinkBaseURLMiddleware, titleMiddleware],
schema: doc.schema,
blobCRUD: doc.blobSync,
docCRUD: {
create: (id: string) => doc.collection.createDoc({ id }),
get: (id: string) => doc.collection.getDoc(id),
delete: (id: string) => doc.collection.removeDoc(id),
},
middlewares: [
docLinkBaseURLMiddleware(doc.collection.id),
titleMiddleware(doc.collection.meta.docMetas),
],
});
const snapshot = job.docToSnapshot(doc);
@@ -101,8 +110,17 @@ async function importMarkdownToBlock({
blockId,
}: ImportMarkdownToBlockOptions) {
const job = new Job({
collection: doc.collection,
middlewares: [defaultImageProxyMiddleware, docLinkBaseURLMiddleware],
schema: doc.schema,
blobCRUD: doc.blobSync,
docCRUD: {
create: (id: string) => doc.collection.createDoc({ id }),
get: (id: string) => doc.collection.getDoc(id),
delete: (id: string) => doc.collection.removeDoc(id),
},
middlewares: [
defaultImageProxyMiddleware,
docLinkBaseURLMiddleware(doc.collection.id),
],
});
const adapter = new MarkdownAdapter(job, provider);
const snapshot = await adapter.toSliceSnapshot({
@@ -137,11 +155,17 @@ async function importMarkdownToDoc({
fileName,
}: ImportMarkdownToDocOptions) {
const job = new Job({
collection,
schema: collection.schema,
blobCRUD: collection.blobSync,
docCRUD: {
create: (id: string) => collection.createDoc({ id }),
get: (id: string) => collection.getDoc(id),
delete: (id: string) => collection.removeDoc(id),
},
middlewares: [
defaultImageProxyMiddleware,
fileNameMiddleware(fileName),
docLinkBaseURLMiddleware,
docLinkBaseURLMiddleware(collection.id),
],
});
const mdAdapter = new MarkdownAdapter(job, provider);
@@ -195,11 +219,17 @@ async function importMarkdownZip({
markdownBlobs.map(async ([fileName, blob]) => {
const fileNameWithoutExt = fileName.replace(/\.[^/.]+$/, '');
const job = new Job({
collection,
schema: collection.schema,
blobCRUD: collection.blobSync,
docCRUD: {
create: (id: string) => collection.createDoc({ id }),
get: (id: string) => collection.getDoc(id),
delete: (id: string) => collection.removeDoc(id),
},
middlewares: [
defaultImageProxyMiddleware,
fileNameMiddleware(fileNameWithoutExt),
docLinkBaseURLMiddleware,
docLinkBaseURLMiddleware(collection.id),
],
});
const assets = job.assets;

View File

@@ -8,193 +8,197 @@ import type {
} from '@blocksuite/affine-model';
import { DEFAULT_IMAGE_PROXY_ENDPOINT } from '@blocksuite/affine-shared/consts';
import { assertExists } from '@blocksuite/global/utils';
import type { DeltaOperation, JobMiddleware } from '@blocksuite/store';
import type { DeltaOperation, DocMeta, JobMiddleware } from '@blocksuite/store';
export const replaceIdMiddleware: JobMiddleware = ({ slots, collection }) => {
const idMap = new Map<string, string>();
slots.afterImport.on(payload => {
if (
payload.type === 'block' &&
payload.snapshot.flavour === 'affine:database'
) {
const model = payload.model as DatabaseBlockModel;
Object.keys(model.cells).forEach(cellId => {
if (idMap.has(cellId)) {
model.cells[idMap.get(cellId)!] = model.cells[cellId];
delete model.cells[cellId];
}
});
}
// replace LinkedPage pageId with new id in paragraph blocks
if (
payload.type === 'block' &&
['affine:list', 'affine:paragraph'].includes(payload.snapshot.flavour)
) {
const model = payload.model as ParagraphBlockModel | ListBlockModel;
let prev = 0;
const delta: DeltaOperation[] = [];
for (const d of model.text.toDelta()) {
if (d.attributes?.reference?.pageId) {
const newId = idMap.get(d.attributes.reference.pageId);
if (!newId) {
prev += d.insert?.length ?? 0;
continue;
}
if (prev > 0) {
delta.push({ retain: prev });
}
delta.push({
retain: d.insert?.length ?? 0,
attributes: {
reference: {
...d.attributes.reference,
pageId: newId,
},
},
});
prev = 0;
} else {
prev += d.insert?.length ?? 0;
}
}
if (delta.length > 0) {
model.text.applyDelta(delta);
}
}
if (
payload.type === 'block' &&
payload.snapshot.flavour === 'affine:surface-ref'
) {
const model = payload.model as SurfaceRefBlockModel;
const original = model.reference;
// If there exists a replacement, replace the reference with the new id.
// Otherwise,
// 1. If the reference is an affine:frame not in doc, generate a new id.
// 2. If the reference is graph, keep the original id.
if (idMap.has(original)) {
model.reference = idMap.get(original)!;
} else if (
model.refFlavour === 'affine:frame' &&
!model.doc.hasBlock(original)
export const replaceIdMiddleware =
(idGenerator: () => string): JobMiddleware =>
({ slots, docCRUD }) => {
const idMap = new Map<string, string>();
slots.afterImport.on(payload => {
if (
payload.type === 'block' &&
payload.snapshot.flavour === 'affine:database'
) {
const newId = collection.idGenerator();
idMap.set(original, newId);
model.reference = newId;
const model = payload.model as DatabaseBlockModel;
Object.keys(model.cells).forEach(cellId => {
if (idMap.has(cellId)) {
model.cells[idMap.get(cellId)!] = model.cells[cellId];
delete model.cells[cellId];
}
});
}
}
// TODO(@fundon): process linked block/element
if (
payload.type === 'block' &&
['affine:embed-linked-doc', 'affine:embed-synced-doc'].includes(
payload.snapshot.flavour
)
) {
const model = payload.model as EmbedLinkedDocModel | EmbedSyncedDocModel;
const original = model.pageId;
// If the pageId is not in the doc, generate a new id.
// If we already have a replacement, use it.
if (!collection.getDoc(original)) {
if (idMap.has(original)) {
model.pageId = idMap.get(original)!;
} else {
const newId = collection.idGenerator();
idMap.set(original, newId);
model.pageId = newId;
// replace LinkedPage pageId with new id in paragraph blocks
if (
payload.type === 'block' &&
['affine:list', 'affine:paragraph'].includes(payload.snapshot.flavour)
) {
const model = payload.model as ParagraphBlockModel | ListBlockModel;
let prev = 0;
const delta: DeltaOperation[] = [];
for (const d of model.text.toDelta()) {
if (d.attributes?.reference?.pageId) {
const newId = idMap.get(d.attributes.reference.pageId);
if (!newId) {
prev += d.insert?.length ?? 0;
continue;
}
if (prev > 0) {
delta.push({ retain: prev });
}
delta.push({
retain: d.insert?.length ?? 0,
attributes: {
reference: {
...d.attributes.reference,
pageId: newId,
},
},
});
prev = 0;
} else {
prev += d.insert?.length ?? 0;
}
}
if (delta.length > 0) {
model.text.applyDelta(delta);
}
}
}
});
slots.beforeImport.on(payload => {
if (payload.type === 'page') {
if (idMap.has(payload.snapshot.meta.id)) {
payload.snapshot.meta.id = idMap.get(payload.snapshot.meta.id)!;
if (
payload.type === 'block' &&
payload.snapshot.flavour === 'affine:surface-ref'
) {
const model = payload.model as SurfaceRefBlockModel;
const original = model.reference;
// If there exists a replacement, replace the reference with the new id.
// Otherwise,
// 1. If the reference is an affine:frame not in doc, generate a new id.
// 2. If the reference is graph, keep the original id.
if (idMap.has(original)) {
model.reference = idMap.get(original)!;
} else if (
model.refFlavour === 'affine:frame' &&
!model.doc.hasBlock(original)
) {
const newId = idGenerator();
idMap.set(original, newId);
model.reference = newId;
}
}
// TODO(@fundon): process linked block/element
if (
payload.type === 'block' &&
['affine:embed-linked-doc', 'affine:embed-synced-doc'].includes(
payload.snapshot.flavour
)
) {
const model = payload.model as
| EmbedLinkedDocModel
| EmbedSyncedDocModel;
const original = model.pageId;
// If the pageId is not in the doc, generate a new id.
// If we already have a replacement, use it.
if (!docCRUD.get(original)) {
if (idMap.has(original)) {
model.pageId = idMap.get(original)!;
} else {
const newId = idGenerator();
idMap.set(original, newId);
model.pageId = newId;
}
}
}
});
slots.beforeImport.on(payload => {
if (payload.type === 'page') {
if (idMap.has(payload.snapshot.meta.id)) {
payload.snapshot.meta.id = idMap.get(payload.snapshot.meta.id)!;
return;
}
const newId = idGenerator();
idMap.set(payload.snapshot.meta.id, newId);
payload.snapshot.meta.id = newId;
return;
}
const newId = collection.idGenerator();
idMap.set(payload.snapshot.meta.id, newId);
payload.snapshot.meta.id = newId;
return;
}
if (payload.type === 'block') {
const { snapshot } = payload;
if (snapshot.flavour === 'affine:page') {
const index = snapshot.children.findIndex(
c => c.flavour === 'affine:surface'
);
if (index !== -1) {
const [surface] = snapshot.children.splice(index, 1);
snapshot.children.push(surface);
if (payload.type === 'block') {
const { snapshot } = payload;
if (snapshot.flavour === 'affine:page') {
const index = snapshot.children.findIndex(
c => c.flavour === 'affine:surface'
);
if (index !== -1) {
const [surface] = snapshot.children.splice(index, 1);
snapshot.children.push(surface);
}
}
const original = snapshot.id;
let newId: string;
if (idMap.has(original)) {
newId = idMap.get(original)!;
} else {
newId = idGenerator();
idMap.set(original, newId);
}
snapshot.id = newId;
if (snapshot.flavour === 'affine:surface') {
// Generate new IDs for images and frames in advance.
snapshot.children.forEach(child => {
const original = child.id;
if (idMap.has(original)) {
newId = idMap.get(original)!;
} else {
newId = idGenerator();
idMap.set(original, newId);
}
});
Object.entries(
snapshot.props.elements as Record<string, Record<string, unknown>>
).forEach(([_, value]) => {
switch (value.type) {
case 'connector': {
let connection = value.source as Record<string, string>;
if (idMap.has(connection.id)) {
const newId = idMap.get(connection.id);
assertExists(newId, 'reference id must exist');
connection.id = newId;
}
connection = value.target as Record<string, string>;
if (idMap.has(connection.id)) {
const newId = idMap.get(connection.id);
assertExists(newId, 'reference id must exist');
connection.id = newId;
}
break;
}
case 'group': {
const json = (value.children as Record<string, unknown>)
.json as Record<string, unknown>;
Object.entries(json).forEach(([key, value]) => {
if (idMap.has(key)) {
delete json[key];
const newKey = idMap.get(key);
assertExists(newKey, 'reference id must exist');
json[newKey] = value;
}
});
break;
}
default:
break;
}
});
}
}
const original = snapshot.id;
let newId: string;
if (idMap.has(original)) {
newId = idMap.get(original)!;
} else {
newId = collection.idGenerator();
idMap.set(original, newId);
}
snapshot.id = newId;
if (snapshot.flavour === 'affine:surface') {
// Generate new IDs for images and frames in advance.
snapshot.children.forEach(child => {
const original = child.id;
if (idMap.has(original)) {
newId = idMap.get(original)!;
} else {
newId = collection.idGenerator();
idMap.set(original, newId);
}
});
Object.entries(
snapshot.props.elements as Record<string, Record<string, unknown>>
).forEach(([_, value]) => {
switch (value.type) {
case 'connector': {
let connection = value.source as Record<string, string>;
if (idMap.has(connection.id)) {
const newId = idMap.get(connection.id);
assertExists(newId, 'reference id must exist');
connection.id = newId;
}
connection = value.target as Record<string, string>;
if (idMap.has(connection.id)) {
const newId = idMap.get(connection.id);
assertExists(newId, 'reference id must exist');
connection.id = newId;
}
break;
}
case 'group': {
const json = (value.children as Record<string, unknown>)
.json as Record<string, unknown>;
Object.entries(json).forEach(([key, value]) => {
if (idMap.has(key)) {
delete json[key];
const newKey = idMap.get(key);
assertExists(newKey, 'reference id must exist');
json[newKey] = value;
}
});
break;
}
default:
break;
}
});
}
}
});
};
});
};
export const customImageProxyMiddleware = (
imageProxyURL: string
@@ -204,46 +208,52 @@ export const customImageProxyMiddleware = (
};
};
const customDocLinkBaseUrlMiddleware = (baseUrl: string): JobMiddleware => {
return ({ adapterConfigs, collection }) => {
const customDocLinkBaseUrlMiddleware = (
baseUrl: string,
collectionId: string
): JobMiddleware => {
return ({ adapterConfigs }) => {
const docLinkBaseUrl = baseUrl
? `${baseUrl}/workspace/${collection.id}`
? `${baseUrl}/workspace/${collectionId}`
: '';
adapterConfigs.set('docLinkBaseUrl', docLinkBaseUrl);
};
};
export const titleMiddleware: JobMiddleware = ({
slots,
collection,
adapterConfigs,
}) => {
slots.beforeExport.on(() => {
for (const meta of collection.meta.docMetas) {
adapterConfigs.set('title:' + meta.id, meta.title);
}
});
};
export const titleMiddleware =
(metas: DocMeta[]): JobMiddleware =>
({ slots, adapterConfigs }) => {
slots.beforeExport.on(() => {
for (const meta of metas) {
adapterConfigs.set('title:' + meta.id, meta.title);
}
});
};
export const docLinkBaseURLMiddlewareBuilder = (baseUrl: string) => {
let middleware = customDocLinkBaseUrlMiddleware(baseUrl);
export const docLinkBaseURLMiddlewareBuilder = (
baseUrl: string,
collectionId: string
) => {
let middleware = customDocLinkBaseUrlMiddleware(baseUrl, collectionId);
return {
get: () => middleware,
set: (url: string) => {
middleware = customDocLinkBaseUrlMiddleware(url);
middleware = customDocLinkBaseUrlMiddleware(url, collectionId);
},
};
};
const defaultDocLinkBaseURLMiddlewareBuilder = docLinkBaseURLMiddlewareBuilder(
typeof window !== 'undefined' ? window.location.origin : '.'
);
const defaultDocLinkBaseURLMiddlewareBuilder = (collectionId: string) =>
docLinkBaseURLMiddlewareBuilder(
typeof window !== 'undefined' ? window.location.origin : '.',
collectionId
);
export const docLinkBaseURLMiddleware =
defaultDocLinkBaseURLMiddlewareBuilder.get();
export const docLinkBaseURLMiddleware = (collectionId: string) =>
defaultDocLinkBaseURLMiddlewareBuilder(collectionId).get();
export const setDocLinkBaseURLMiddleware =
defaultDocLinkBaseURLMiddlewareBuilder.set;
export const setDocLinkBaseURLMiddleware = (collectionId: string) =>
defaultDocLinkBaseURLMiddlewareBuilder(collectionId).set;
const imageProxyMiddlewareBuilder = () => {
let middleware = customImageProxyMiddleware(DEFAULT_IMAGE_PROXY_ENDPOINT);

View File

@@ -119,7 +119,13 @@ async function importNotionZip({
}
const pagePromises = Array.from(pagePaths).map(async path => {
const job = new Job({
collection: collection,
schema: collection.schema,
blobCRUD: collection.blobSync,
docCRUD: {
create: (id: string) => collection.createDoc({ id }),
get: (id: string) => collection.getDoc(id),
delete: (id: string) => collection.removeDoc(id),
},
middlewares: [defaultImageProxyMiddleware],
});
const htmlAdapter = new NotionHtmlAdapter(job, provider);

View File

@@ -7,12 +7,21 @@ import { replaceIdMiddleware, titleMiddleware } from './middlewares.js';
async function exportDocs(collection: DocCollection, docs: Doc[]) {
const zip = new Zip();
const job = new Job({ collection });
const job = new Job({
schema: collection.schema,
blobCRUD: collection.blobSync,
docCRUD: {
create: (id: string) => collection.createDoc({ id }),
get: (id: string) => collection.getDoc(id),
delete: (id: string) => collection.removeDoc(id),
},
middlewares: [
replaceIdMiddleware(collection.idGenerator),
titleMiddleware(collection.meta.docMetas),
],
});
const snapshots = await Promise.all(docs.map(job.docToSnapshot));
const collectionInfo = job.collectionInfoToSnapshot();
await zip.file('info.json', JSON.stringify(collectionInfo, null, 2));
await Promise.all(
snapshots
.filter((snapshot): snapshot is DocSnapshot => !!snapshot)
@@ -69,8 +78,17 @@ async function importDocs(collection: DocCollection, imported: Blob) {
}
const job = new Job({
collection,
middlewares: [replaceIdMiddleware, titleMiddleware],
schema: collection.schema,
blobCRUD: collection.blobSync,
docCRUD: {
create: (id: string) => collection.createDoc({ id }),
get: (id: string) => collection.getDoc(id),
delete: (id: string) => collection.removeDoc(id),
},
middlewares: [
replaceIdMiddleware(collection.idGenerator),
titleMiddleware(collection.meta.docMetas),
],
});
const assetsMap = job.assets;

View File

@@ -59,8 +59,12 @@ export class PageClipboard {
const paste = pasteMiddleware(this._std);
this._std.clipboard.use(copy);
this._std.clipboard.use(paste);
this._std.clipboard.use(replaceIdMiddleware);
this._std.clipboard.use(titleMiddleware);
this._std.clipboard.use(
replaceIdMiddleware(this._std.doc.collection.idGenerator)
);
this._std.clipboard.use(
titleMiddleware(this._std.doc.collection.meta.docMetas)
);
this._std.clipboard.use(defaultImageProxyMiddleware);
this._disposables.add({
@@ -80,8 +84,12 @@ export class PageClipboard {
this._std.clipboard.unregisterAdapter('*/*');
this._std.clipboard.unuse(copy);
this._std.clipboard.unuse(paste);
this._std.clipboard.unuse(replaceIdMiddleware);
this._std.clipboard.unuse(titleMiddleware);
this._std.clipboard.unuse(
replaceIdMiddleware(this._std.doc.collection.idGenerator)
);
this._std.clipboard.unuse(
titleMiddleware(this._std.doc.collection.meta.docMetas)
);
this._std.clipboard.unuse(defaultImageProxyMiddleware);
},
});

View File

@@ -369,7 +369,15 @@ export class EdgelessClipboardController extends PageClipboard {
if (mayBeSurfaceDataJson !== undefined) {
const elementsRawData = JSON.parse(mayBeSurfaceDataJson);
const { snapshot, blobs } = elementsRawData;
const job = new Job({ collection: this.std.collection });
const job = new Job({
schema: this.std.collection.schema,
blobCRUD: this.std.collection.blobSync,
docCRUD: {
create: (id: string) => this.std.collection.createDoc({ id }),
get: (id: string) => this.std.collection.getDoc(id),
delete: (id: string) => this.std.collection.removeDoc(id),
},
});
const map = job.assetsManager.getAssets();
decodeClipboardBlobs(blobs, map);
for (const blobId of map.keys()) {
@@ -1366,7 +1374,13 @@ export async function prepareClipboardData(
std: BlockStdScope
) {
const job = new Job({
collection: std.collection,
schema: std.collection.schema,
blobCRUD: std.collection.blobSync,
docCRUD: {
create: (id: string) => std.collection.createDoc({ id }),
get: (id: string) => std.collection.getDoc(id),
delete: (id: string) => std.collection.removeDoc(id),
},
});
const selected = await Promise.all(
selectedAll.map(async selected => {

View File

@@ -87,7 +87,16 @@ export class TemplateJob {
type: TemplateType;
constructor({ model, type, middlewares }: TemplateJobConfig) {
this.job = new Job({ collection: model.doc.collection, middlewares: [] });
this.job = new Job({
schema: model.doc.collection.schema,
blobCRUD: model.doc.collection.blobSync,
docCRUD: {
create: (id: string) => model.doc.collection.createDoc({ id }),
get: (id: string) => model.doc.collection.getDoc(id),
delete: (id: string) => model.doc.collection.removeDoc(id),
},
middlewares: [],
});
this.model = model;
this.type = TEMPLATE_TYPES.includes(type as TemplateType)
? (type as TemplateType)

View File

@@ -40,7 +40,13 @@ export function getSortedCloneElements(elements: GfxModel[]) {
export function prepareCloneData(elements: GfxModel[], std: BlockStdScope) {
elements = sortEdgelessElements(elements);
const job = new Job({
collection: std.collection,
schema: std.collection.schema,
blobCRUD: std.collection.blobSync,
docCRUD: {
create: (id: string) => std.collection.createDoc({ id }),
get: (id: string) => std.collection.getDoc(id),
delete: (id: string) => std.collection.removeDoc(id),
},
});
const res = elements.map(element => {
const data = serializeElement(element, elements, job);

View File

@@ -247,7 +247,15 @@ export const markdownToMindmap = (
provider: ServiceProvider
) => {
let result: Node | null = null;
const job = new Job({ collection: doc.collection });
const job = new Job({
schema: doc.collection.schema,
blobCRUD: doc.collection.blobSync,
docCRUD: {
create: (id: string) => doc.collection.createDoc({ id }),
get: (id: string) => doc.collection.getDoc(id),
delete: (id: string) => doc.collection.removeDoc(id),
},
});
const markdown = new MarkdownAdapter(job, provider);
const ast: Root = markdown['_markdownToAst'](answer);
const traverse = (