mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-29 08:09:52 +08:00
chore(editor): reorg packages (#10702)
This commit is contained in:
@@ -0,0 +1,203 @@
|
||||
import { defaultImageProxyMiddleware } from '@blocksuite/affine-block-image';
|
||||
import {
|
||||
docLinkBaseURLMiddleware,
|
||||
fileNameMiddleware,
|
||||
HtmlAdapter,
|
||||
titleMiddleware,
|
||||
} from '@blocksuite/affine-shared/adapters';
|
||||
import { SpecProvider } from '@blocksuite/affine-shared/utils';
|
||||
import { Container } from '@blocksuite/global/di';
|
||||
import { sha } from '@blocksuite/global/utils';
|
||||
import type { Schema, Store, Workspace } from '@blocksuite/store';
|
||||
import { extMimeMap, Transformer } from '@blocksuite/store';
|
||||
|
||||
import { createAssetsArchive, download, Unzip } from './utils.js';
|
||||
|
||||
type ImportHTMLToDocOptions = {
|
||||
collection: Workspace;
|
||||
schema: Schema;
|
||||
html: string;
|
||||
fileName?: string;
|
||||
};
|
||||
|
||||
type ImportHTMLZipOptions = {
|
||||
collection: Workspace;
|
||||
schema: Schema;
|
||||
imported: Blob;
|
||||
};
|
||||
|
||||
function getProvider() {
|
||||
const container = new Container();
|
||||
const exts = SpecProvider._.getSpec('store').value;
|
||||
exts.forEach(ext => {
|
||||
ext.setup(container);
|
||||
});
|
||||
return container.provider();
|
||||
}
|
||||
|
||||
/**
|
||||
* Exports a doc to HTML format.
|
||||
*
|
||||
* @param doc - The doc to be exported.
|
||||
* @returns A Promise that resolves when the export is complete.
|
||||
*/
|
||||
async function exportDoc(doc: Store) {
|
||||
const provider = getProvider();
|
||||
const job = doc.getTransformer([
|
||||
docLinkBaseURLMiddleware(doc.workspace.id),
|
||||
titleMiddleware(doc.workspace.meta.docMetas),
|
||||
]);
|
||||
const snapshot = job.docToSnapshot(doc);
|
||||
const adapter = new HtmlAdapter(job, provider);
|
||||
if (!snapshot) {
|
||||
return;
|
||||
}
|
||||
const htmlResult = await adapter.fromDocSnapshot({
|
||||
snapshot,
|
||||
assets: job.assetsManager,
|
||||
});
|
||||
|
||||
let downloadBlob: Blob;
|
||||
const docTitle = doc.meta?.title || 'Untitled';
|
||||
let name: string;
|
||||
const contentBlob = new Blob([htmlResult.file], { type: 'plain/text' });
|
||||
if (htmlResult.assetsIds.length > 0) {
|
||||
const zip = await createAssetsArchive(job.assets, htmlResult.assetsIds);
|
||||
|
||||
await zip.file('index.html', contentBlob);
|
||||
|
||||
downloadBlob = await zip.generate();
|
||||
name = `${docTitle}.zip`;
|
||||
} else {
|
||||
downloadBlob = contentBlob;
|
||||
name = `${docTitle}.html`;
|
||||
}
|
||||
download(downloadBlob, name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports HTML content into a new doc within a collection.
|
||||
*
|
||||
* @param options - The import options.
|
||||
* @param options.collection - The target doc collection.
|
||||
* @param options.schema - The schema of the target doc collection.
|
||||
* @param options.html - The HTML content to import.
|
||||
* @param options.fileName - Optional filename for the imported doc.
|
||||
* @returns A Promise that resolves to the ID of the newly created doc, or undefined if import fails.
|
||||
*/
|
||||
async function importHTMLToDoc({
|
||||
collection,
|
||||
schema,
|
||||
html,
|
||||
fileName,
|
||||
}: ImportHTMLToDocOptions) {
|
||||
const provider = getProvider();
|
||||
const job = new Transformer({
|
||||
schema,
|
||||
blobCRUD: collection.blobSync,
|
||||
docCRUD: {
|
||||
create: (id: string) => collection.createDoc({ id }),
|
||||
get: (id: string) => collection.getDoc(id),
|
||||
delete: (id: string) => collection.removeDoc(id),
|
||||
},
|
||||
middlewares: [
|
||||
defaultImageProxyMiddleware,
|
||||
fileNameMiddleware(fileName),
|
||||
docLinkBaseURLMiddleware(collection.id),
|
||||
],
|
||||
});
|
||||
const htmlAdapter = new HtmlAdapter(job, provider);
|
||||
const page = await htmlAdapter.toDoc({
|
||||
file: html,
|
||||
assets: job.assetsManager,
|
||||
});
|
||||
if (!page) {
|
||||
return;
|
||||
}
|
||||
return page.id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports a zip file containing HTML files and assets into a collection.
|
||||
*
|
||||
* @param options - The import options.
|
||||
* @param options.collection - The target doc collection.
|
||||
* @param options.schema - The schema of the target doc collection.
|
||||
* @param options.imported - The zip file as a Blob.
|
||||
* @returns A Promise that resolves to an array of IDs of the newly created docs.
|
||||
*/
|
||||
async function importHTMLZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
}: ImportHTMLZipOptions) {
|
||||
const provider = getProvider();
|
||||
const unzip = new Unzip();
|
||||
await unzip.load(imported);
|
||||
|
||||
const docIds: string[] = [];
|
||||
const pendingAssets = new Map<string, File>();
|
||||
const pendingPathBlobIdMap = new Map<string, string>();
|
||||
const htmlBlobs: [string, Blob][] = [];
|
||||
|
||||
for (const { path, content: blob } of unzip) {
|
||||
if (path.includes('__MACOSX') || path.includes('.DS_Store')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fileName = path.split('/').pop() ?? '';
|
||||
if (fileName.endsWith('.html')) {
|
||||
htmlBlobs.push([fileName, blob]);
|
||||
} else {
|
||||
const ext = path.split('.').at(-1) ?? '';
|
||||
const mime = extMimeMap.get(ext) ?? '';
|
||||
const key = await sha(await blob.arrayBuffer());
|
||||
pendingPathBlobIdMap.set(path, key);
|
||||
pendingAssets.set(key, new File([blob], fileName, { type: mime }));
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
htmlBlobs.map(async ([fileName, blob]) => {
|
||||
const fileNameWithoutExt = fileName.replace(/\.[^/.]+$/, '');
|
||||
const job = new Transformer({
|
||||
schema,
|
||||
blobCRUD: collection.blobSync,
|
||||
docCRUD: {
|
||||
create: (id: string) => collection.createDoc({ id }),
|
||||
get: (id: string) => collection.getDoc(id),
|
||||
delete: (id: string) => collection.removeDoc(id),
|
||||
},
|
||||
middlewares: [
|
||||
defaultImageProxyMiddleware,
|
||||
fileNameMiddleware(fileNameWithoutExt),
|
||||
docLinkBaseURLMiddleware(collection.id),
|
||||
],
|
||||
});
|
||||
const assets = job.assets;
|
||||
const pathBlobIdMap = job.assetsManager.getPathBlobIdMap();
|
||||
for (const [key, value] of pendingAssets.entries()) {
|
||||
assets.set(key, value);
|
||||
}
|
||||
for (const [key, value] of pendingPathBlobIdMap.entries()) {
|
||||
pathBlobIdMap.set(key, value);
|
||||
}
|
||||
const htmlAdapter = new HtmlAdapter(job, provider);
|
||||
const html = await blob.text();
|
||||
const doc = await htmlAdapter.toDoc({
|
||||
file: html,
|
||||
assets: job.assetsManager,
|
||||
});
|
||||
if (doc) {
|
||||
docIds.push(doc.id);
|
||||
}
|
||||
})
|
||||
);
|
||||
return docIds;
|
||||
}
|
||||
|
||||
export const HtmlTransformer = {
|
||||
exportDoc,
|
||||
importHTMLToDoc,
|
||||
importHTMLZip,
|
||||
};
|
||||
@@ -0,0 +1,5 @@
|
||||
export { HtmlTransformer } from './html.js';
|
||||
export { MarkdownTransformer } from './markdown.js';
|
||||
export { NotionHtmlTransformer } from './notion-html.js';
|
||||
export { createAssetsArchive, download } from './utils.js';
|
||||
export { ZipTransformer } from './zip.js';
|
||||
@@ -0,0 +1,255 @@
|
||||
import { defaultImageProxyMiddleware } from '@blocksuite/affine-block-image';
|
||||
import {
|
||||
docLinkBaseURLMiddleware,
|
||||
fileNameMiddleware,
|
||||
MarkdownAdapter,
|
||||
titleMiddleware,
|
||||
} from '@blocksuite/affine-shared/adapters';
|
||||
import { SpecProvider } from '@blocksuite/affine-shared/utils';
|
||||
import { Container } from '@blocksuite/global/di';
|
||||
import { BlockSuiteError, ErrorCode } from '@blocksuite/global/exceptions';
|
||||
import { sha } from '@blocksuite/global/utils';
|
||||
import type { Schema, Store, Workspace } from '@blocksuite/store';
|
||||
import { extMimeMap, Transformer } from '@blocksuite/store';
|
||||
|
||||
import { createAssetsArchive, download, Unzip } from './utils.js';
|
||||
|
||||
function getProvider() {
|
||||
const container = new Container();
|
||||
const exts = SpecProvider._.getSpec('store').value;
|
||||
exts.forEach(ext => {
|
||||
ext.setup(container);
|
||||
});
|
||||
return container.provider();
|
||||
}
|
||||
|
||||
type ImportMarkdownToBlockOptions = {
|
||||
doc: Store;
|
||||
markdown: string;
|
||||
blockId: string;
|
||||
};
|
||||
|
||||
type ImportMarkdownToDocOptions = {
|
||||
collection: Workspace;
|
||||
schema: Schema;
|
||||
markdown: string;
|
||||
fileName?: string;
|
||||
};
|
||||
|
||||
type ImportMarkdownZipOptions = {
|
||||
collection: Workspace;
|
||||
schema: Schema;
|
||||
imported: Blob;
|
||||
};
|
||||
|
||||
/**
|
||||
* Exports a doc to a Markdown file or a zip archive containing Markdown and assets.
|
||||
* @param doc The doc to export
|
||||
* @returns A Promise that resolves when the export is complete
|
||||
*/
|
||||
async function exportDoc(doc: Store) {
|
||||
const provider = getProvider();
|
||||
const job = doc.getTransformer([
|
||||
docLinkBaseURLMiddleware(doc.workspace.id),
|
||||
titleMiddleware(doc.workspace.meta.docMetas),
|
||||
]);
|
||||
const snapshot = job.docToSnapshot(doc);
|
||||
|
||||
const adapter = new MarkdownAdapter(job, provider);
|
||||
if (!snapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
const markdownResult = await adapter.fromDocSnapshot({
|
||||
snapshot,
|
||||
assets: job.assetsManager,
|
||||
});
|
||||
|
||||
let downloadBlob: Blob;
|
||||
const docTitle = doc.meta?.title || 'Untitled';
|
||||
let name: string;
|
||||
const contentBlob = new Blob([markdownResult.file], { type: 'plain/text' });
|
||||
if (markdownResult.assetsIds.length > 0) {
|
||||
if (!job.assets) {
|
||||
throw new BlockSuiteError(ErrorCode.ValueNotExists, 'No assets found');
|
||||
}
|
||||
const zip = await createAssetsArchive(job.assets, markdownResult.assetsIds);
|
||||
|
||||
await zip.file('index.md', contentBlob);
|
||||
|
||||
downloadBlob = await zip.generate();
|
||||
name = `${docTitle}.zip`;
|
||||
} else {
|
||||
downloadBlob = contentBlob;
|
||||
name = `${docTitle}.md`;
|
||||
}
|
||||
download(downloadBlob, name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports Markdown content into a specific block within a doc.
|
||||
* @param options Object containing import options
|
||||
* @param options.doc The target doc
|
||||
* @param options.markdown The Markdown content to import
|
||||
* @param options.blockId The ID of the block where the content will be imported
|
||||
* @returns A Promise that resolves when the import is complete
|
||||
*/
|
||||
async function importMarkdownToBlock({
|
||||
doc,
|
||||
markdown,
|
||||
blockId,
|
||||
}: ImportMarkdownToBlockOptions) {
|
||||
const provider = getProvider();
|
||||
const job = doc.getTransformer([
|
||||
defaultImageProxyMiddleware,
|
||||
docLinkBaseURLMiddleware(doc.workspace.id),
|
||||
]);
|
||||
const adapter = new MarkdownAdapter(job, provider);
|
||||
const snapshot = await adapter.toSliceSnapshot({
|
||||
file: markdown,
|
||||
assets: job.assetsManager,
|
||||
workspaceId: doc.workspace.id,
|
||||
pageId: doc.id,
|
||||
});
|
||||
|
||||
if (!snapshot) {
|
||||
throw new BlockSuiteError(
|
||||
BlockSuiteError.ErrorCode.ValueNotExists,
|
||||
'import markdown failed, expected to get a snapshot'
|
||||
);
|
||||
}
|
||||
|
||||
const blocks = snapshot.content.flatMap(x => x.children);
|
||||
|
||||
for (const block of blocks) {
|
||||
await job.snapshotToBlock(block, doc, blockId);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports Markdown content into a new doc within a collection.
|
||||
* @param options Object containing import options
|
||||
* @param options.collection The target doc collection
|
||||
* @param options.schema The schema of the target doc collection
|
||||
* @param options.markdown The Markdown content to import
|
||||
* @param options.fileName Optional filename for the imported doc
|
||||
* @returns A Promise that resolves to the ID of the newly created doc, or undefined if import fails
|
||||
*/
|
||||
async function importMarkdownToDoc({
|
||||
collection,
|
||||
schema,
|
||||
markdown,
|
||||
fileName,
|
||||
}: ImportMarkdownToDocOptions) {
|
||||
const provider = getProvider();
|
||||
const job = new Transformer({
|
||||
schema,
|
||||
blobCRUD: collection.blobSync,
|
||||
docCRUD: {
|
||||
create: (id: string) => collection.createDoc({ id }),
|
||||
get: (id: string) => collection.getDoc(id),
|
||||
delete: (id: string) => collection.removeDoc(id),
|
||||
},
|
||||
middlewares: [
|
||||
defaultImageProxyMiddleware,
|
||||
fileNameMiddleware(fileName),
|
||||
docLinkBaseURLMiddleware(collection.id),
|
||||
],
|
||||
});
|
||||
const mdAdapter = new MarkdownAdapter(job, provider);
|
||||
const page = await mdAdapter.toDoc({
|
||||
file: markdown,
|
||||
assets: job.assetsManager,
|
||||
});
|
||||
if (!page) {
|
||||
return;
|
||||
}
|
||||
return page.id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports a zip file containing Markdown files and assets into a collection.
|
||||
* @param options Object containing import options
|
||||
* @param options.collection The target doc collection
|
||||
* @param options.schema The schema of the target doc collection
|
||||
* @param options.imported The zip file as a Blob
|
||||
* @returns A Promise that resolves to an array of IDs of the newly created docs
|
||||
*/
|
||||
async function importMarkdownZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
}: ImportMarkdownZipOptions) {
|
||||
const provider = getProvider();
|
||||
const unzip = new Unzip();
|
||||
await unzip.load(imported);
|
||||
|
||||
const docIds: string[] = [];
|
||||
const pendingAssets = new Map<string, File>();
|
||||
const pendingPathBlobIdMap = new Map<string, string>();
|
||||
const markdownBlobs: [string, Blob][] = [];
|
||||
|
||||
for (const { path, content: blob } of unzip) {
|
||||
if (path.includes('__MACOSX') || path.includes('.DS_Store')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fileName = path.split('/').pop() ?? '';
|
||||
if (fileName.endsWith('.md')) {
|
||||
markdownBlobs.push([fileName, blob]);
|
||||
} else {
|
||||
const ext = path.split('.').at(-1) ?? '';
|
||||
const mime = extMimeMap.get(ext) ?? '';
|
||||
const key = await sha(await blob.arrayBuffer());
|
||||
pendingPathBlobIdMap.set(path, key);
|
||||
pendingAssets.set(key, new File([blob], fileName, { type: mime }));
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
markdownBlobs.map(async ([fileName, blob]) => {
|
||||
const fileNameWithoutExt = fileName.replace(/\.[^/.]+$/, '');
|
||||
const job = new Transformer({
|
||||
schema,
|
||||
blobCRUD: collection.blobSync,
|
||||
docCRUD: {
|
||||
create: (id: string) => collection.createDoc({ id }),
|
||||
get: (id: string) => collection.getDoc(id),
|
||||
delete: (id: string) => collection.removeDoc(id),
|
||||
},
|
||||
middlewares: [
|
||||
defaultImageProxyMiddleware,
|
||||
fileNameMiddleware(fileNameWithoutExt),
|
||||
docLinkBaseURLMiddleware(collection.id),
|
||||
],
|
||||
});
|
||||
const assets = job.assets;
|
||||
const pathBlobIdMap = job.assetsManager.getPathBlobIdMap();
|
||||
for (const [key, value] of pendingAssets.entries()) {
|
||||
assets.set(key, value);
|
||||
}
|
||||
for (const [key, value] of pendingPathBlobIdMap.entries()) {
|
||||
pathBlobIdMap.set(key, value);
|
||||
}
|
||||
const mdAdapter = new MarkdownAdapter(job, provider);
|
||||
const markdown = await blob.text();
|
||||
const doc = await mdAdapter.toDoc({
|
||||
file: markdown,
|
||||
assets: job.assetsManager,
|
||||
});
|
||||
if (doc) {
|
||||
docIds.push(doc.id);
|
||||
}
|
||||
})
|
||||
);
|
||||
return docIds;
|
||||
}
|
||||
|
||||
export const MarkdownTransformer = {
|
||||
exportDoc,
|
||||
importMarkdownToBlock,
|
||||
importMarkdownToDoc,
|
||||
importMarkdownZip,
|
||||
};
|
||||
@@ -0,0 +1,171 @@
|
||||
import { defaultImageProxyMiddleware } from '@blocksuite/affine-block-image';
|
||||
import { NotionHtmlAdapter } from '@blocksuite/affine-shared/adapters';
|
||||
import { SpecProvider } from '@blocksuite/affine-shared/utils';
|
||||
import { Container } from '@blocksuite/global/di';
|
||||
import { sha } from '@blocksuite/global/utils';
|
||||
import {
|
||||
extMimeMap,
|
||||
type Schema,
|
||||
Transformer,
|
||||
type Workspace,
|
||||
} from '@blocksuite/store';
|
||||
|
||||
import { Unzip } from './utils.js';
|
||||
|
||||
type ImportNotionZipOptions = {
|
||||
collection: Workspace;
|
||||
schema: Schema;
|
||||
imported: Blob;
|
||||
};
|
||||
|
||||
function getProvider() {
|
||||
const container = new Container();
|
||||
const exts = SpecProvider._.getSpec('store').value;
|
||||
exts.forEach(ext => {
|
||||
ext.setup(container);
|
||||
});
|
||||
return container.provider();
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports a Notion zip file into the BlockSuite collection.
|
||||
*
|
||||
* @param options - The options for importing.
|
||||
* @param options.collection - The BlockSuite document collection.
|
||||
* @param options.schema - The schema of the BlockSuite document collection.
|
||||
* @param options.imported - The imported zip file as a Blob.
|
||||
*
|
||||
* @returns A promise that resolves to an object containing:
|
||||
* - entryId: The ID of the entry page (if any).
|
||||
* - pageIds: An array of imported page IDs.
|
||||
* - isWorkspaceFile: Whether the imported file is a workspace file.
|
||||
* - hasMarkdown: Whether the zip contains markdown files.
|
||||
*/
|
||||
async function importNotionZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
}: ImportNotionZipOptions) {
|
||||
const provider = getProvider();
|
||||
const pageIds: string[] = [];
|
||||
let isWorkspaceFile = false;
|
||||
let hasMarkdown = false;
|
||||
let entryId: string | undefined;
|
||||
const parseZipFile = async (path: File | Blob) => {
|
||||
const unzip = new Unzip();
|
||||
await unzip.load(path);
|
||||
const zipFile = new Map<string, Blob>();
|
||||
const pageMap = new Map<string, string>();
|
||||
const pagePaths: string[] = [];
|
||||
const promises: Promise<void>[] = [];
|
||||
const pendingAssets = new Map<string, Blob>();
|
||||
const pendingPathBlobIdMap = new Map<string, string>();
|
||||
for (const { path, content, index } of unzip) {
|
||||
if (path.startsWith('__MACOSX/')) continue;
|
||||
|
||||
zipFile.set(path, content);
|
||||
|
||||
const lastSplitIndex = path.lastIndexOf('/');
|
||||
|
||||
const fileName = path.substring(lastSplitIndex + 1);
|
||||
if (fileName.endsWith('.md')) {
|
||||
hasMarkdown = true;
|
||||
continue;
|
||||
}
|
||||
if (fileName.endsWith('.html')) {
|
||||
if (path.endsWith('/index.html')) {
|
||||
isWorkspaceFile = true;
|
||||
continue;
|
||||
}
|
||||
if (lastSplitIndex !== -1) {
|
||||
const text = await content.text();
|
||||
const doc = new DOMParser().parseFromString(text, 'text/html');
|
||||
const pageBody = doc.querySelector('.page-body');
|
||||
if (pageBody && pageBody.children.length === 0) {
|
||||
// Skip empty pages
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const id = collection.idGenerator();
|
||||
const splitPath = path.split('/');
|
||||
while (splitPath.length > 0) {
|
||||
pageMap.set(splitPath.join('/'), id);
|
||||
splitPath.shift();
|
||||
}
|
||||
pagePaths.push(path);
|
||||
if (entryId === undefined && lastSplitIndex === -1) {
|
||||
entryId = id;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (index === 0 && fileName.endsWith('.csv')) {
|
||||
window.open(
|
||||
'https://affine.pro/blog/import-your-data-from-notion-into-affine',
|
||||
'_blank'
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (fileName.endsWith('.zip')) {
|
||||
const innerZipFile = content;
|
||||
if (innerZipFile) {
|
||||
promises.push(...(await parseZipFile(innerZipFile)));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const blob = content;
|
||||
const ext = path.split('.').at(-1) ?? '';
|
||||
const mime = extMimeMap.get(ext) ?? '';
|
||||
const key = await sha(await blob.arrayBuffer());
|
||||
const filePathSplit = path.split('/');
|
||||
while (filePathSplit.length > 1) {
|
||||
pendingPathBlobIdMap.set(filePathSplit.join('/'), key);
|
||||
filePathSplit.shift();
|
||||
}
|
||||
pendingAssets.set(key, new File([blob], fileName, { type: mime }));
|
||||
}
|
||||
const pagePromises = Array.from(pagePaths).map(async path => {
|
||||
const job = new Transformer({
|
||||
schema,
|
||||
blobCRUD: collection.blobSync,
|
||||
docCRUD: {
|
||||
create: (id: string) => collection.createDoc({ id }),
|
||||
get: (id: string) => collection.getDoc(id),
|
||||
delete: (id: string) => collection.removeDoc(id),
|
||||
},
|
||||
middlewares: [defaultImageProxyMiddleware],
|
||||
});
|
||||
const htmlAdapter = new NotionHtmlAdapter(job, provider);
|
||||
const assets = job.assetsManager.getAssets();
|
||||
const pathBlobIdMap = job.assetsManager.getPathBlobIdMap();
|
||||
for (const [key, value] of pendingAssets.entries()) {
|
||||
if (!assets.has(key)) {
|
||||
assets.set(key, value);
|
||||
}
|
||||
}
|
||||
for (const [key, value] of pendingPathBlobIdMap.entries()) {
|
||||
if (!pathBlobIdMap.has(key)) {
|
||||
pathBlobIdMap.set(key, value);
|
||||
}
|
||||
}
|
||||
const page = await htmlAdapter.toDoc({
|
||||
file: await zipFile.get(path)!.text(),
|
||||
pageId: pageMap.get(path),
|
||||
pageMap,
|
||||
assets: job.assetsManager,
|
||||
});
|
||||
if (page) {
|
||||
pageIds.push(page.id);
|
||||
}
|
||||
});
|
||||
promises.push(...pagePromises);
|
||||
return promises;
|
||||
};
|
||||
const allPromises = await parseZipFile(imported);
|
||||
await Promise.all(allPromises.flat());
|
||||
entryId = entryId ?? pageIds[0];
|
||||
return { entryId, pageIds, isWorkspaceFile, hasMarkdown };
|
||||
}
|
||||
|
||||
export const NotionHtmlTransformer = {
|
||||
importNotionZip,
|
||||
};
|
||||
@@ -0,0 +1,115 @@
|
||||
import { extMimeMap, getAssetName } from '@blocksuite/store';
|
||||
import * as fflate from 'fflate';
|
||||
|
||||
export class Zip {
|
||||
private compressed = new Uint8Array();
|
||||
|
||||
private finalize?: () => void;
|
||||
|
||||
private finalized = false;
|
||||
|
||||
private readonly zip = new fflate.Zip((err, chunk, final) => {
|
||||
if (!err) {
|
||||
const temp = new Uint8Array(this.compressed.length + chunk.length);
|
||||
temp.set(this.compressed);
|
||||
temp.set(chunk, this.compressed.length);
|
||||
this.compressed = temp;
|
||||
}
|
||||
if (final) {
|
||||
this.finalized = true;
|
||||
this.finalize?.();
|
||||
}
|
||||
});
|
||||
|
||||
async file(path: string, content: Blob | File | string) {
|
||||
const deflate = new fflate.ZipDeflate(path);
|
||||
this.zip.add(deflate);
|
||||
if (typeof content === 'string') {
|
||||
deflate.push(fflate.strToU8(content), true);
|
||||
} else {
|
||||
deflate.push(new Uint8Array(await content.arrayBuffer()), true);
|
||||
}
|
||||
}
|
||||
|
||||
folder(folderPath: string) {
|
||||
return {
|
||||
folder: (folderPath2: string) => {
|
||||
return this.folder(`${folderPath}/${folderPath2}`);
|
||||
},
|
||||
file: async (name: string, blob: Blob) => {
|
||||
await this.file(`${folderPath}/${name}`, blob);
|
||||
},
|
||||
generate: async () => {
|
||||
return this.generate();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async generate() {
|
||||
this.zip.end();
|
||||
return new Promise<Blob>(resolve => {
|
||||
if (this.finalized) {
|
||||
resolve(new Blob([this.compressed], { type: 'application/zip' }));
|
||||
} else {
|
||||
this.finalize = () =>
|
||||
resolve(new Blob([this.compressed], { type: 'application/zip' }));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export class Unzip {
|
||||
private unzipped?: ReturnType<typeof fflate.unzipSync>;
|
||||
|
||||
async load(blob: Blob) {
|
||||
this.unzipped = fflate.unzipSync(new Uint8Array(await blob.arrayBuffer()));
|
||||
}
|
||||
|
||||
*[Symbol.iterator]() {
|
||||
const keys = Object.keys(this.unzipped ?? {});
|
||||
let index = 0;
|
||||
while (keys.length) {
|
||||
const path = keys.shift()!;
|
||||
if (path.includes('__MACOSX') || path.includes('DS_Store')) {
|
||||
continue;
|
||||
}
|
||||
const lastSplitIndex = path.lastIndexOf('/');
|
||||
const fileName = path.substring(lastSplitIndex + 1);
|
||||
const fileExt =
|
||||
fileName.lastIndexOf('.') === -1 ? '' : fileName.split('.').at(-1);
|
||||
const mime = extMimeMap.get(fileExt ?? '');
|
||||
const content = new File([this.unzipped![path]], fileName, {
|
||||
type: mime ?? '',
|
||||
}) as Blob;
|
||||
yield { path, content, index };
|
||||
index++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function createAssetsArchive(
|
||||
assetsMap: Map<string, Blob>,
|
||||
assetsIds: string[]
|
||||
) {
|
||||
const zip = new Zip();
|
||||
|
||||
for (const [id, blob] of assetsMap) {
|
||||
if (!assetsIds.includes(id)) continue;
|
||||
const name = getAssetName(assetsMap, id);
|
||||
await zip.folder('assets').file(name, blob);
|
||||
}
|
||||
|
||||
return zip;
|
||||
}
|
||||
|
||||
export function download(blob: Blob, name: string) {
|
||||
const element = document.createElement('a');
|
||||
element.setAttribute('download', name);
|
||||
const fileURL = URL.createObjectURL(blob);
|
||||
element.setAttribute('href', fileURL);
|
||||
element.style.display = 'none';
|
||||
document.body.append(element);
|
||||
element.click();
|
||||
element.remove();
|
||||
URL.revokeObjectURL(fileURL);
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
import {
|
||||
replaceIdMiddleware,
|
||||
titleMiddleware,
|
||||
} from '@blocksuite/affine-shared/adapters';
|
||||
import { sha } from '@blocksuite/global/utils';
|
||||
import type { DocSnapshot, Schema, Store, Workspace } from '@blocksuite/store';
|
||||
import { extMimeMap, getAssetName, Transformer } from '@blocksuite/store';
|
||||
|
||||
import { download, Unzip, Zip } from '../transformers/utils.js';
|
||||
|
||||
async function exportDocs(
|
||||
collection: Workspace,
|
||||
schema: Schema,
|
||||
docs: Store[]
|
||||
) {
|
||||
const zip = new Zip();
|
||||
const job = new Transformer({
|
||||
schema,
|
||||
blobCRUD: collection.blobSync,
|
||||
docCRUD: {
|
||||
create: (id: string) => collection.createDoc({ id }),
|
||||
get: (id: string) => collection.getDoc(id),
|
||||
delete: (id: string) => collection.removeDoc(id),
|
||||
},
|
||||
middlewares: [
|
||||
replaceIdMiddleware(collection.idGenerator),
|
||||
titleMiddleware(collection.meta.docMetas),
|
||||
],
|
||||
});
|
||||
const snapshots = await Promise.all(docs.map(job.docToSnapshot));
|
||||
|
||||
await Promise.all(
|
||||
snapshots
|
||||
.filter((snapshot): snapshot is DocSnapshot => !!snapshot)
|
||||
.map(async snapshot => {
|
||||
// Use the title and id as the snapshot file name
|
||||
const title = snapshot.meta.title || 'untitled';
|
||||
const id = snapshot.meta.id;
|
||||
const snapshotName = `${title}-${id}.snapshot.json`;
|
||||
await zip.file(snapshotName, JSON.stringify(snapshot, null, 2));
|
||||
})
|
||||
);
|
||||
|
||||
const assets = zip.folder('assets');
|
||||
const pathBlobIdMap = job.assetsManager.getPathBlobIdMap();
|
||||
const assetsMap = job.assets;
|
||||
|
||||
// Add blobs to assets folder, if failed, log the error and continue
|
||||
const results = await Promise.all(
|
||||
Array.from(pathBlobIdMap.values()).map(async blobId => {
|
||||
try {
|
||||
await job.assetsManager.readFromBlob(blobId);
|
||||
const ext = getAssetName(assetsMap, blobId).split('.').at(-1);
|
||||
const blob = assetsMap.get(blobId);
|
||||
if (blob) {
|
||||
await assets.file(`${blobId}.${ext}`, blob);
|
||||
return { success: true, blobId };
|
||||
}
|
||||
return { success: false, blobId, error: 'Blob not found' };
|
||||
} catch (error) {
|
||||
console.error(`Failed to process blob: ${blobId}`, error);
|
||||
return { success: false, blobId, error };
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
const failures = results.filter(r => !r.success);
|
||||
if (failures.length > 0) {
|
||||
console.warn(`Failed to process ${failures.length} blobs:`, failures);
|
||||
}
|
||||
|
||||
const downloadBlob = await zip.generate();
|
||||
// Use the collection id as the zip file name
|
||||
return download(downloadBlob, `${collection.id}.bs.zip`);
|
||||
}
|
||||
|
||||
async function importDocs(
|
||||
collection: Workspace,
|
||||
schema: Schema,
|
||||
imported: Blob
|
||||
) {
|
||||
const unzip = new Unzip();
|
||||
await unzip.load(imported);
|
||||
|
||||
const assetBlobs: [string, Blob][] = [];
|
||||
const snapshotsBlobs: Blob[] = [];
|
||||
|
||||
for (const { path, content: blob } of unzip) {
|
||||
if (path.includes('MACOSX') || path.includes('DS_Store')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (path.startsWith('assets/')) {
|
||||
assetBlobs.push([path, blob]);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (path === 'info.json') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (path.endsWith('.snapshot.json')) {
|
||||
snapshotsBlobs.push(blob);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const job = new Transformer({
|
||||
schema,
|
||||
blobCRUD: collection.blobSync,
|
||||
docCRUD: {
|
||||
create: (id: string) => collection.createDoc({ id }),
|
||||
get: (id: string) => collection.getDoc(id),
|
||||
delete: (id: string) => collection.removeDoc(id),
|
||||
},
|
||||
middlewares: [
|
||||
replaceIdMiddleware(collection.idGenerator),
|
||||
titleMiddleware(collection.meta.docMetas),
|
||||
],
|
||||
});
|
||||
const assetsMap = job.assets;
|
||||
|
||||
assetBlobs.forEach(([name, blob]) => {
|
||||
const nameWithExt = name.replace('assets/', '');
|
||||
const assetsId = nameWithExt.replace(/\.[^/.]+$/, '');
|
||||
const ext = nameWithExt.split('.').at(-1) ?? '';
|
||||
const mime = extMimeMap.get(ext) ?? '';
|
||||
const file = new File([blob], nameWithExt, {
|
||||
type: mime,
|
||||
});
|
||||
assetsMap.set(assetsId, file);
|
||||
});
|
||||
|
||||
return Promise.all(
|
||||
snapshotsBlobs.map(async blob => {
|
||||
const json = await blob.text();
|
||||
const snapshot = JSON.parse(json) as DocSnapshot;
|
||||
const tasks: Promise<void>[] = [];
|
||||
|
||||
job.walk(snapshot, block => {
|
||||
const sourceId = block.props?.sourceId as string | undefined;
|
||||
|
||||
if (sourceId && sourceId.startsWith('/')) {
|
||||
const removeSlashId = sourceId.replace(/^\//, '');
|
||||
|
||||
if (assetsMap.has(removeSlashId)) {
|
||||
const blob = assetsMap.get(removeSlashId)!;
|
||||
|
||||
tasks.push(
|
||||
blob
|
||||
.arrayBuffer()
|
||||
.then(buffer => sha(buffer))
|
||||
.then(hash => {
|
||||
assetsMap.set(hash, blob);
|
||||
block.props.sourceId = hash;
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(tasks);
|
||||
|
||||
return job.snapshotToDoc(snapshot);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export const ZipTransformer = {
|
||||
exportDocs,
|
||||
importDocs,
|
||||
};
|
||||
Reference in New Issue
Block a user