feat: add new tool

This commit is contained in:
DarkSky
2026-02-01 04:37:13 +08:00
parent 759aa1b684
commit b49e48b467
12 changed files with 800 additions and 2 deletions

3
.gitignore vendored
View File

@@ -47,8 +47,7 @@ testem.log
.pnpm-debug.log .pnpm-debug.log
/typings /typings
tsconfig.tsbuildinfo tsconfig.tsbuildinfo
rfc*.md .context
todo.md
# System Files # System Files
.DS_Store .DS_Store

View File

@@ -0,0 +1,22 @@
{
"name": "@affine/doc-diff",
"version": "0.26.0",
"private": true,
"type": "module",
"description": "Diff AFFiNE Yjs snapshot docs (rootdoc/folder/favorite)",
"main": "index.ts",
"scripts": {
"diff": "r ./src/index.ts",
"diff:root": "r ./src/index.ts root",
"diff:folder": "r ./src/index.ts folder",
"diff:favorite": "r ./src/index.ts favorite"
},
"dependencies": {
"@affine-tools/cli": "workspace:*",
"typescript": "^5.7.2",
"yjs": "^13.6.27"
},
"devDependencies": {
"@types/node": "^22.0.0"
}
}

View File

@@ -0,0 +1,83 @@
import { isEqual, MISSING, type PlainValue } from './plain';
export type KeyedRecord = { [key: string]: PlainValue };
export type FieldChange = {
key: string;
from: PlainValue | typeof MISSING;
to: PlainValue | typeof MISSING;
};
export type RecordChange = {
id: string;
fromRecord: KeyedRecord;
toRecord: KeyedRecord;
fields: FieldChange[];
};
export type KeyedDiff = {
added: { id: string; record: KeyedRecord }[];
removed: { id: string; record: KeyedRecord }[];
changed: RecordChange[];
};
export function diffKeyedRecords(
fromRecords: Map<string, KeyedRecord>,
toRecords: Map<string, KeyedRecord>
): KeyedDiff {
const added: { id: string; record: KeyedRecord }[] = [];
const removed: { id: string; record: KeyedRecord }[] = [];
const changed: RecordChange[] = [];
const fromIds = new Set(fromRecords.keys());
const toIds = new Set(toRecords.keys());
for (const id of Array.from(toIds).sort()) {
if (!fromIds.has(id)) {
const record = toRecords.get(id);
if (record) {
added.push({ id, record });
}
}
}
for (const id of Array.from(fromIds).sort()) {
if (!toIds.has(id)) {
const record = fromRecords.get(id);
if (record) {
removed.push({ id, record });
}
}
}
for (const id of Array.from(fromIds).sort()) {
if (!toIds.has(id)) {
continue;
}
const fromRecord = fromRecords.get(id);
const toRecord = toRecords.get(id);
if (!fromRecord || !toRecord) {
continue;
}
const keys = Array.from(
new Set([...Object.keys(fromRecord), ...Object.keys(toRecord)])
).sort();
const fields: FieldChange[] = [];
for (const key of keys) {
const fromValue = Object.hasOwn(fromRecord, key)
? fromRecord[key]
: MISSING;
const toValue = Object.hasOwn(toRecord, key) ? toRecord[key] : MISSING;
if (!isEqual(fromValue, toValue)) {
fields.push({ key, from: fromValue, to: toValue });
}
}
if (fields.length) {
changed.push({ id, fromRecord, toRecord, fields });
}
}
return { added, removed, changed };
}

143
tools/doc-diff/src/index.ts Normal file
View File

@@ -0,0 +1,143 @@
import path from 'node:path';
import { getDisplayLabel, readYjsDocFromFile } from './io';
import { extractRootDocPagesMeta, printRootDocPairDiff } from './rootdoc';
import {
extractYjsTable,
printFavoritePairDiff,
printFolderPairDiff,
} from './table';
type Mode = 'rootdoc' | 'folder' | 'favorite';
const HELP_TEXT = `
Diff AFFiNE Yjs snapshot docs between multiple binaries.
Usage:
# Root doc: diff meta.pages
r ./tools/doc-diff/index.ts <rootDoc1.bin> <rootDoc2.bin> [...more]
r ./tools/doc-diff/index.ts root <rootDoc1.bin> <rootDoc2.bin> [...more]
# Organize: diff db$...$folder (table doc)
r ./tools/doc-diff/index.ts folder <folderDoc1.bin> <folderDoc2.bin> [...more]
# Favorites: diff userdata$...$favorite (table doc)
r ./tools/doc-diff/index.ts favorite <favoriteDoc1.bin> <favoriteDoc2.bin> [...more]
Notes:
- Every argument after the optional subcommand is treated as a file path (relative or absolute).
- Files must be the same doc type (no mixing).
`.trim();
function fail(message: string): never {
console.error(message);
console.error('');
console.error(HELP_TEXT);
process.exit(1);
}
function parseMode(value: string): Mode | null {
switch (value.toLowerCase()) {
case 'root':
case 'rootdoc':
case 'meta':
return 'rootdoc';
case 'folder':
case 'folders':
return 'folder';
case 'favorite':
case 'favourite':
case 'favorites':
return 'favorite';
default:
return null;
}
}
function parseArgs(argv: string[]) {
if (argv.some(arg => arg === '-h' || arg === '--help')) {
console.log(HELP_TEXT);
process.exit(0);
}
let mode: Mode = 'rootdoc';
let cursor = 0;
const maybeMode = argv[cursor];
if (maybeMode) {
if (maybeMode.startsWith('-')) {
fail(`Unknown argument: ${maybeMode}`);
}
const parsed = parseMode(maybeMode);
if (parsed) {
mode = parsed;
cursor += 1;
}
}
const files = argv.slice(cursor);
for (const file of files) {
if (file.startsWith('-')) {
fail(`Unknown argument: ${file}`);
}
}
if (files.length < 2) {
fail('Please provide at least two snapshot file paths.');
}
return { mode, files };
}
const { mode, files } = parseArgs(process.argv.slice(2));
const resolvedFiles = files.map(f => path.resolve(process.cwd(), f));
const docs = resolvedFiles.map(filePath => {
try {
return readYjsDocFromFile(filePath);
} catch (error) {
const details = error instanceof Error ? error.message : String(error);
fail(`Failed to read/parse snapshot file "${filePath}": ${details}`);
}
});
switch (mode) {
case 'rootdoc': {
const metas = docs.map(extractRootDocPagesMeta);
for (let i = 0; i < files.length - 1; i += 1) {
printRootDocPairDiff({
fromLabel: getDisplayLabel(files[i]!),
toLabel: getDisplayLabel(files[i + 1]!),
fromMeta: metas[i]!,
toMeta: metas[i + 1]!,
});
}
break;
}
case 'folder': {
const tables = docs.map(doc => extractYjsTable(doc, 'id'));
for (let i = 0; i < files.length - 1; i += 1) {
printFolderPairDiff({
fromLabel: getDisplayLabel(files[i]!),
toLabel: getDisplayLabel(files[i + 1]!),
fromTable: tables[i]!,
toTable: tables[i + 1]!,
});
}
break;
}
case 'favorite': {
const tables = docs.map(doc => extractYjsTable(doc, 'key'));
for (let i = 0; i < files.length - 1; i += 1) {
printFavoritePairDiff({
fromLabel: getDisplayLabel(files[i]!),
toLabel: getDisplayLabel(files[i + 1]!),
fromTable: tables[i]!,
toTable: tables[i + 1]!,
});
}
break;
}
default: {
mode satisfies never;
fail(`Unknown mode: ${mode}`);
}
}

19
tools/doc-diff/src/io.ts Normal file
View File

@@ -0,0 +1,19 @@
import { readFileSync } from 'node:fs';
import path from 'node:path';
import { applyUpdate, Doc } from 'yjs';
export function getDisplayLabel(inputPath: string) {
const resolved = path.resolve(process.cwd(), inputPath);
if (resolved === inputPath) {
return inputPath;
}
return `${inputPath} (${resolved})`;
}
export function readYjsDocFromFile(filePath: string): Doc {
const bin = readFileSync(filePath);
const doc = new Doc();
applyUpdate(doc, new Uint8Array(bin));
return doc;
}

136
tools/doc-diff/src/plain.ts Normal file
View File

@@ -0,0 +1,136 @@
import { Array as YArray, Doc, Map as YMap, Text } from 'yjs';
export type PlainValue =
| null
| boolean
| number
| string
| PlainValue[]
| { [key: string]: PlainValue };
export const MISSING: unique symbol = Symbol('missing');
export function toPlain(value: unknown): PlainValue {
if (value === null) {
return null;
}
switch (typeof value) {
case 'boolean':
case 'number':
case 'string':
return value;
case 'undefined':
return null;
}
if (value instanceof Text) {
return value.toString();
}
if (value instanceof Doc) {
return { __type: 'YDoc', guid: value.guid };
}
if (value instanceof YArray) {
return value.toArray().map(toPlain);
}
if (value instanceof YMap) {
const keys = Array.from(value.keys()).sort();
const obj: Record<string, PlainValue> = {};
for (const key of keys) {
obj[key] = toPlain(value.get(key));
}
return obj;
}
if (value instanceof Uint8Array) {
return Buffer.from(value).toString('base64');
}
if (Array.isArray(value)) {
return value.map(toPlain);
}
if (value && typeof value === 'object') {
const record = value as Record<string, unknown>;
const keys = Object.keys(record).sort();
const obj: Record<string, PlainValue> = {};
for (const key of keys) {
obj[key] = toPlain(record[key]);
}
return obj;
}
return String(value);
}
function stableComparable(value: unknown): unknown {
if (value === MISSING) {
return { __missing: true };
}
if (value === null || typeof value !== 'object') {
if (value === undefined) {
return { __undefined: true };
}
return value;
}
if (value instanceof Uint8Array) {
return { __uint8array_base64: Buffer.from(value).toString('base64') };
}
if (Array.isArray(value)) {
return value.map(stableComparable);
}
const record = value as Record<string, unknown>;
const keys = Object.keys(record).sort();
const obj: Record<string, unknown> = {};
for (const key of keys) {
obj[key] = stableComparable(record[key]);
}
return obj;
}
export function isEqual(a: unknown, b: unknown): boolean {
return (
JSON.stringify(stableComparable(a)) === JSON.stringify(stableComparable(b))
);
}
export function truncate(s: string, maxLen = 200) {
if (s.length <= maxLen) {
return s;
}
return `${s.slice(0, Math.max(0, maxLen - 3))}...`;
}
function isLikelyTimestampKey(key: string) {
const lower = key.toLowerCase();
return (
lower.endsWith('date') ||
lower.endsWith('at') ||
lower === 'created' ||
lower === 'updated' ||
lower.includes('timestamp')
);
}
function formatTimestampMillis(ms: number) {
if (!Number.isFinite(ms)) {
return String(ms);
}
const date = new Date(ms);
if (Number.isNaN(date.getTime())) {
return String(ms);
}
return `${ms} (${date.toISOString()})`;
}
export function formatValue(key: string, value: PlainValue | typeof MISSING) {
if (value === MISSING) {
return '<missing>';
}
if (typeof value === 'number' && isLikelyTimestampKey(key)) {
return formatTimestampMillis(value);
}
return truncate(JSON.stringify(value));
}

View File

@@ -0,0 +1,144 @@
import { Array as YArray, Doc, Map as YMap } from 'yjs';
import { diffKeyedRecords, type KeyedDiff, type KeyedRecord } from './diff';
import { formatValue, toPlain } from './plain';
type RootDocMetaExtract = {
recordsById: Map<string, KeyedRecord>;
duplicateIds: string[];
};
export function extractRootDocPagesMeta(rootDoc: Doc): RootDocMetaExtract {
const recordsById = new Map<string, KeyedRecord>();
const duplicateIds: string[] = [];
const meta = rootDoc.getMap('meta');
const pages = meta.get('pages');
const entries =
pages instanceof YArray
? pages.toArray()
: Array.isArray(pages)
? pages
: null;
if (!entries) {
return { recordsById, duplicateIds };
}
for (const entry of entries) {
let id: string | null = null;
if (entry instanceof YMap) {
const idRaw = entry.get('id');
id =
typeof idRaw === 'string'
? idRaw
: typeof idRaw?.toString === 'function'
? idRaw.toString()
: null;
} else if (entry && typeof entry === 'object' && !Array.isArray(entry)) {
const record = entry as Record<string, unknown>;
id =
typeof record.id === 'string'
? record.id
: (record.id?.toString?.() ?? null);
}
if (!id) {
continue;
}
if (recordsById.has(id)) {
duplicateIds.push(id);
}
const metaPlain = toPlain(entry) as unknown;
if (
metaPlain &&
typeof metaPlain === 'object' &&
!Array.isArray(metaPlain)
) {
recordsById.set(id, metaPlain as KeyedRecord);
}
}
return { recordsById, duplicateIds };
}
function docLabel(id: string, meta: KeyedRecord) {
const title = meta.title;
if (typeof title === 'string' && title.trim()) {
return `${id} "${title}"`;
}
return id;
}
export function printRootDocPairDiff(opts: {
fromLabel: string;
toLabel: string;
fromMeta: RootDocMetaExtract;
toMeta: RootDocMetaExtract;
}) {
const diff = diffKeyedRecords(
opts.fromMeta.recordsById,
opts.toMeta.recordsById
);
console.log(`\n=== ${opts.fromLabel} -> ${opts.toLabel} ===`);
console.log(
`Docs: ${opts.fromMeta.recordsById.size} -> ${opts.toMeta.recordsById.size} (+${diff.added.length} / -${diff.removed.length} / ~${diff.changed.length})`
);
if (opts.fromMeta.duplicateIds.length) {
console.log(
`! Warning: duplicate page ids in FROM: ${Array.from(new Set(opts.fromMeta.duplicateIds)).sort().join(', ')}`
);
}
if (opts.toMeta.duplicateIds.length) {
console.log(
`! Warning: duplicate page ids in TO: ${Array.from(new Set(opts.toMeta.duplicateIds)).sort().join(', ')}`
);
}
if (diff.added.length) {
console.log(`\n+ Added (${diff.added.length})`);
for (const { id, record } of diff.added) {
console.log(` + ${docLabel(id, record)}`);
}
}
if (diff.removed.length) {
console.log(`\n- Removed (${diff.removed.length})`);
for (const { id, record } of diff.removed) {
console.log(` - ${docLabel(id, record)}`);
}
}
if (diff.changed.length) {
console.log(`\n~ Changed (${diff.changed.length})`);
for (const change of diff.changed) {
const fromTitle = change.fromRecord.title;
const toTitle = change.toRecord.title;
let header = ` ~ ${change.id}`;
if (
typeof fromTitle === 'string' &&
typeof toTitle === 'string' &&
fromTitle !== toTitle
) {
header += ` "${fromTitle}" -> "${toTitle}"`;
} else if (typeof toTitle === 'string' && toTitle.trim()) {
header += ` "${toTitle}"`;
}
console.log(header);
for (const field of change.fields) {
console.log(
` - ${field.key}: ${formatValue(field.key, field.from)} -> ${formatValue(field.key, field.to)}`
);
}
}
}
if (!diff.added.length && !diff.removed.length && !diff.changed.length) {
console.log('\n(no changes)');
}
}

226
tools/doc-diff/src/table.ts Normal file
View File

@@ -0,0 +1,226 @@
import { Doc, Map as YMap } from 'yjs';
import { diffKeyedRecords, type KeyedRecord } from './diff';
import { formatValue, toPlain } from './plain';
const DELETE_FLAG_KEY = '$$DELETED';
type TableExtract = {
recordsByKey: Map<string, KeyedRecord>;
duplicateKeys: string[];
};
function getRecordKey(record: YMap<any>, keyField: string): string | null {
const keyRaw = record.get(keyField);
if (typeof keyRaw === 'string') {
return keyRaw;
}
if (typeof keyRaw?.toString === 'function') {
return keyRaw.toString();
}
return null;
}
function isDeletedRecord(record: YMap<any>): boolean {
return record.get(DELETE_FLAG_KEY) === true || record.size === 0;
}
export function extractYjsTable(doc: Doc, keyField: string): TableExtract {
const recordsByKey = new Map<string, KeyedRecord>();
const duplicateKeys: string[] = [];
for (const sharedKey of doc.share.keys()) {
let record: unknown;
try {
record = doc.getMap(sharedKey);
} catch {
// Not a YMap shared type; ignore.
continue;
}
if (!(record instanceof YMap)) {
continue;
}
const key = getRecordKey(record, keyField);
if (!key) {
continue;
}
if (isDeletedRecord(record)) {
continue;
}
if (recordsByKey.has(key)) {
duplicateKeys.push(key);
}
const recordPlain = toPlain(record) as unknown;
if (
recordPlain &&
typeof recordPlain === 'object' &&
!Array.isArray(recordPlain)
) {
recordsByKey.set(key, recordPlain as KeyedRecord);
}
}
return { recordsByKey, duplicateKeys };
}
function folderLabel(id: string, record: KeyedRecord) {
const type = record.type;
const data = record.data;
if (type === 'folder') {
const name = typeof data === 'string' && data.trim() ? `"${data}"` : '';
return `folder:${id}${name ? ` ${name}` : ''}`;
}
if (typeof type === 'string' && typeof data === 'string') {
return `${type}:${data} (id=${id})`;
}
return `id=${id}`;
}
function favoriteLabel(key: string) {
return key;
}
export function printFolderPairDiff(opts: {
fromLabel: string;
toLabel: string;
fromTable: TableExtract;
toTable: TableExtract;
}) {
const diff = diffKeyedRecords(
opts.fromTable.recordsByKey,
opts.toTable.recordsByKey
);
console.log(`\n=== ${opts.fromLabel} -> ${opts.toLabel} ===`);
console.log(
`Rows: ${opts.fromTable.recordsByKey.size} -> ${opts.toTable.recordsByKey.size} (+${diff.added.length} / -${diff.removed.length} / ~${diff.changed.length})`
);
if (opts.fromTable.duplicateKeys.length) {
console.log(
`! Warning: duplicate keys in FROM: ${Array.from(new Set(opts.fromTable.duplicateKeys)).sort().join(', ')}`
);
}
if (opts.toTable.duplicateKeys.length) {
console.log(
`! Warning: duplicate keys in TO: ${Array.from(new Set(opts.toTable.duplicateKeys)).sort().join(', ')}`
);
}
if (diff.added.length) {
console.log(`\n+ Added (${diff.added.length})`);
for (const { id, record } of diff.added) {
const parentId = record.parentId;
const index = record.index;
console.log(
` + ${folderLabel(id, record)} (parentId=${formatValue('parentId', parentId ?? null)}, index=${formatValue('index', index ?? null)})`
);
}
}
if (diff.removed.length) {
console.log(`\n- Removed (${diff.removed.length})`);
for (const { id, record } of diff.removed) {
const parentId = record.parentId;
const index = record.index;
console.log(
` - ${folderLabel(id, record)} (parentId=${formatValue('parentId', parentId ?? null)}, index=${formatValue('index', index ?? null)})`
);
}
}
if (diff.changed.length) {
console.log(`\n~ Changed (${diff.changed.length})`);
for (const change of diff.changed) {
const fromName =
change.fromRecord.type === 'folder' ? change.fromRecord.data : null;
const toName =
change.toRecord.type === 'folder' ? change.toRecord.data : null;
let header = ` ~ ${folderLabel(change.id, change.toRecord)}`;
if (
typeof fromName === 'string' &&
typeof toName === 'string' &&
fromName !== toName
) {
header += ` ("${fromName}" -> "${toName}")`;
}
console.log(header);
for (const field of change.fields) {
console.log(
` - ${field.key}: ${formatValue(field.key, field.from)} -> ${formatValue(field.key, field.to)}`
);
}
}
}
if (!diff.added.length && !diff.removed.length && !diff.changed.length) {
console.log('\n(no changes)');
}
}
export function printFavoritePairDiff(opts: {
fromLabel: string;
toLabel: string;
fromTable: TableExtract;
toTable: TableExtract;
}) {
const diff = diffKeyedRecords(
opts.fromTable.recordsByKey,
opts.toTable.recordsByKey
);
console.log(`\n=== ${opts.fromLabel} -> ${opts.toLabel} ===`);
console.log(
`Rows: ${opts.fromTable.recordsByKey.size} -> ${opts.toTable.recordsByKey.size} (+${diff.added.length} / -${diff.removed.length} / ~${diff.changed.length})`
);
if (opts.fromTable.duplicateKeys.length) {
console.log(
`! Warning: duplicate keys in FROM: ${Array.from(new Set(opts.fromTable.duplicateKeys)).sort().join(', ')}`
);
}
if (opts.toTable.duplicateKeys.length) {
console.log(
`! Warning: duplicate keys in TO: ${Array.from(new Set(opts.toTable.duplicateKeys)).sort().join(', ')}`
);
}
if (diff.added.length) {
console.log(`\n+ Added (${diff.added.length})`);
for (const { id, record } of diff.added) {
console.log(
` + ${favoriteLabel(id)} (index=${formatValue('index', record.index ?? null)})`
);
}
}
if (diff.removed.length) {
console.log(`\n- Removed (${diff.removed.length})`);
for (const { id, record } of diff.removed) {
console.log(
` - ${favoriteLabel(id)} (index=${formatValue('index', record.index ?? null)})`
);
}
}
if (diff.changed.length) {
console.log(`\n~ Changed (${diff.changed.length})`);
for (const change of diff.changed) {
console.log(` ~ ${favoriteLabel(change.id)}`);
for (const field of change.fields) {
console.log(
` - ${field.key}: ${formatValue(field.key, field.from)} -> ${formatValue(field.key, field.to)}`
);
}
}
}
if (!diff.added.length && !diff.removed.length && !diff.changed.length) {
console.log('\n(no changes)');
}
}

View File

@@ -0,0 +1,8 @@
{
"extends": "../../tsconfig.node.json",
"compilerOptions": {
"outDir": "dist"
},
"files": ["index.ts"],
"references": [{ "path": "../cli" }]
}

View File

@@ -1474,6 +1474,11 @@ export const PackageList = [
name: '@affine/copilot-result', name: '@affine/copilot-result',
workspaceDependencies: [], workspaceDependencies: [],
}, },
{
location: 'tools/doc-diff',
name: '@affine/doc-diff',
workspaceDependencies: ['tools/cli'],
},
{ {
location: 'tools/playstore-auto-bump', location: 'tools/playstore-auto-bump',
name: '@affine/playstore-auto-bump', name: '@affine/playstore-auto-bump',
@@ -1605,6 +1610,7 @@ export type PackageName =
| '@affine-tools/cli' | '@affine-tools/cli'
| '@affine/commitlint-config' | '@affine/commitlint-config'
| '@affine/copilot-result' | '@affine/copilot-result'
| '@affine/doc-diff'
| '@affine/playstore-auto-bump' | '@affine/playstore-auto-bump'
| '@affine/revert-update' | '@affine/revert-update'
| '@affine-tools/utils'; | '@affine-tools/utils';

View File

@@ -156,6 +156,7 @@
{ "path": "./tests/blocksuite" }, { "path": "./tests/blocksuite" },
{ "path": "./tests/kit" }, { "path": "./tests/kit" },
{ "path": "./tools/cli" }, { "path": "./tools/cli" },
{ "path": "./tools/doc-diff" },
{ "path": "./tools/playstore-auto-bump" }, { "path": "./tools/playstore-auto-bump" },
{ "path": "./tools/revert-update" }, { "path": "./tools/revert-update" },
{ "path": "./tools/utils" } { "path": "./tools/utils" }

View File

@@ -505,6 +505,17 @@ __metadata:
languageName: unknown languageName: unknown
linkType: soft linkType: soft
"@affine/doc-diff@workspace:tools/doc-diff":
version: 0.0.0-use.local
resolution: "@affine/doc-diff@workspace:tools/doc-diff"
dependencies:
"@affine-tools/cli": "workspace:*"
"@types/node": "npm:^22.0.0"
typescript: "npm:^5.7.2"
yjs: "npm:^13.6.27"
languageName: unknown
linkType: soft
"@affine/docs@workspace:docs/reference": "@affine/docs@workspace:docs/reference":
version: 0.0.0-use.local version: 0.0.0-use.local
resolution: "@affine/docs@workspace:docs/reference" resolution: "@affine/docs@workspace:docs/reference"