feat(server): add data migration system

This commit is contained in:
forehalo
2023-10-27 17:31:52 +08:00
parent ae6376edee
commit 98d0ac3c90
11 changed files with 466 additions and 67 deletions
@@ -2,7 +2,6 @@ import {
Inject,
Injectable,
Logger,
OnApplicationBootstrap,
OnModuleDestroy,
OnModuleInit,
} from '@nestjs/common';
@@ -14,7 +13,6 @@ import { Config } from '../../config';
import { Metrics } from '../../metrics/metrics';
import { PrismaService } from '../../prisma';
import { mergeUpdatesInApplyWay as jwstMergeUpdates } from '../../storage';
import { DocID } from '../../utils/doc';
function compare(yBinary: Buffer, jwstBinary: Buffer, strict = false): boolean {
if (yBinary.equals(jwstBinary)) {
@@ -44,9 +42,7 @@ const MAX_SEQ_NUM = 0x3fffffff; // u31
* along side all the updates that have not been applies to that snapshot(timestamp).
*/
@Injectable()
export class DocManager
implements OnModuleInit, OnModuleDestroy, OnApplicationBootstrap
{
export class DocManager implements OnModuleInit, OnModuleDestroy {
protected logger = new Logger(DocManager.name);
private job: NodeJS.Timeout | null = null;
private seqMap = new Map<string, number>();
@@ -60,12 +56,6 @@ export class DocManager
protected readonly metrics: Metrics
) {}
async onApplicationBootstrap() {
if (!this.config.node.test) {
await this.refreshDocGuid();
}
}
onModuleInit() {
if (this.automation) {
this.logger.log('Use Database');
@@ -421,56 +411,4 @@ export class DocManager
return last + 1;
}
}
/**
* deal with old records that has wrong guid format
* correct guid with `${non-wsId}:${variant}:${subId}` to `${subId}`
*
* @TODO delete in next release
* @deprecated
*/
private async refreshDocGuid() {
let turn = 0;
let lastTurnCount = 100;
while (lastTurnCount === 100) {
const docs = await this.db.snapshot.findMany({
select: {
workspaceId: true,
id: true,
},
skip: turn * 100,
take: 100,
orderBy: {
createdAt: 'asc',
},
});
lastTurnCount = docs.length;
for (const doc of docs) {
const docId = new DocID(doc.id, doc.workspaceId);
if (docId && !docId.isWorkspace && docId.guid !== doc.id) {
await this.db.snapshot.deleteMany({
where: {
id: docId.guid,
workspaceId: doc.workspaceId,
},
});
await this.db.snapshot.update({
where: {
id_workspaceId: {
id: doc.id,
workspaceId: doc.workspaceId,
},
},
data: {
id: docId.guid,
},
});
}
}
turn++;
}
}
}