feat(server): impl storage runtime (#15181)

#### PR Dependency Tree


* **PR #15181** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added an additional storage backend option: asset-pack based storage
(provider for avatar, blob, and copilot).
* Introduced a dedicated storage runtime with provider capability
reporting and expanded object operations (put/head/get/list/delete),
including presigned and multipart flows where supported.
* Cloudflare R2 `jurisdiction` now uses an explicit default when
omitted.
* **Bug Fixes**
  * Broadened avatar access to allow both fs and asset-pack providers.
* Improved workspace blob upload completion validation and handling when
stored objects are missing or mismatched.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-07-01 22:24:10 +08:00
committed by GitHub
parent da7d438377
commit 8ebdb7452f
102 changed files with 6487 additions and 4508 deletions
@@ -12,7 +12,6 @@ test('backend-runtime provider starts once, runs migrations once, and reports he
health: Sinon.stub().resolves({
started: true,
databaseConnected: true,
objectStorageConfigured: true,
}),
};
(provider as any).runtime = runtime;
@@ -25,7 +24,6 @@ test('backend-runtime provider starts once, runs migrations once, and reports he
t.is(runtime.start.callCount, 2);
t.is(runtime.runMigrations.callCount, 1);
t.true(health.databaseConnected);
t.true(health.objectStorageConfigured);
t.is(runtime.stop.callCount, 1);
});
@@ -1,17 +1,12 @@
import { Global, Module } from '@nestjs/common';
import { BackendRuntimeBlobJob } from './blob-job';
import { BackendRuntimeHousekeepingJob } from './job';
import { BackendRuntimeProvider } from './provider';
@Global()
@Module({
providers: [
BackendRuntimeProvider,
BackendRuntimeBlobJob,
BackendRuntimeHousekeepingJob,
],
exports: [BackendRuntimeProvider, BackendRuntimeBlobJob],
providers: [BackendRuntimeProvider, BackendRuntimeHousekeepingJob],
exports: [BackendRuntimeProvider],
})
export class BackendRuntimeModule {}
@@ -30,9 +30,7 @@ export class BackendRuntimeProvider
await this.runtime.start();
await this.runMigrationsOnce();
const health = await this.runtime.health();
this.logger.log(
`backend runtime started: db=${health.databaseConnected} objectStorage=${health.objectStorageConfigured}`
);
this.logger.log(`backend runtime started: db=${health.databaseConnected}`);
}
async stop() {
@@ -44,18 +42,6 @@ export class BackendRuntimeProvider
return await this.runtime.health();
}
async cleanupExpiredPendingBlobs(cutoffMs: number, limit: number) {
return await this.measured('cleanupExpiredPendingBlobs', rt =>
rt.cleanupExpiredPendingBlobs(cutoffMs, limit)
);
}
async releaseDeletedBlobs(workspaceId: string, limit: number) {
return await this.measured('releaseDeletedBlobs', rt =>
rt.releaseDeletedBlobs(workspaceId, limit)
);
}
async cleanupExpiredSnapshotHistories(limit: number) {
return await this.measured('cleanupExpiredSnapshotHistories', rt =>
rt.cleanupExpiredSnapshotHistories(limit)
@@ -80,41 +66,6 @@ export class BackendRuntimeProvider
);
}
async backfillMissingBlobMetadata(
workspaceId: string | null | undefined,
limit: number
) {
return await this.measured('backfillMissingBlobMetadata', rt =>
rt.backfillMissingBlobMetadata(workspaceId, limit)
);
}
async rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number) {
return await this.measured('rebuildWorkspaceDocBlobRefs', rt =>
rt.rebuildWorkspaceDocBlobRefs(workspaceId, limit)
);
}
async planUnreferencedWorkspaceBlobs(
workspaceId: string,
gracePeriodDays: number,
limit: number
) {
return await this.measured('planUnreferencedWorkspaceBlobs', rt =>
rt.planUnreferencedWorkspaceBlobs(workspaceId, gracePeriodDays, limit)
);
}
async executeBlobCleanupCandidates(
runId: string,
gracePeriodDays: number,
limit: number
) {
return await this.measured('executeBlobCleanupCandidates', rt =>
rt.executeBlobCleanupCandidates(runId, gracePeriodDays, limit)
);
}
private async measured<T>(
method: string,
fn: (runtime: RuntimeInstance) => Promise<T>
@@ -52,8 +52,19 @@ test.before(async t => {
t.context.state = module.get(QuotaStateService);
});
test.beforeEach(async t => {
await t.context.module.initTestingDB();
});
test.after.always(async t => {
await t.context.module.close();
});
test('quota service ignores dirty legacy commercial features', async t => {
const { owner, workspace } = await createWorkspace(t);
await t.context.state.reconcileUserQuotaState(owner.id);
await t.context.state.reconcileWorkspaceQuotaState(workspace.id);
await t.context.models.userFeature.add(
owner.id,
'pro_plan_v1',
@@ -306,6 +317,8 @@ test('ai entitlement is a capability overlay on free quota', async t => {
test('workspace team status ignores dirty legacy feature', async t => {
const { workspace } = await createWorkspace(t);
await t.context.state.reconcileWorkspaceQuotaState(workspace.id);
await t.context.models.workspaceFeature.add(
workspace.id,
'team_plan_v1',
@@ -324,6 +337,7 @@ test('workspace team status ignores dirty legacy feature', async t => {
status: 'active',
quantity: 5,
});
await t.context.state.reconcileWorkspaceQuotaState(workspace.id);
t.true(await t.context.models.workspace.isTeamWorkspace(workspace.id));
});
@@ -357,14 +371,6 @@ test('selfhosted builtin free has cloud pro quota rights', async t => {
}
});
test.beforeEach(async t => {
await t.context.module.initTestingDB();
});
test.after.always(async t => {
await t.context.module.close();
});
test('reconciles quota states from entitlements and business tables', async t => {
const previousDeploymentType = globalThis.env.DEPLOYMENT_TYPE;
// @ts-expect-error test mutates env singleton for cloud entitlement semantics
@@ -0,0 +1,93 @@
import test from 'ava';
import Sinon from 'sinon';
import { StorageRuntimeProvider } from '../provider';
function createProvider() {
const provider = new StorageRuntimeProvider({
db: {
datasourceUrl: 'postgresql://localhost:5432/affine',
},
storages: {
blob: {
storage: {
provider: 'fs',
bucket: 'blobs',
config: { path: '~/.affine/storage' },
},
},
avatar: {
storage: {
provider: 'fs',
bucket: 'avatars',
config: { path: '~/.affine/storage' },
},
},
},
copilot: {
storage: {
provider: 'fs',
bucket: 'copilot',
config: { path: '~/.affine/storage' },
},
},
} as any);
const runtime = {
configure: Sinon.stub(),
start: Sinon.stub().resolves(),
stop: Sinon.stub().resolves(),
runMigrations: Sinon.stub().resolves(),
health: Sinon.stub().resolves({
started: true,
databaseConnected: true,
provider: 'fs',
}),
};
(provider as any).runtime = runtime;
return { provider, runtime };
}
test('storage-runtime provider restarts on storage config changes', async t => {
const { provider, runtime } = createProvider();
await provider.start();
await provider.onConfigChanged({ updates: { storages: {} } });
t.is(runtime.stop.callCount, 1);
t.is(runtime.configure.callCount, 2);
t.is(runtime.start.callCount, 2);
t.is(runtime.runMigrations.callCount, 2);
});
test('storage-runtime provider restarts on copilot storage config changes', async t => {
const { provider, runtime } = createProvider();
await provider.start();
await provider.onConfigChanged({
updates: {
copilot: {
storage: {
provider: 'fs',
bucket: 'new-copilot',
config: { path: '~/.affine/storage' },
},
},
},
});
t.is(runtime.stop.callCount, 1);
t.is(runtime.configure.callCount, 2);
t.is(runtime.start.callCount, 2);
t.is(runtime.runMigrations.callCount, 2);
});
test('storage-runtime provider ignores unrelated config changes', async t => {
const { provider, runtime } = createProvider();
await provider.start();
await provider.onConfigChanged({ updates: { flags: {} } });
t.is(runtime.stop.callCount, 0);
t.is(runtime.start.callCount, 1);
t.is(runtime.runMigrations.callCount, 1);
});
@@ -0,0 +1,15 @@
import { Global, Module } from '@nestjs/common';
import { StorageRuntimeProvider } from './provider';
@Global()
@Module({
providers: [StorageRuntimeProvider],
exports: [StorageRuntimeProvider],
})
export class StorageRuntimeModule {}
export {
type StorageRuntimeGetObjectResult,
StorageRuntimeProvider,
} from './provider';
@@ -0,0 +1,374 @@
import { Readable } from 'node:stream';
import {
Injectable,
Logger,
type OnApplicationBootstrap,
type OnApplicationShutdown,
} from '@nestjs/common';
import type {
BlobOutputType,
GetObjectMetadata,
ListObjectsMetadata,
PresignedUpload,
PutObjectMetadata,
} from '../../base';
import { Config, OnEvent } from '../../base';
import { wrapCallMetric } from '../../base/metrics';
import {
type RuntimeObjectGetResult,
type RuntimeObjectListEntry,
type RuntimeObjectMetadata,
type RuntimePresignedObjectRequest,
type StorageProviderCapabilities,
StorageRuntime,
type StorageRuntimeHealth,
} from '../../native';
type RuntimeInstance = InstanceType<typeof StorageRuntime>;
@Injectable()
export class StorageRuntimeProvider
implements OnApplicationBootstrap, OnApplicationShutdown
{
private readonly logger = new Logger(StorageRuntimeProvider.name);
private readonly runtime: RuntimeInstance = new StorageRuntime();
private migrationsStarted = false;
constructor(private readonly config: Config) {}
async onApplicationBootstrap() {
await this.start();
}
async onApplicationShutdown() {
await this.stop();
}
async start() {
this.configureRuntime();
await this.runtime.start();
await this.runMigrationsOnce();
const health = await this.runtime.health();
this.logger.log(
`storage runtime started: db=${health.databaseConnected} provider=${health.provider ?? 'none'}`
);
}
async stop() {
await this.runtime.stop();
this.logger.log('storage runtime stopped');
}
@OnEvent('config.init')
async onConfigInit() {
await this.start();
}
@OnEvent('config.changed')
async onConfigChanged({ updates }: Events['config.changed']) {
if (
!('storages' in updates) &&
!('db' in updates) &&
!updates.copilot?.storage
) {
return;
}
await this.restart();
}
async health(): Promise<StorageRuntimeHealth> {
return await this.runtime.health();
}
async providerCapabilities(
scope: string
): Promise<StorageProviderCapabilities> {
return await this.measured('providerCapabilities', rt =>
rt.providerCapabilities(scope)
);
}
async putObject(
scope: string,
key: string,
body: Buffer,
metadata?: PutObjectMetadata
) {
const result = await this.measured('putObject', rt =>
rt.putObject(scope, key, body, toRuntimeMetadata(metadata))
);
return fromRuntimeMetadata(result);
}
async headObject(scope: string, key: string) {
const metadata = await this.measured('headObject', rt =>
rt.headObject(scope, key)
);
return metadata ? fromRuntimeMetadata(metadata) : undefined;
}
async getObject(
scope: string,
key: string
): Promise<StorageRuntimeGetObjectResult> {
const result = await this.measured('getObject', rt =>
rt.getObject(scope, key)
);
return result ? fromRuntimeGetResult(result) : {};
}
async listObjects(scope: string, prefix?: string) {
const entries = await this.measured('listObjects', rt =>
rt.listObjects(scope, prefix)
);
return entries.map(fromRuntimeListEntry);
}
async deleteObject(scope: string, key: string) {
await this.measured('deleteObject', rt => rt.deleteObject(scope, key));
}
async presignPut(scope: string, key: string, metadata?: PutObjectMetadata) {
const result = await this.measured('presignPut', rt =>
rt.presignPut(scope, key, toRuntimeMetadata(metadata))
);
return result ? fromRuntimePresigned(result) : undefined;
}
async presignGet(scope: string, key: string) {
const result = await this.measured('presignGet', rt =>
rt.presignGet(scope, key)
);
return result ? fromRuntimePresigned(result) : undefined;
}
async createMultipartUpload(
scope: string,
key: string,
metadata?: PutObjectMetadata
) {
const result = await this.measured('createMultipartUpload', rt =>
rt.createMultipartUpload(scope, key, toRuntimeMetadata(metadata))
);
return result
? { uploadId: result.uploadId, expiresAt: new Date(result.expiresAtMs) }
: undefined;
}
async presignUploadPart(
scope: string,
key: string,
uploadId: string,
partNumber: number
) {
const result = await this.measured('presignUploadPart', rt =>
rt.presignUploadPart(scope, key, uploadId, partNumber)
);
return result ? fromRuntimePresigned(result) : undefined;
}
async listMultipartUploadParts(scope: string, key: string, uploadId: string) {
return (
(await this.measured('listMultipartUploadParts', rt =>
rt.listMultipartUploadParts(scope, key, uploadId)
)) ?? undefined
);
}
async proxyUploadPart(
scope: string,
key: string,
uploadId: string,
partNumber: number,
body: Buffer,
contentLength?: number
) {
return (
(await this.measured('proxyUploadPart', rt =>
rt.proxyUploadPart(
scope,
key,
uploadId,
partNumber,
body,
contentLength
)
)) ?? undefined
);
}
async completeMultipartUpload(
scope: string,
key: string,
uploadId: string,
parts: { partNumber: number; etag: string }[]
) {
return await this.measured('completeMultipartUpload', rt =>
rt.completeMultipartUpload(scope, key, uploadId, parts)
);
}
async abortMultipartUpload(scope: string, key: string, uploadId: string) {
return await this.measured('abortMultipartUpload', rt =>
rt.abortMultipartUpload(scope, key, uploadId)
);
}
async completeWorkspaceBlobUpload(
workspaceId: string,
key: string,
expected: { size: number; mime: string }
) {
return await this.measured('completeWorkspaceBlobUpload', rt =>
rt.completeWorkspaceBlobUpload(
workspaceId,
key,
expected.size,
expected.mime
)
);
}
async cleanupExpiredPendingBlobs(cutoffMs: number, limit: number) {
return await this.measured('cleanupExpiredPendingBlobs', rt =>
rt.cleanupExpiredPendingBlobs(cutoffMs, limit)
);
}
async releaseDeletedBlobs(workspaceId: string, limit: number) {
return await this.measured('releaseDeletedBlobs', rt =>
rt.releaseDeletedBlobs(workspaceId, limit)
);
}
async backfillMissingBlobMetadata(
workspaceId: string | null | undefined,
limit: number
) {
return await this.measured('backfillMissingBlobMetadata', rt =>
rt.backfillMissingBlobMetadata(workspaceId, limit)
);
}
async rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number) {
return await this.measured('rebuildWorkspaceDocBlobRefs', rt =>
rt.rebuildWorkspaceDocBlobRefs(workspaceId, limit)
);
}
async planUnreferencedWorkspaceBlobs(
workspaceId: string,
gracePeriodDays: number,
limit: number
) {
return await this.measured('planUnreferencedWorkspaceBlobs', rt =>
rt.planUnreferencedWorkspaceBlobs(workspaceId, gracePeriodDays, limit)
);
}
async executeBlobCleanupCandidates(
runId: string,
gracePeriodDays: number,
limit: number
) {
return await this.measured('executeBlobCleanupCandidates', rt =>
rt.executeBlobCleanupCandidates(runId, gracePeriodDays, limit)
);
}
private async measured<T>(
method: string,
fn: (runtime: RuntimeInstance) => Promise<T>
): Promise<T> {
return await wrapCallMetric(() => fn(this.runtime), 'storage', 'runtime', {
method,
})();
}
private async runMigrationsOnce() {
if (this.migrationsStarted) {
return;
}
await this.runtime.runMigrations();
this.migrationsStarted = true;
}
private async restart() {
await this.runtime.stop();
this.migrationsStarted = false;
await this.start();
}
private configureRuntime() {
this.runtime.configure(
JSON.stringify({
db: {
datasourceUrl: this.config.db.datasourceUrl,
},
storages: {
'blob.storage': this.config.storages.blob.storage,
'avatar.storage': this.config.storages.avatar.storage,
},
copilot: {
storage: this.config.copilot.storage,
},
})
);
}
}
function toRuntimeMetadata(metadata?: PutObjectMetadata) {
return metadata
? {
contentType: metadata.contentType,
contentLength: metadata.contentLength,
checksumCrc32: metadata.checksumCRC32,
}
: undefined;
}
function fromRuntimeMetadata(
metadata: RuntimeObjectMetadata
): GetObjectMetadata {
return {
contentType: metadata.contentType,
contentLength: metadata.contentLength,
lastModified: new Date(metadata.lastModifiedMs),
checksumCRC32: metadata.checksumCrc32,
};
}
function fromRuntimeGetResult(result: RuntimeObjectGetResult) {
return {
body: Readable.from(result.body),
metadata: fromRuntimeMetadata(result.metadata),
};
}
function fromRuntimeListEntry(
entry: RuntimeObjectListEntry
): ListObjectsMetadata {
return {
key: entry.key,
contentLength: entry.contentLength,
lastModified: new Date(entry.lastModifiedMs),
};
}
function fromRuntimePresigned(
request: RuntimePresignedObjectRequest
): PresignedUpload {
return {
url: request.url,
headers: JSON.parse(request.headersJson) as Record<string, string>,
expiresAt: new Date(request.expiresAtMs),
};
}
export type StorageRuntimeGetObjectResult = {
redirectUrl?: string;
body?: BlobOutputType;
metadata?: GetObjectMetadata;
};
@@ -1,7 +1,7 @@
import ava, { TestFn } from 'ava';
import Sinon from 'sinon';
import { BackendRuntimeBlobJob } from '../blob-job';
import { StorageBlobJob } from '../blob-job';
interface Context {
runtime: {
@@ -22,7 +22,7 @@ interface Context {
findMany: Sinon.SinonStub;
};
};
job: BackendRuntimeBlobJob;
job: StorageBlobJob;
}
const test = ava as TestFn<Context>;
@@ -31,7 +31,8 @@ test.beforeEach(t => {
t.context.runtime = {
health: Sinon.stub().resolves({
databaseConnected: true,
objectStorageConfigured: true,
providerConfigured: true,
provider: 'fs',
}),
backfillMissingBlobMetadata: Sinon.stub(),
rebuildWorkspaceDocBlobRefs: Sinon.stub(),
@@ -49,7 +50,7 @@ test.beforeEach(t => {
findMany: Sinon.stub(),
},
};
t.context.job = new BackendRuntimeBlobJob(
t.context.job = new StorageBlobJob(
t.context.runtime as any,
t.context.event as any,
t.context.queue as any,
@@ -103,7 +104,8 @@ for (const scenario of objectStorageRequiredCases) {
test(`${scenario.name} skips when object storage is not configured`, async t => {
t.context.runtime.health.resolves({
databaseConnected: true,
objectStorageConfigured: false,
providerConfigured: true,
provider: undefined,
});
await scenario.run(t.context);
@@ -5,17 +5,66 @@ import test from 'ava';
import { createModule } from '../../../__tests__/create-module';
import { Mockers } from '../../../__tests__/mocks';
import { initTestingDB } from '../../../__tests__/utils';
import { Models } from '../../../models';
import { getMime } from '../../../native';
import { StorageRuntimeProvider } from '../../storage-runtime';
import { CommentAttachmentStorage, StorageModule } from '../index';
const objects = new Map<
string,
{
body: Buffer;
metadata: {
contentType: string;
contentLength: number;
lastModified: Date;
};
}
>();
const storageRuntime = {
putObject: async (
_scope: string,
key: string,
body: Buffer,
_metadata?: { contentType?: string; contentLength?: number }
) => {
const object = {
body,
metadata: {
contentType: getMime(body),
contentLength: body.length,
lastModified: new Date(),
},
};
objects.set(key, object);
return object.metadata;
},
headObject: async (_scope: string, key: string) => objects.get(key)?.metadata,
getObject: async (_scope: string, key: string) => {
const object = objects.get(key);
return object
? { body: Readable.from(object.body), metadata: object.metadata }
: {};
},
deleteObject: async (_scope: string, key: string) => {
objects.delete(key);
},
presignGet: async () => undefined,
};
const module = await createModule({
imports: [StorageModule],
tapModule: builder => {
builder.overrideProvider(StorageRuntimeProvider).useValue(storageRuntime);
},
});
const storage = module.get(CommentAttachmentStorage);
const models = module.get(Models);
test.before(async () => {
await storage.onConfigInit();
test.beforeEach(async () => {
await initTestingDB(module);
objects.clear();
});
test.after.always(async () => {
@@ -3,8 +3,10 @@ import { Cron, CronExpression } from '@nestjs/schedule';
import { PrismaClient } from '@prisma/client';
import { EventBus, JobQueue, OnJob } from '../../base';
import { BackendRuntimeProvider } from './provider';
import { StorageRuntimeProvider } from '../storage-runtime';
// Queue keys are persisted API; keep the legacy backendRuntime.* names while
// StorageBlobJob and StorageRuntimeProvider own the implementation.
declare global {
interface Jobs {
'backendRuntime.backfillMissingBlobMetadata': {
@@ -45,11 +47,11 @@ declare global {
}
@Injectable()
export class BackendRuntimeBlobJob {
private readonly logger = new Logger(BackendRuntimeBlobJob.name);
export class StorageBlobJob {
private readonly logger = new Logger(StorageBlobJob.name);
constructor(
private readonly rt: BackendRuntimeProvider,
private readonly rt: StorageRuntimeProvider,
private readonly event: EventBus,
private readonly queue: JobQueue,
private readonly db: PrismaClient
@@ -355,7 +357,7 @@ export class BackendRuntimeBlobJob {
limit
);
await Promise.all(
result.workspaceIds.map(workspaceId =>
result.workspaceIds.map((workspaceId: string) =>
this.event.emitAsync('workspace.blobs.updated', { workspaceId })
)
);
@@ -375,7 +377,7 @@ export class BackendRuntimeBlobJob {
limit
);
await Promise.all(
result.workspaceIds.map(workspaceId =>
result.workspaceIds.map((workspaceId: string) =>
this.event.emitAsync('workspace.blobs.updated', { workspaceId })
)
);
@@ -409,12 +411,12 @@ export class BackendRuntimeBlobJob {
private async hasObjectStorage(operation: string) {
const health = await this.rt.health();
if (health.objectStorageConfigured) {
if (health.provider) {
return true;
}
this.logger.warn(
`skip ${operation}: BackendRuntime object storage is not configured`
`skip ${operation}: StorageRuntime provider is not configured`
);
return false;
}
@@ -2,7 +2,8 @@ import './config';
import { Module } from '@nestjs/common';
import { BackendRuntimeModule } from '../backend-runtime';
import { StorageRuntimeModule } from '../storage-runtime';
import { StorageBlobJob } from './blob-job';
import { BlobUploadCleanupJob } from './job';
import { R2UploadController } from './r2-proxy';
import {
@@ -12,16 +13,23 @@ import {
} from './wrappers';
@Module({
imports: [BackendRuntimeModule],
imports: [StorageRuntimeModule],
controllers: [R2UploadController],
providers: [
WorkspaceBlobStorage,
AvatarStorage,
CommentAttachmentStorage,
StorageBlobJob,
BlobUploadCleanupJob,
],
exports: [WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage],
exports: [
WorkspaceBlobStorage,
AvatarStorage,
CommentAttachmentStorage,
StorageBlobJob,
],
})
export class StorageModule {}
export { StorageBlobJob } from './blob-job';
export { AvatarStorage, CommentAttachmentStorage, WorkspaceBlobStorage };
@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { EventBus, JobQueue, OneDay, OnJob } from '../../base';
import { BackendRuntimeProvider } from '../backend-runtime';
import { StorageRuntimeProvider } from '../storage-runtime';
declare global {
interface Jobs {
@@ -15,7 +15,7 @@ export class BlobUploadCleanupJob {
private readonly logger = new Logger(BlobUploadCleanupJob.name);
constructor(
private readonly rt: BackendRuntimeProvider,
private readonly rt: StorageRuntimeProvider,
private readonly event: EventBus,
private readonly queue: JobQueue
) {}
@@ -7,19 +7,16 @@ import {
BlobInvalid,
CallMetric,
Config,
OnEvent,
PROXY_MULTIPART_PATH,
PROXY_UPLOAD_PATH,
type R2StorageConfig,
STORAGE_PROXY_ROOT,
StorageProviderConfig,
StorageProviderFactory,
type StorageProviderConfig,
toBuffer,
} from '../../base';
import {
R2StorageConfig,
R2StorageProvider,
} from '../../base/storage/providers/r2';
import { Models } from '../../models';
import { Public } from '../auth/guard';
import { StorageRuntimeProvider } from '../storage-runtime';
import { MULTIPART_PART_SIZE } from './constants';
type R2BlobStorageConfig = StorageProviderConfig & {
@@ -37,21 +34,13 @@ type R2Config = {
@Controller(STORAGE_PROXY_ROOT)
export class R2UploadController {
private readonly logger = new Logger(R2UploadController.name);
private provider: R2StorageProvider | null = null;
constructor(
private readonly config: Config,
private readonly models: Models,
private readonly storageFactory: StorageProviderFactory
private readonly rt: StorageRuntimeProvider
) {}
@OnEvent('config.changed')
onConfigChanged(event: Events['config.changed']) {
if (event.updates.storages?.blob?.storage) {
this.provider = null;
}
}
private getR2Config(): R2Config {
const storage = this.config.storages.blob.storage as StorageProviderConfig;
if (storage.provider !== 'cloudflare-r2') {
@@ -69,16 +58,6 @@ export class R2UploadController {
return { storage: storage as R2BlobStorageConfig, signKey };
}
private getProvider(storage: R2BlobStorageConfig) {
if (!this.provider) {
const candidate = this.storageFactory.create(storage);
if (candidate instanceof R2StorageProvider) {
this.provider = candidate;
}
}
return this.provider;
}
private sign(canonical: string, signKey: string) {
return createHmac('sha256', signKey).update(canonical).digest('base64');
}
@@ -173,7 +152,7 @@ export class R2UploadController {
@Put('upload')
@CallMetric('controllers', 'r2_proxy_upload')
async upload(@Req() req: Request, @Res() res: Response) {
const { storage, signKey } = this.getR2Config();
const { signKey } = this.getR2Config();
const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId');
const key = this.expectString(req.query.key, 'key');
@@ -229,16 +208,16 @@ export class R2UploadController {
throw new BlobInvalid('Mime type mismatch');
}
const provider = this.getProvider(storage);
if (!provider) {
throw new BlobInvalid('R2 provider is not available');
}
try {
await provider.proxyPutObject(`${workspaceId}/${key}`, req, {
contentType: mime,
contentLength,
});
await this.rt.putObject(
'blob',
`${workspaceId}/${key}`,
await toBuffer(req),
{
contentType: mime,
contentLength,
}
);
} catch (error) {
this.logger.error('Failed to proxy upload', error as Error);
throw new BlobInvalid('Upload failed');
@@ -251,7 +230,7 @@ export class R2UploadController {
@Put('multipart')
@CallMetric('controllers', 'r2_proxy_multipart')
async uploadPart(@Req() req: Request, @Res() res: Response) {
const { storage, signKey } = this.getR2Config();
const { signKey } = this.getR2Config();
const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId');
const key = this.expectString(req.query.key, 'key');
@@ -305,18 +284,14 @@ export class R2UploadController {
throw new BlobInvalid('Part size exceeds upload metadata');
}
const provider = this.getProvider(storage);
if (!provider) {
throw new BlobInvalid('R2 provider is not available');
}
try {
const etag = await provider.proxyUploadPart(
const etag = await this.rt.proxyUploadPart(
'blob',
`${workspaceId}/${key}`,
uploadId,
partNumber,
req,
{ contentLength }
await toBuffer(req),
contentLength
);
if (etag) {
res.setHeader('etag', etag);
@@ -1,21 +1,11 @@
import { Injectable } from '@nestjs/common';
import type {
BlobInputType,
PutObjectMetadata,
StorageProvider,
} from '../../../base';
import {
Config,
OnEvent,
StorageProviderFactory,
URLHelper,
} from '../../../base';
import type { BlobInputType, PutObjectMetadata } from '../../../base';
import { Config, OnEvent, toBuffer, URLHelper } from '../../../base';
import { StorageRuntimeProvider } from '../../storage-runtime';
@Injectable()
export class AvatarStorage {
private provider!: StorageProvider;
get config() {
return this.AFFiNEConfig.storages.avatar;
}
@@ -23,23 +13,11 @@ export class AvatarStorage {
constructor(
private readonly AFFiNEConfig: Config,
private readonly url: URLHelper,
private readonly storageFactory: StorageProviderFactory
private readonly rt: StorageRuntimeProvider
) {}
@OnEvent('config.init')
async onConfigInit() {
this.provider = this.storageFactory.create(this.config.storage);
}
@OnEvent('config.changed')
async onConfigChanged(event: Events['config.changed']) {
if (event.updates.storages?.avatar?.storage) {
this.provider = this.storageFactory.create(this.config.storage);
}
}
async put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) {
await this.provider.put(key, blob, metadata);
await this.rt.putObject('avatar', key, await toBuffer(blob), metadata);
let link = this.config.publicPath + key;
if (link.startsWith('/')) {
@@ -50,11 +28,11 @@ export class AvatarStorage {
}
get(key: string) {
return this.provider.get(key);
return this.rt.getObject('avatar', key);
}
delete(link: string) {
return this.provider.delete(link.split('/').pop() as string);
return this.rt.deleteObject('avatar', link.split('/').pop() as string);
}
@OnEvent('user.deleted')
@@ -1,20 +1,24 @@
import { createHash } from 'node:crypto';
import { createHmac } from 'node:crypto';
import { Injectable, Logger } from '@nestjs/common';
import {
autoMetadata,
type BlobOutputType,
Config,
EventBus,
type GetObjectMetadata,
OnEvent,
PutObjectMetadata,
type StorageProvider,
StorageProviderFactory,
PROXY_MULTIPART_PATH,
PROXY_UPLOAD_PATH,
type PutObjectMetadata,
type R2StorageConfig,
SIGNED_URL_EXPIRED,
type StorageProviderConfig,
URLHelper,
} from '../../../base';
import { Models } from '../../../models';
import { BackendRuntimeProvider } from '../../backend-runtime';
import type { StorageProviderCapabilities } from '../../../native';
import { StorageRuntimeProvider } from '../../storage-runtime';
declare global {
interface Events {
@@ -36,52 +40,75 @@ type BlobCompleteResult =
| 'not_found'
| 'size_mismatch'
| 'mime_mismatch'
| 'checksum_mismatch';
| 'checksum_mismatch'
| 'size_too_large';
};
type BlobGetResult = {
redirectUrl?: string;
body?: BlobOutputType;
metadata?: GetObjectMetadata;
};
type R2ProxyConfig = {
signKey: string;
urlPrefix: string;
};
@Injectable()
export class WorkspaceBlobStorage {
private readonly logger = new Logger(WorkspaceBlobStorage.name);
private provider!: StorageProvider;
get config() {
return this.AFFiNEConfig.storages.blob;
}
constructor(
private readonly AFFiNEConfig: Config,
private readonly event: EventBus,
private readonly storageFactory: StorageProviderFactory,
private readonly models: Models,
private readonly url: URLHelper,
private readonly rt: BackendRuntimeProvider
private readonly rt: StorageRuntimeProvider,
private readonly config: Config
) {}
@OnEvent('config.init')
async onConfigInit() {
this.provider = this.storageFactory.create(this.config.storage);
}
@OnEvent('config.changed')
async onConfigChanged(event: Events['config.changed']) {
if (event.updates.storages?.blob?.storage) {
this.provider = this.storageFactory.create(this.config.storage);
}
}
async put(workspaceId: string, key: string, blob: Buffer) {
const meta: PutObjectMetadata = autoMetadata(blob);
await this.provider.put(`${workspaceId}/${key}`, blob, meta);
const metadata = await this.rt.putObject(
'blob',
`${workspaceId}/${key}`,
blob
);
await this.upsert(workspaceId, key, {
contentType: meta.contentType ?? 'application/octet-stream',
contentLength: blob.length,
lastModified: new Date(),
contentType: metadata.contentType,
contentLength: metadata.contentLength,
lastModified: metadata.lastModified,
});
}
async get(workspaceId: string, key: string, signedUrl?: boolean) {
return this.provider.get(`${workspaceId}/${key}`, signedUrl);
async capabilities(): Promise<StorageProviderCapabilities> {
const capabilities = await this.rt.providerCapabilities('blob');
if (!this.r2ProxyConfig()) {
return capabilities;
}
return {
...capabilities,
presignPut: true,
multipartDirect: true,
proxyUpload: true,
serverMediatedOnly: false,
};
}
async get(
workspaceId: string,
key: string,
signedUrl?: boolean
): Promise<BlobGetResult> {
if (signedUrl) {
const presigned = await this.rt.presignGet(
'blob',
`${workspaceId}/${key}`
);
if (presigned) {
return { redirectUrl: presigned.url };
}
}
return this.rt.getObject('blob', `${workspaceId}/${key}`);
}
async presignPut(
@@ -89,7 +116,11 @@ export class WorkspaceBlobStorage {
key: string,
metadata?: PutObjectMetadata
) {
return this.provider.presignPut?.(`${workspaceId}/${key}`, metadata);
const proxy = this.r2ProxyConfig();
if (proxy) {
return this.createProxyUploadUrl(workspaceId, key, metadata, proxy);
}
return this.rt.presignPut('blob', `${workspaceId}/${key}`, metadata);
}
async createMultipartUpload(
@@ -97,7 +128,8 @@ export class WorkspaceBlobStorage {
key: string,
metadata?: PutObjectMetadata
) {
return this.provider.createMultipartUpload?.(
return this.rt.createMultipartUpload(
'blob',
`${workspaceId}/${key}`,
metadata
);
@@ -109,7 +141,18 @@ export class WorkspaceBlobStorage {
uploadId: string,
partNumber: number
) {
return this.provider.presignUploadPart?.(
const proxy = this.r2ProxyConfig();
if (proxy) {
return this.createProxyMultipartUrl(
workspaceId,
key,
uploadId,
partNumber,
proxy
);
}
return this.rt.presignUploadPart(
'blob',
`${workspaceId}/${key}`,
uploadId,
partNumber
@@ -121,7 +164,8 @@ export class WorkspaceBlobStorage {
key: string,
uploadId: string
) {
return this.provider.listMultipartUploadParts?.(
return this.rt.listMultipartUploadParts(
'blob',
`${workspaceId}/${key}`,
uploadId
);
@@ -133,16 +177,12 @@ export class WorkspaceBlobStorage {
uploadId: string,
parts: { partNumber: number; etag: string }[]
) {
if (!this.provider.completeMultipartUpload) {
return false;
}
await this.provider.completeMultipartUpload(
return await this.rt.completeMultipartUpload(
'blob',
`${workspaceId}/${key}`,
uploadId,
parts
);
return true;
}
async abortMultipartUpload(
@@ -150,16 +190,15 @@ export class WorkspaceBlobStorage {
key: string,
uploadId: string
) {
if (!this.provider.abortMultipartUpload) {
return false;
}
await this.provider.abortMultipartUpload(`${workspaceId}/${key}`, uploadId);
return true;
return await this.rt.abortMultipartUpload(
'blob',
`${workspaceId}/${key}`,
uploadId
);
}
async head(workspaceId: string, key: string) {
return this.provider.head(`${workspaceId}/${key}`);
return this.rt.headObject('blob', `${workspaceId}/${key}`);
}
async complete(
@@ -167,57 +206,28 @@ export class WorkspaceBlobStorage {
key: string,
expected: { size: number; mime: string }
): Promise<BlobCompleteResult> {
const metadata = await this.head(workspaceId, key);
if (!metadata) {
return { ok: false, reason: 'not_found' };
}
if (metadata.contentLength !== expected.size) {
return { ok: false, reason: 'size_mismatch' };
}
if (expected.mime && metadata.contentType !== expected.mime) {
return { ok: false, reason: 'mime_mismatch' };
}
const object = await this.provider.get(`${workspaceId}/${key}`);
if (!object.body) {
return { ok: false, reason: 'not_found' };
}
const checksum = createHash('sha256');
try {
for await (const chunk of object.body) {
checksum.update(chunk as Buffer);
}
} catch (e) {
this.logger.error('failed to read blob for checksum verification', e);
return { ok: false, reason: 'checksum_mismatch' };
}
const base64 = checksum.digest('base64');
const base64urlWithPadding = base64.replace(/\+/g, '-').replace(/\//g, '_');
if (base64urlWithPadding !== key) {
try {
await this.provider.delete(`${workspaceId}/${key}`);
} catch (e) {
// never throw
this.logger.error('failed to delete invalid blob', e);
}
return { ok: false, reason: 'checksum_mismatch' };
}
await this.models.blob.upsert({
const result = await this.rt.completeWorkspaceBlobUpload(
workspaceId,
key,
mime: metadata.contentType,
size: metadata.contentLength,
status: 'completed',
uploadId: null,
});
return { ok: true, metadata };
expected
);
if (!result.ok) {
return {
ok: false,
reason: (result.reason ?? 'checksum_mismatch') as Exclude<
BlobCompleteResult,
{ ok: true }
>['reason'],
};
}
return {
ok: true,
metadata: {
contentType: result.contentType ?? 'application/octet-stream',
contentLength: result.contentLength ?? expected.size,
lastModified: new Date(result.lastModifiedMs ?? Date.now()),
},
};
}
async list(workspaceId: string) {
@@ -226,7 +236,7 @@ export class WorkspaceBlobStorage {
async delete(workspaceId: string, key: string, permanently = false) {
if (permanently) {
await this.provider.delete(`${workspaceId}/${key}`);
await this.rt.deleteObject('blob', `${workspaceId}/${key}`);
}
await this.models.blob.delete(workspaceId, key, permanently);
if (!permanently) {
@@ -297,4 +307,115 @@ export class WorkspaceBlobStorage {
}: Events['workspace.blob.delete']) {
await this.delete(workspaceId, key, true);
}
private r2ProxyConfig() {
const storage = this.config.storages.blob.storage as StorageProviderConfig;
if (storage.provider !== 'cloudflare-r2') {
return;
}
const r2 = storage.config as R2StorageConfig;
const usePresignedURL = r2.usePresignedURL;
if (
!usePresignedURL?.enabled ||
!usePresignedURL.urlPrefix ||
!usePresignedURL.signKey
) {
return;
}
return {
signKey: usePresignedURL.signKey,
urlPrefix: usePresignedURL.urlPrefix,
};
}
private signProxy(
path: string,
canonicalFields: (string | number | undefined)[],
exp: number,
signKey: string
) {
const canonical = [
path,
...canonicalFields.map(field =>
field === undefined ? '' : field.toString()
),
exp.toString(),
].join('\n');
return `${exp}-${createHmac('sha256', signKey).update(canonical).digest('base64')}`;
}
private createProxyUploadUrl(
workspaceId: string,
key: string,
metadata: PutObjectMetadata | undefined,
proxy: R2ProxyConfig
) {
const contentType = metadata?.contentType ?? 'application/octet-stream';
const contentLength = metadata?.contentLength;
const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000);
const exp = Math.floor(expiresAt.getTime() / 1000);
const token = this.signProxy(
PROXY_UPLOAD_PATH,
[workspaceId, key, contentType, contentLength],
exp,
proxy.signKey
);
return {
url: this.linkProxyUrl(proxy.urlPrefix, PROXY_UPLOAD_PATH, {
workspaceId,
key,
contentType,
contentLength,
exp,
token,
}),
headers: {},
expiresAt,
};
}
private createProxyMultipartUrl(
workspaceId: string,
key: string,
uploadId: string,
partNumber: number,
proxy: R2ProxyConfig
) {
const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000);
const exp = Math.floor(expiresAt.getTime() / 1000);
const token = this.signProxy(
PROXY_MULTIPART_PATH,
[workspaceId, key, uploadId, partNumber],
exp,
proxy.signKey
);
return {
url: this.linkProxyUrl(proxy.urlPrefix, PROXY_MULTIPART_PATH, {
workspaceId,
key,
uploadId,
partNumber,
exp,
token,
}),
headers: {},
expiresAt,
};
}
private linkProxyUrl(
urlPrefix: string,
path: string,
query: Record<string, string | number | undefined>
) {
const url = new URL(
`${urlPrefix.replace(/\/+$/, '')}${path.startsWith('/') ? path : `/${path}`}`
);
for (const [key, value] of Object.entries(query)) {
if (value !== undefined) {
url.searchParams.set(key, value.toString());
}
}
return url.toString();
}
}
@@ -1,16 +1,11 @@
import { Injectable, Logger } from '@nestjs/common';
import {
autoMetadata,
Config,
EventBus,
metrics,
OnEvent,
type StorageProvider,
StorageProviderFactory,
URLHelper,
} from '../../../base';
import { EventBus, metrics, OnEvent, URLHelper } from '../../../base';
import { Models } from '../../../models';
import {
type StorageRuntimeGetObjectResult,
StorageRuntimeProvider,
} from '../../storage-runtime';
declare global {
interface Events {
@@ -25,32 +20,14 @@ declare global {
@Injectable()
export class CommentAttachmentStorage {
private readonly logger = new Logger(CommentAttachmentStorage.name);
private provider!: StorageProvider;
get config() {
return this.AFFiNEConfig.storages.blob;
}
constructor(
private readonly AFFiNEConfig: Config,
private readonly event: EventBus,
private readonly storageFactory: StorageProviderFactory,
private readonly models: Models,
private readonly url: URLHelper
private readonly url: URLHelper,
private readonly rt: StorageRuntimeProvider
) {}
@OnEvent('config.init')
async onConfigInit() {
this.provider = this.storageFactory.create(this.config.storage);
}
@OnEvent('config.changed')
async onConfigChanged(event: Events['config.changed']) {
if (event.updates.storages?.blob?.storage) {
this.provider = this.storageFactory.create(this.config.storage);
}
}
private storageKey(workspaceId: string, docId: string, key: string) {
return `comment-attachments/${workspaceId}/${docId}/${key}`;
}
@@ -63,15 +40,13 @@ export class CommentAttachmentStorage {
blob: Buffer,
userId: string
) {
const meta = autoMetadata(blob);
await this.provider.put(
const metadata = await this.rt.putObject(
'blob',
this.storageKey(workspaceId, docId, key),
blob,
meta
blob
);
const mime = meta.contentType ?? 'application/octet-stream';
const size = blob.length;
const mime = metadata.contentType;
const size = metadata.contentLength;
await this.models.commentAttachment.upsert({
workspaceId,
docId,
@@ -94,15 +69,22 @@ export class CommentAttachmentStorage {
docId: string,
key: string,
signedUrl?: boolean
) {
return await this.provider.get(
this.storageKey(workspaceId, docId, key),
signedUrl
);
): Promise<StorageRuntimeGetObjectResult> {
const storageKey = this.storageKey(workspaceId, docId, key);
if (signedUrl) {
const presigned = await this.rt.presignGet('blob', storageKey);
if (presigned) {
return { redirectUrl: presigned.url };
}
}
return await this.rt.getObject('blob', storageKey);
}
async delete(workspaceId: string, docId: string, key: string) {
await this.provider.delete(this.storageKey(workspaceId, docId, key));
await this.rt.deleteObject(
'blob',
this.storageKey(workspaceId, docId, key)
);
await this.models.commentAttachment.delete(workspaceId, docId, key);
this.logger.log(
`deleted comment attachment ${workspaceId}/${docId}/${key}`
@@ -16,9 +16,10 @@ export class UserAvatarController {
@Get('/:id')
async getAvatar(@Res() res: Response, @Param('id') id: string) {
if (this.storage.config.storage.provider !== 'fs') {
const provider = this.storage.config.storage.provider;
if (!['assetpack', 'fs'].includes(provider)) {
throw new ActionForbidden(
'Only available when avatar storage provider set to fs.'
'Only available when avatar storage provider is fs or assetpack.'
);
}
@@ -248,11 +248,12 @@ export class WorkspaceBlobResolver {
}
const metadata = { contentType: mime, contentLength: size };
const capabilities = await this.storage.capabilities();
let init: BlobUploadInit | null = null;
let uploadIdForRecord: string | null = null;
// try to resume multipart uploads
if (record && record.uploadId) {
if (capabilities.multipartDirect && record && record.uploadId) {
const uploadedParts = await this.storage.listMultipartUploadParts(
workspaceId,
key,
@@ -270,7 +271,7 @@ export class WorkspaceBlobResolver {
}
}
if (size >= MULTIPART_THRESHOLD) {
if (capabilities.multipartDirect && size >= MULTIPART_THRESHOLD) {
const multipart = await this.storage.createMultipartUpload(
workspaceId,
key,
@@ -289,7 +290,7 @@ export class WorkspaceBlobResolver {
}
}
if (!init) {
if (!init && capabilities.presignPut) {
const presigned = await this.storage.presignPut(
workspaceId,
key,
@@ -397,6 +398,9 @@ export class WorkspaceBlobResolver {
if (result.reason === 'mime_mismatch') {
throw new BlobInvalid('Blob mime mismatch');
}
if (result.reason === 'size_too_large') {
throw new BlobInvalid('Blob size too large');
}
throw new BlobInvalid('Blob key mismatch');
}