mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-08-04 11:01:44 +08:00
feat(server): impl storage runtime (#15181)
#### PR Dependency Tree * **PR #15181** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added an additional storage backend option: asset-pack based storage (provider for avatar, blob, and copilot). * Introduced a dedicated storage runtime with provider capability reporting and expanded object operations (put/head/get/list/delete), including presigned and multipart flows where supported. * Cloudflare R2 `jurisdiction` now uses an explicit default when omitted. * **Bug Fixes** * Broadened avatar access to allow both fs and asset-pack providers. * Improved workspace blob upload completion validation and handling when stored objects are missing or mismatched. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
@@ -12,7 +12,6 @@ test('backend-runtime provider starts once, runs migrations once, and reports he
|
||||
health: Sinon.stub().resolves({
|
||||
started: true,
|
||||
databaseConnected: true,
|
||||
objectStorageConfigured: true,
|
||||
}),
|
||||
};
|
||||
(provider as any).runtime = runtime;
|
||||
@@ -25,7 +24,6 @@ test('backend-runtime provider starts once, runs migrations once, and reports he
|
||||
t.is(runtime.start.callCount, 2);
|
||||
t.is(runtime.runMigrations.callCount, 1);
|
||||
t.true(health.databaseConnected);
|
||||
t.true(health.objectStorageConfigured);
|
||||
t.is(runtime.stop.callCount, 1);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
|
||||
import { BackendRuntimeBlobJob } from './blob-job';
|
||||
import { BackendRuntimeHousekeepingJob } from './job';
|
||||
import { BackendRuntimeProvider } from './provider';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [
|
||||
BackendRuntimeProvider,
|
||||
BackendRuntimeBlobJob,
|
||||
BackendRuntimeHousekeepingJob,
|
||||
],
|
||||
exports: [BackendRuntimeProvider, BackendRuntimeBlobJob],
|
||||
providers: [BackendRuntimeProvider, BackendRuntimeHousekeepingJob],
|
||||
exports: [BackendRuntimeProvider],
|
||||
})
|
||||
export class BackendRuntimeModule {}
|
||||
|
||||
|
||||
@@ -30,9 +30,7 @@ export class BackendRuntimeProvider
|
||||
await this.runtime.start();
|
||||
await this.runMigrationsOnce();
|
||||
const health = await this.runtime.health();
|
||||
this.logger.log(
|
||||
`backend runtime started: db=${health.databaseConnected} objectStorage=${health.objectStorageConfigured}`
|
||||
);
|
||||
this.logger.log(`backend runtime started: db=${health.databaseConnected}`);
|
||||
}
|
||||
|
||||
async stop() {
|
||||
@@ -44,18 +42,6 @@ export class BackendRuntimeProvider
|
||||
return await this.runtime.health();
|
||||
}
|
||||
|
||||
async cleanupExpiredPendingBlobs(cutoffMs: number, limit: number) {
|
||||
return await this.measured('cleanupExpiredPendingBlobs', rt =>
|
||||
rt.cleanupExpiredPendingBlobs(cutoffMs, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async releaseDeletedBlobs(workspaceId: string, limit: number) {
|
||||
return await this.measured('releaseDeletedBlobs', rt =>
|
||||
rt.releaseDeletedBlobs(workspaceId, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async cleanupExpiredSnapshotHistories(limit: number) {
|
||||
return await this.measured('cleanupExpiredSnapshotHistories', rt =>
|
||||
rt.cleanupExpiredSnapshotHistories(limit)
|
||||
@@ -80,41 +66,6 @@ export class BackendRuntimeProvider
|
||||
);
|
||||
}
|
||||
|
||||
async backfillMissingBlobMetadata(
|
||||
workspaceId: string | null | undefined,
|
||||
limit: number
|
||||
) {
|
||||
return await this.measured('backfillMissingBlobMetadata', rt =>
|
||||
rt.backfillMissingBlobMetadata(workspaceId, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number) {
|
||||
return await this.measured('rebuildWorkspaceDocBlobRefs', rt =>
|
||||
rt.rebuildWorkspaceDocBlobRefs(workspaceId, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async planUnreferencedWorkspaceBlobs(
|
||||
workspaceId: string,
|
||||
gracePeriodDays: number,
|
||||
limit: number
|
||||
) {
|
||||
return await this.measured('planUnreferencedWorkspaceBlobs', rt =>
|
||||
rt.planUnreferencedWorkspaceBlobs(workspaceId, gracePeriodDays, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async executeBlobCleanupCandidates(
|
||||
runId: string,
|
||||
gracePeriodDays: number,
|
||||
limit: number
|
||||
) {
|
||||
return await this.measured('executeBlobCleanupCandidates', rt =>
|
||||
rt.executeBlobCleanupCandidates(runId, gracePeriodDays, limit)
|
||||
);
|
||||
}
|
||||
|
||||
private async measured<T>(
|
||||
method: string,
|
||||
fn: (runtime: RuntimeInstance) => Promise<T>
|
||||
|
||||
@@ -52,8 +52,19 @@ test.before(async t => {
|
||||
t.context.state = module.get(QuotaStateService);
|
||||
});
|
||||
|
||||
test.beforeEach(async t => {
|
||||
await t.context.module.initTestingDB();
|
||||
});
|
||||
|
||||
test.after.always(async t => {
|
||||
await t.context.module.close();
|
||||
});
|
||||
|
||||
test('quota service ignores dirty legacy commercial features', async t => {
|
||||
const { owner, workspace } = await createWorkspace(t);
|
||||
await t.context.state.reconcileUserQuotaState(owner.id);
|
||||
await t.context.state.reconcileWorkspaceQuotaState(workspace.id);
|
||||
|
||||
await t.context.models.userFeature.add(
|
||||
owner.id,
|
||||
'pro_plan_v1',
|
||||
@@ -306,6 +317,8 @@ test('ai entitlement is a capability overlay on free quota', async t => {
|
||||
|
||||
test('workspace team status ignores dirty legacy feature', async t => {
|
||||
const { workspace } = await createWorkspace(t);
|
||||
await t.context.state.reconcileWorkspaceQuotaState(workspace.id);
|
||||
|
||||
await t.context.models.workspaceFeature.add(
|
||||
workspace.id,
|
||||
'team_plan_v1',
|
||||
@@ -324,6 +337,7 @@ test('workspace team status ignores dirty legacy feature', async t => {
|
||||
status: 'active',
|
||||
quantity: 5,
|
||||
});
|
||||
await t.context.state.reconcileWorkspaceQuotaState(workspace.id);
|
||||
|
||||
t.true(await t.context.models.workspace.isTeamWorkspace(workspace.id));
|
||||
});
|
||||
@@ -357,14 +371,6 @@ test('selfhosted builtin free has cloud pro quota rights', async t => {
|
||||
}
|
||||
});
|
||||
|
||||
test.beforeEach(async t => {
|
||||
await t.context.module.initTestingDB();
|
||||
});
|
||||
|
||||
test.after.always(async t => {
|
||||
await t.context.module.close();
|
||||
});
|
||||
|
||||
test('reconciles quota states from entitlements and business tables', async t => {
|
||||
const previousDeploymentType = globalThis.env.DEPLOYMENT_TYPE;
|
||||
// @ts-expect-error test mutates env singleton for cloud entitlement semantics
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import test from 'ava';
|
||||
import Sinon from 'sinon';
|
||||
|
||||
import { StorageRuntimeProvider } from '../provider';
|
||||
|
||||
function createProvider() {
|
||||
const provider = new StorageRuntimeProvider({
|
||||
db: {
|
||||
datasourceUrl: 'postgresql://localhost:5432/affine',
|
||||
},
|
||||
storages: {
|
||||
blob: {
|
||||
storage: {
|
||||
provider: 'fs',
|
||||
bucket: 'blobs',
|
||||
config: { path: '~/.affine/storage' },
|
||||
},
|
||||
},
|
||||
avatar: {
|
||||
storage: {
|
||||
provider: 'fs',
|
||||
bucket: 'avatars',
|
||||
config: { path: '~/.affine/storage' },
|
||||
},
|
||||
},
|
||||
},
|
||||
copilot: {
|
||||
storage: {
|
||||
provider: 'fs',
|
||||
bucket: 'copilot',
|
||||
config: { path: '~/.affine/storage' },
|
||||
},
|
||||
},
|
||||
} as any);
|
||||
const runtime = {
|
||||
configure: Sinon.stub(),
|
||||
start: Sinon.stub().resolves(),
|
||||
stop: Sinon.stub().resolves(),
|
||||
runMigrations: Sinon.stub().resolves(),
|
||||
health: Sinon.stub().resolves({
|
||||
started: true,
|
||||
databaseConnected: true,
|
||||
provider: 'fs',
|
||||
}),
|
||||
};
|
||||
(provider as any).runtime = runtime;
|
||||
return { provider, runtime };
|
||||
}
|
||||
|
||||
test('storage-runtime provider restarts on storage config changes', async t => {
|
||||
const { provider, runtime } = createProvider();
|
||||
|
||||
await provider.start();
|
||||
await provider.onConfigChanged({ updates: { storages: {} } });
|
||||
|
||||
t.is(runtime.stop.callCount, 1);
|
||||
t.is(runtime.configure.callCount, 2);
|
||||
t.is(runtime.start.callCount, 2);
|
||||
t.is(runtime.runMigrations.callCount, 2);
|
||||
});
|
||||
|
||||
test('storage-runtime provider restarts on copilot storage config changes', async t => {
|
||||
const { provider, runtime } = createProvider();
|
||||
|
||||
await provider.start();
|
||||
await provider.onConfigChanged({
|
||||
updates: {
|
||||
copilot: {
|
||||
storage: {
|
||||
provider: 'fs',
|
||||
bucket: 'new-copilot',
|
||||
config: { path: '~/.affine/storage' },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
t.is(runtime.stop.callCount, 1);
|
||||
t.is(runtime.configure.callCount, 2);
|
||||
t.is(runtime.start.callCount, 2);
|
||||
t.is(runtime.runMigrations.callCount, 2);
|
||||
});
|
||||
|
||||
test('storage-runtime provider ignores unrelated config changes', async t => {
|
||||
const { provider, runtime } = createProvider();
|
||||
|
||||
await provider.start();
|
||||
await provider.onConfigChanged({ updates: { flags: {} } });
|
||||
|
||||
t.is(runtime.stop.callCount, 0);
|
||||
t.is(runtime.start.callCount, 1);
|
||||
t.is(runtime.runMigrations.callCount, 1);
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
|
||||
import { StorageRuntimeProvider } from './provider';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [StorageRuntimeProvider],
|
||||
exports: [StorageRuntimeProvider],
|
||||
})
|
||||
export class StorageRuntimeModule {}
|
||||
|
||||
export {
|
||||
type StorageRuntimeGetObjectResult,
|
||||
StorageRuntimeProvider,
|
||||
} from './provider';
|
||||
@@ -0,0 +1,374 @@
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
type OnApplicationBootstrap,
|
||||
type OnApplicationShutdown,
|
||||
} from '@nestjs/common';
|
||||
|
||||
import type {
|
||||
BlobOutputType,
|
||||
GetObjectMetadata,
|
||||
ListObjectsMetadata,
|
||||
PresignedUpload,
|
||||
PutObjectMetadata,
|
||||
} from '../../base';
|
||||
import { Config, OnEvent } from '../../base';
|
||||
import { wrapCallMetric } from '../../base/metrics';
|
||||
import {
|
||||
type RuntimeObjectGetResult,
|
||||
type RuntimeObjectListEntry,
|
||||
type RuntimeObjectMetadata,
|
||||
type RuntimePresignedObjectRequest,
|
||||
type StorageProviderCapabilities,
|
||||
StorageRuntime,
|
||||
type StorageRuntimeHealth,
|
||||
} from '../../native';
|
||||
|
||||
type RuntimeInstance = InstanceType<typeof StorageRuntime>;
|
||||
|
||||
@Injectable()
|
||||
export class StorageRuntimeProvider
|
||||
implements OnApplicationBootstrap, OnApplicationShutdown
|
||||
{
|
||||
private readonly logger = new Logger(StorageRuntimeProvider.name);
|
||||
private readonly runtime: RuntimeInstance = new StorageRuntime();
|
||||
private migrationsStarted = false;
|
||||
|
||||
constructor(private readonly config: Config) {}
|
||||
|
||||
async onApplicationBootstrap() {
|
||||
await this.start();
|
||||
}
|
||||
|
||||
async onApplicationShutdown() {
|
||||
await this.stop();
|
||||
}
|
||||
|
||||
async start() {
|
||||
this.configureRuntime();
|
||||
await this.runtime.start();
|
||||
await this.runMigrationsOnce();
|
||||
const health = await this.runtime.health();
|
||||
this.logger.log(
|
||||
`storage runtime started: db=${health.databaseConnected} provider=${health.provider ?? 'none'}`
|
||||
);
|
||||
}
|
||||
|
||||
async stop() {
|
||||
await this.runtime.stop();
|
||||
this.logger.log('storage runtime stopped');
|
||||
}
|
||||
|
||||
@OnEvent('config.init')
|
||||
async onConfigInit() {
|
||||
await this.start();
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
async onConfigChanged({ updates }: Events['config.changed']) {
|
||||
if (
|
||||
!('storages' in updates) &&
|
||||
!('db' in updates) &&
|
||||
!updates.copilot?.storage
|
||||
) {
|
||||
return;
|
||||
}
|
||||
await this.restart();
|
||||
}
|
||||
|
||||
async health(): Promise<StorageRuntimeHealth> {
|
||||
return await this.runtime.health();
|
||||
}
|
||||
|
||||
async providerCapabilities(
|
||||
scope: string
|
||||
): Promise<StorageProviderCapabilities> {
|
||||
return await this.measured('providerCapabilities', rt =>
|
||||
rt.providerCapabilities(scope)
|
||||
);
|
||||
}
|
||||
|
||||
async putObject(
|
||||
scope: string,
|
||||
key: string,
|
||||
body: Buffer,
|
||||
metadata?: PutObjectMetadata
|
||||
) {
|
||||
const result = await this.measured('putObject', rt =>
|
||||
rt.putObject(scope, key, body, toRuntimeMetadata(metadata))
|
||||
);
|
||||
return fromRuntimeMetadata(result);
|
||||
}
|
||||
|
||||
async headObject(scope: string, key: string) {
|
||||
const metadata = await this.measured('headObject', rt =>
|
||||
rt.headObject(scope, key)
|
||||
);
|
||||
return metadata ? fromRuntimeMetadata(metadata) : undefined;
|
||||
}
|
||||
|
||||
async getObject(
|
||||
scope: string,
|
||||
key: string
|
||||
): Promise<StorageRuntimeGetObjectResult> {
|
||||
const result = await this.measured('getObject', rt =>
|
||||
rt.getObject(scope, key)
|
||||
);
|
||||
return result ? fromRuntimeGetResult(result) : {};
|
||||
}
|
||||
|
||||
async listObjects(scope: string, prefix?: string) {
|
||||
const entries = await this.measured('listObjects', rt =>
|
||||
rt.listObjects(scope, prefix)
|
||||
);
|
||||
return entries.map(fromRuntimeListEntry);
|
||||
}
|
||||
|
||||
async deleteObject(scope: string, key: string) {
|
||||
await this.measured('deleteObject', rt => rt.deleteObject(scope, key));
|
||||
}
|
||||
|
||||
async presignPut(scope: string, key: string, metadata?: PutObjectMetadata) {
|
||||
const result = await this.measured('presignPut', rt =>
|
||||
rt.presignPut(scope, key, toRuntimeMetadata(metadata))
|
||||
);
|
||||
return result ? fromRuntimePresigned(result) : undefined;
|
||||
}
|
||||
|
||||
async presignGet(scope: string, key: string) {
|
||||
const result = await this.measured('presignGet', rt =>
|
||||
rt.presignGet(scope, key)
|
||||
);
|
||||
return result ? fromRuntimePresigned(result) : undefined;
|
||||
}
|
||||
|
||||
async createMultipartUpload(
|
||||
scope: string,
|
||||
key: string,
|
||||
metadata?: PutObjectMetadata
|
||||
) {
|
||||
const result = await this.measured('createMultipartUpload', rt =>
|
||||
rt.createMultipartUpload(scope, key, toRuntimeMetadata(metadata))
|
||||
);
|
||||
return result
|
||||
? { uploadId: result.uploadId, expiresAt: new Date(result.expiresAtMs) }
|
||||
: undefined;
|
||||
}
|
||||
|
||||
async presignUploadPart(
|
||||
scope: string,
|
||||
key: string,
|
||||
uploadId: string,
|
||||
partNumber: number
|
||||
) {
|
||||
const result = await this.measured('presignUploadPart', rt =>
|
||||
rt.presignUploadPart(scope, key, uploadId, partNumber)
|
||||
);
|
||||
return result ? fromRuntimePresigned(result) : undefined;
|
||||
}
|
||||
|
||||
async listMultipartUploadParts(scope: string, key: string, uploadId: string) {
|
||||
return (
|
||||
(await this.measured('listMultipartUploadParts', rt =>
|
||||
rt.listMultipartUploadParts(scope, key, uploadId)
|
||||
)) ?? undefined
|
||||
);
|
||||
}
|
||||
|
||||
async proxyUploadPart(
|
||||
scope: string,
|
||||
key: string,
|
||||
uploadId: string,
|
||||
partNumber: number,
|
||||
body: Buffer,
|
||||
contentLength?: number
|
||||
) {
|
||||
return (
|
||||
(await this.measured('proxyUploadPart', rt =>
|
||||
rt.proxyUploadPart(
|
||||
scope,
|
||||
key,
|
||||
uploadId,
|
||||
partNumber,
|
||||
body,
|
||||
contentLength
|
||||
)
|
||||
)) ?? undefined
|
||||
);
|
||||
}
|
||||
|
||||
async completeMultipartUpload(
|
||||
scope: string,
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: { partNumber: number; etag: string }[]
|
||||
) {
|
||||
return await this.measured('completeMultipartUpload', rt =>
|
||||
rt.completeMultipartUpload(scope, key, uploadId, parts)
|
||||
);
|
||||
}
|
||||
|
||||
async abortMultipartUpload(scope: string, key: string, uploadId: string) {
|
||||
return await this.measured('abortMultipartUpload', rt =>
|
||||
rt.abortMultipartUpload(scope, key, uploadId)
|
||||
);
|
||||
}
|
||||
|
||||
async completeWorkspaceBlobUpload(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
expected: { size: number; mime: string }
|
||||
) {
|
||||
return await this.measured('completeWorkspaceBlobUpload', rt =>
|
||||
rt.completeWorkspaceBlobUpload(
|
||||
workspaceId,
|
||||
key,
|
||||
expected.size,
|
||||
expected.mime
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async cleanupExpiredPendingBlobs(cutoffMs: number, limit: number) {
|
||||
return await this.measured('cleanupExpiredPendingBlobs', rt =>
|
||||
rt.cleanupExpiredPendingBlobs(cutoffMs, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async releaseDeletedBlobs(workspaceId: string, limit: number) {
|
||||
return await this.measured('releaseDeletedBlobs', rt =>
|
||||
rt.releaseDeletedBlobs(workspaceId, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async backfillMissingBlobMetadata(
|
||||
workspaceId: string | null | undefined,
|
||||
limit: number
|
||||
) {
|
||||
return await this.measured('backfillMissingBlobMetadata', rt =>
|
||||
rt.backfillMissingBlobMetadata(workspaceId, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number) {
|
||||
return await this.measured('rebuildWorkspaceDocBlobRefs', rt =>
|
||||
rt.rebuildWorkspaceDocBlobRefs(workspaceId, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async planUnreferencedWorkspaceBlobs(
|
||||
workspaceId: string,
|
||||
gracePeriodDays: number,
|
||||
limit: number
|
||||
) {
|
||||
return await this.measured('planUnreferencedWorkspaceBlobs', rt =>
|
||||
rt.planUnreferencedWorkspaceBlobs(workspaceId, gracePeriodDays, limit)
|
||||
);
|
||||
}
|
||||
|
||||
async executeBlobCleanupCandidates(
|
||||
runId: string,
|
||||
gracePeriodDays: number,
|
||||
limit: number
|
||||
) {
|
||||
return await this.measured('executeBlobCleanupCandidates', rt =>
|
||||
rt.executeBlobCleanupCandidates(runId, gracePeriodDays, limit)
|
||||
);
|
||||
}
|
||||
|
||||
private async measured<T>(
|
||||
method: string,
|
||||
fn: (runtime: RuntimeInstance) => Promise<T>
|
||||
): Promise<T> {
|
||||
return await wrapCallMetric(() => fn(this.runtime), 'storage', 'runtime', {
|
||||
method,
|
||||
})();
|
||||
}
|
||||
|
||||
private async runMigrationsOnce() {
|
||||
if (this.migrationsStarted) {
|
||||
return;
|
||||
}
|
||||
await this.runtime.runMigrations();
|
||||
this.migrationsStarted = true;
|
||||
}
|
||||
|
||||
private async restart() {
|
||||
await this.runtime.stop();
|
||||
this.migrationsStarted = false;
|
||||
await this.start();
|
||||
}
|
||||
|
||||
private configureRuntime() {
|
||||
this.runtime.configure(
|
||||
JSON.stringify({
|
||||
db: {
|
||||
datasourceUrl: this.config.db.datasourceUrl,
|
||||
},
|
||||
storages: {
|
||||
'blob.storage': this.config.storages.blob.storage,
|
||||
'avatar.storage': this.config.storages.avatar.storage,
|
||||
},
|
||||
copilot: {
|
||||
storage: this.config.copilot.storage,
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function toRuntimeMetadata(metadata?: PutObjectMetadata) {
|
||||
return metadata
|
||||
? {
|
||||
contentType: metadata.contentType,
|
||||
contentLength: metadata.contentLength,
|
||||
checksumCrc32: metadata.checksumCRC32,
|
||||
}
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function fromRuntimeMetadata(
|
||||
metadata: RuntimeObjectMetadata
|
||||
): GetObjectMetadata {
|
||||
return {
|
||||
contentType: metadata.contentType,
|
||||
contentLength: metadata.contentLength,
|
||||
lastModified: new Date(metadata.lastModifiedMs),
|
||||
checksumCRC32: metadata.checksumCrc32,
|
||||
};
|
||||
}
|
||||
|
||||
function fromRuntimeGetResult(result: RuntimeObjectGetResult) {
|
||||
return {
|
||||
body: Readable.from(result.body),
|
||||
metadata: fromRuntimeMetadata(result.metadata),
|
||||
};
|
||||
}
|
||||
|
||||
function fromRuntimeListEntry(
|
||||
entry: RuntimeObjectListEntry
|
||||
): ListObjectsMetadata {
|
||||
return {
|
||||
key: entry.key,
|
||||
contentLength: entry.contentLength,
|
||||
lastModified: new Date(entry.lastModifiedMs),
|
||||
};
|
||||
}
|
||||
|
||||
function fromRuntimePresigned(
|
||||
request: RuntimePresignedObjectRequest
|
||||
): PresignedUpload {
|
||||
return {
|
||||
url: request.url,
|
||||
headers: JSON.parse(request.headersJson) as Record<string, string>,
|
||||
expiresAt: new Date(request.expiresAtMs),
|
||||
};
|
||||
}
|
||||
|
||||
export type StorageRuntimeGetObjectResult = {
|
||||
redirectUrl?: string;
|
||||
body?: BlobOutputType;
|
||||
metadata?: GetObjectMetadata;
|
||||
};
|
||||
+7
-5
@@ -1,7 +1,7 @@
|
||||
import ava, { TestFn } from 'ava';
|
||||
import Sinon from 'sinon';
|
||||
|
||||
import { BackendRuntimeBlobJob } from '../blob-job';
|
||||
import { StorageBlobJob } from '../blob-job';
|
||||
|
||||
interface Context {
|
||||
runtime: {
|
||||
@@ -22,7 +22,7 @@ interface Context {
|
||||
findMany: Sinon.SinonStub;
|
||||
};
|
||||
};
|
||||
job: BackendRuntimeBlobJob;
|
||||
job: StorageBlobJob;
|
||||
}
|
||||
|
||||
const test = ava as TestFn<Context>;
|
||||
@@ -31,7 +31,8 @@ test.beforeEach(t => {
|
||||
t.context.runtime = {
|
||||
health: Sinon.stub().resolves({
|
||||
databaseConnected: true,
|
||||
objectStorageConfigured: true,
|
||||
providerConfigured: true,
|
||||
provider: 'fs',
|
||||
}),
|
||||
backfillMissingBlobMetadata: Sinon.stub(),
|
||||
rebuildWorkspaceDocBlobRefs: Sinon.stub(),
|
||||
@@ -49,7 +50,7 @@ test.beforeEach(t => {
|
||||
findMany: Sinon.stub(),
|
||||
},
|
||||
};
|
||||
t.context.job = new BackendRuntimeBlobJob(
|
||||
t.context.job = new StorageBlobJob(
|
||||
t.context.runtime as any,
|
||||
t.context.event as any,
|
||||
t.context.queue as any,
|
||||
@@ -103,7 +104,8 @@ for (const scenario of objectStorageRequiredCases) {
|
||||
test(`${scenario.name} skips when object storage is not configured`, async t => {
|
||||
t.context.runtime.health.resolves({
|
||||
databaseConnected: true,
|
||||
objectStorageConfigured: false,
|
||||
providerConfigured: true,
|
||||
provider: undefined,
|
||||
});
|
||||
|
||||
await scenario.run(t.context);
|
||||
@@ -5,17 +5,66 @@ import test from 'ava';
|
||||
|
||||
import { createModule } from '../../../__tests__/create-module';
|
||||
import { Mockers } from '../../../__tests__/mocks';
|
||||
import { initTestingDB } from '../../../__tests__/utils';
|
||||
import { Models } from '../../../models';
|
||||
import { getMime } from '../../../native';
|
||||
import { StorageRuntimeProvider } from '../../storage-runtime';
|
||||
import { CommentAttachmentStorage, StorageModule } from '../index';
|
||||
|
||||
const objects = new Map<
|
||||
string,
|
||||
{
|
||||
body: Buffer;
|
||||
metadata: {
|
||||
contentType: string;
|
||||
contentLength: number;
|
||||
lastModified: Date;
|
||||
};
|
||||
}
|
||||
>();
|
||||
const storageRuntime = {
|
||||
putObject: async (
|
||||
_scope: string,
|
||||
key: string,
|
||||
body: Buffer,
|
||||
_metadata?: { contentType?: string; contentLength?: number }
|
||||
) => {
|
||||
const object = {
|
||||
body,
|
||||
metadata: {
|
||||
contentType: getMime(body),
|
||||
contentLength: body.length,
|
||||
lastModified: new Date(),
|
||||
},
|
||||
};
|
||||
objects.set(key, object);
|
||||
return object.metadata;
|
||||
},
|
||||
headObject: async (_scope: string, key: string) => objects.get(key)?.metadata,
|
||||
getObject: async (_scope: string, key: string) => {
|
||||
const object = objects.get(key);
|
||||
return object
|
||||
? { body: Readable.from(object.body), metadata: object.metadata }
|
||||
: {};
|
||||
},
|
||||
deleteObject: async (_scope: string, key: string) => {
|
||||
objects.delete(key);
|
||||
},
|
||||
presignGet: async () => undefined,
|
||||
};
|
||||
|
||||
const module = await createModule({
|
||||
imports: [StorageModule],
|
||||
tapModule: builder => {
|
||||
builder.overrideProvider(StorageRuntimeProvider).useValue(storageRuntime);
|
||||
},
|
||||
});
|
||||
const storage = module.get(CommentAttachmentStorage);
|
||||
const models = module.get(Models);
|
||||
|
||||
test.before(async () => {
|
||||
await storage.onConfigInit();
|
||||
test.beforeEach(async () => {
|
||||
await initTestingDB(module);
|
||||
objects.clear();
|
||||
});
|
||||
|
||||
test.after.always(async () => {
|
||||
|
||||
+10
-8
@@ -3,8 +3,10 @@ import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
import { EventBus, JobQueue, OnJob } from '../../base';
|
||||
import { BackendRuntimeProvider } from './provider';
|
||||
import { StorageRuntimeProvider } from '../storage-runtime';
|
||||
|
||||
// Queue keys are persisted API; keep the legacy backendRuntime.* names while
|
||||
// StorageBlobJob and StorageRuntimeProvider own the implementation.
|
||||
declare global {
|
||||
interface Jobs {
|
||||
'backendRuntime.backfillMissingBlobMetadata': {
|
||||
@@ -45,11 +47,11 @@ declare global {
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class BackendRuntimeBlobJob {
|
||||
private readonly logger = new Logger(BackendRuntimeBlobJob.name);
|
||||
export class StorageBlobJob {
|
||||
private readonly logger = new Logger(StorageBlobJob.name);
|
||||
|
||||
constructor(
|
||||
private readonly rt: BackendRuntimeProvider,
|
||||
private readonly rt: StorageRuntimeProvider,
|
||||
private readonly event: EventBus,
|
||||
private readonly queue: JobQueue,
|
||||
private readonly db: PrismaClient
|
||||
@@ -355,7 +357,7 @@ export class BackendRuntimeBlobJob {
|
||||
limit
|
||||
);
|
||||
await Promise.all(
|
||||
result.workspaceIds.map(workspaceId =>
|
||||
result.workspaceIds.map((workspaceId: string) =>
|
||||
this.event.emitAsync('workspace.blobs.updated', { workspaceId })
|
||||
)
|
||||
);
|
||||
@@ -375,7 +377,7 @@ export class BackendRuntimeBlobJob {
|
||||
limit
|
||||
);
|
||||
await Promise.all(
|
||||
result.workspaceIds.map(workspaceId =>
|
||||
result.workspaceIds.map((workspaceId: string) =>
|
||||
this.event.emitAsync('workspace.blobs.updated', { workspaceId })
|
||||
)
|
||||
);
|
||||
@@ -409,12 +411,12 @@ export class BackendRuntimeBlobJob {
|
||||
|
||||
private async hasObjectStorage(operation: string) {
|
||||
const health = await this.rt.health();
|
||||
if (health.objectStorageConfigured) {
|
||||
if (health.provider) {
|
||||
return true;
|
||||
}
|
||||
|
||||
this.logger.warn(
|
||||
`skip ${operation}: BackendRuntime object storage is not configured`
|
||||
`skip ${operation}: StorageRuntime provider is not configured`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
@@ -2,7 +2,8 @@ import './config';
|
||||
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { BackendRuntimeModule } from '../backend-runtime';
|
||||
import { StorageRuntimeModule } from '../storage-runtime';
|
||||
import { StorageBlobJob } from './blob-job';
|
||||
import { BlobUploadCleanupJob } from './job';
|
||||
import { R2UploadController } from './r2-proxy';
|
||||
import {
|
||||
@@ -12,16 +13,23 @@ import {
|
||||
} from './wrappers';
|
||||
|
||||
@Module({
|
||||
imports: [BackendRuntimeModule],
|
||||
imports: [StorageRuntimeModule],
|
||||
controllers: [R2UploadController],
|
||||
providers: [
|
||||
WorkspaceBlobStorage,
|
||||
AvatarStorage,
|
||||
CommentAttachmentStorage,
|
||||
StorageBlobJob,
|
||||
BlobUploadCleanupJob,
|
||||
],
|
||||
exports: [WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage],
|
||||
exports: [
|
||||
WorkspaceBlobStorage,
|
||||
AvatarStorage,
|
||||
CommentAttachmentStorage,
|
||||
StorageBlobJob,
|
||||
],
|
||||
})
|
||||
export class StorageModule {}
|
||||
|
||||
export { StorageBlobJob } from './blob-job';
|
||||
export { AvatarStorage, CommentAttachmentStorage, WorkspaceBlobStorage };
|
||||
|
||||
@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
|
||||
import { EventBus, JobQueue, OneDay, OnJob } from '../../base';
|
||||
import { BackendRuntimeProvider } from '../backend-runtime';
|
||||
import { StorageRuntimeProvider } from '../storage-runtime';
|
||||
|
||||
declare global {
|
||||
interface Jobs {
|
||||
@@ -15,7 +15,7 @@ export class BlobUploadCleanupJob {
|
||||
private readonly logger = new Logger(BlobUploadCleanupJob.name);
|
||||
|
||||
constructor(
|
||||
private readonly rt: BackendRuntimeProvider,
|
||||
private readonly rt: StorageRuntimeProvider,
|
||||
private readonly event: EventBus,
|
||||
private readonly queue: JobQueue
|
||||
) {}
|
||||
|
||||
@@ -7,19 +7,16 @@ import {
|
||||
BlobInvalid,
|
||||
CallMetric,
|
||||
Config,
|
||||
OnEvent,
|
||||
PROXY_MULTIPART_PATH,
|
||||
PROXY_UPLOAD_PATH,
|
||||
type R2StorageConfig,
|
||||
STORAGE_PROXY_ROOT,
|
||||
StorageProviderConfig,
|
||||
StorageProviderFactory,
|
||||
type StorageProviderConfig,
|
||||
toBuffer,
|
||||
} from '../../base';
|
||||
import {
|
||||
R2StorageConfig,
|
||||
R2StorageProvider,
|
||||
} from '../../base/storage/providers/r2';
|
||||
import { Models } from '../../models';
|
||||
import { Public } from '../auth/guard';
|
||||
import { StorageRuntimeProvider } from '../storage-runtime';
|
||||
import { MULTIPART_PART_SIZE } from './constants';
|
||||
|
||||
type R2BlobStorageConfig = StorageProviderConfig & {
|
||||
@@ -37,21 +34,13 @@ type R2Config = {
|
||||
@Controller(STORAGE_PROXY_ROOT)
|
||||
export class R2UploadController {
|
||||
private readonly logger = new Logger(R2UploadController.name);
|
||||
private provider: R2StorageProvider | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly models: Models,
|
||||
private readonly storageFactory: StorageProviderFactory
|
||||
private readonly rt: StorageRuntimeProvider
|
||||
) {}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
onConfigChanged(event: Events['config.changed']) {
|
||||
if (event.updates.storages?.blob?.storage) {
|
||||
this.provider = null;
|
||||
}
|
||||
}
|
||||
|
||||
private getR2Config(): R2Config {
|
||||
const storage = this.config.storages.blob.storage as StorageProviderConfig;
|
||||
if (storage.provider !== 'cloudflare-r2') {
|
||||
@@ -69,16 +58,6 @@ export class R2UploadController {
|
||||
return { storage: storage as R2BlobStorageConfig, signKey };
|
||||
}
|
||||
|
||||
private getProvider(storage: R2BlobStorageConfig) {
|
||||
if (!this.provider) {
|
||||
const candidate = this.storageFactory.create(storage);
|
||||
if (candidate instanceof R2StorageProvider) {
|
||||
this.provider = candidate;
|
||||
}
|
||||
}
|
||||
return this.provider;
|
||||
}
|
||||
|
||||
private sign(canonical: string, signKey: string) {
|
||||
return createHmac('sha256', signKey).update(canonical).digest('base64');
|
||||
}
|
||||
@@ -173,7 +152,7 @@ export class R2UploadController {
|
||||
@Put('upload')
|
||||
@CallMetric('controllers', 'r2_proxy_upload')
|
||||
async upload(@Req() req: Request, @Res() res: Response) {
|
||||
const { storage, signKey } = this.getR2Config();
|
||||
const { signKey } = this.getR2Config();
|
||||
|
||||
const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId');
|
||||
const key = this.expectString(req.query.key, 'key');
|
||||
@@ -229,16 +208,16 @@ export class R2UploadController {
|
||||
throw new BlobInvalid('Mime type mismatch');
|
||||
}
|
||||
|
||||
const provider = this.getProvider(storage);
|
||||
if (!provider) {
|
||||
throw new BlobInvalid('R2 provider is not available');
|
||||
}
|
||||
|
||||
try {
|
||||
await provider.proxyPutObject(`${workspaceId}/${key}`, req, {
|
||||
contentType: mime,
|
||||
contentLength,
|
||||
});
|
||||
await this.rt.putObject(
|
||||
'blob',
|
||||
`${workspaceId}/${key}`,
|
||||
await toBuffer(req),
|
||||
{
|
||||
contentType: mime,
|
||||
contentLength,
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to proxy upload', error as Error);
|
||||
throw new BlobInvalid('Upload failed');
|
||||
@@ -251,7 +230,7 @@ export class R2UploadController {
|
||||
@Put('multipart')
|
||||
@CallMetric('controllers', 'r2_proxy_multipart')
|
||||
async uploadPart(@Req() req: Request, @Res() res: Response) {
|
||||
const { storage, signKey } = this.getR2Config();
|
||||
const { signKey } = this.getR2Config();
|
||||
|
||||
const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId');
|
||||
const key = this.expectString(req.query.key, 'key');
|
||||
@@ -305,18 +284,14 @@ export class R2UploadController {
|
||||
throw new BlobInvalid('Part size exceeds upload metadata');
|
||||
}
|
||||
|
||||
const provider = this.getProvider(storage);
|
||||
if (!provider) {
|
||||
throw new BlobInvalid('R2 provider is not available');
|
||||
}
|
||||
|
||||
try {
|
||||
const etag = await provider.proxyUploadPart(
|
||||
const etag = await this.rt.proxyUploadPart(
|
||||
'blob',
|
||||
`${workspaceId}/${key}`,
|
||||
uploadId,
|
||||
partNumber,
|
||||
req,
|
||||
{ contentLength }
|
||||
await toBuffer(req),
|
||||
contentLength
|
||||
);
|
||||
if (etag) {
|
||||
res.setHeader('etag', etag);
|
||||
|
||||
@@ -1,21 +1,11 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import type {
|
||||
BlobInputType,
|
||||
PutObjectMetadata,
|
||||
StorageProvider,
|
||||
} from '../../../base';
|
||||
import {
|
||||
Config,
|
||||
OnEvent,
|
||||
StorageProviderFactory,
|
||||
URLHelper,
|
||||
} from '../../../base';
|
||||
import type { BlobInputType, PutObjectMetadata } from '../../../base';
|
||||
import { Config, OnEvent, toBuffer, URLHelper } from '../../../base';
|
||||
import { StorageRuntimeProvider } from '../../storage-runtime';
|
||||
|
||||
@Injectable()
|
||||
export class AvatarStorage {
|
||||
private provider!: StorageProvider;
|
||||
|
||||
get config() {
|
||||
return this.AFFiNEConfig.storages.avatar;
|
||||
}
|
||||
@@ -23,23 +13,11 @@ export class AvatarStorage {
|
||||
constructor(
|
||||
private readonly AFFiNEConfig: Config,
|
||||
private readonly url: URLHelper,
|
||||
private readonly storageFactory: StorageProviderFactory
|
||||
private readonly rt: StorageRuntimeProvider
|
||||
) {}
|
||||
|
||||
@OnEvent('config.init')
|
||||
async onConfigInit() {
|
||||
this.provider = this.storageFactory.create(this.config.storage);
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
async onConfigChanged(event: Events['config.changed']) {
|
||||
if (event.updates.storages?.avatar?.storage) {
|
||||
this.provider = this.storageFactory.create(this.config.storage);
|
||||
}
|
||||
}
|
||||
|
||||
async put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) {
|
||||
await this.provider.put(key, blob, metadata);
|
||||
await this.rt.putObject('avatar', key, await toBuffer(blob), metadata);
|
||||
let link = this.config.publicPath + key;
|
||||
|
||||
if (link.startsWith('/')) {
|
||||
@@ -50,11 +28,11 @@ export class AvatarStorage {
|
||||
}
|
||||
|
||||
get(key: string) {
|
||||
return this.provider.get(key);
|
||||
return this.rt.getObject('avatar', key);
|
||||
}
|
||||
|
||||
delete(link: string) {
|
||||
return this.provider.delete(link.split('/').pop() as string);
|
||||
return this.rt.deleteObject('avatar', link.split('/').pop() as string);
|
||||
}
|
||||
|
||||
@OnEvent('user.deleted')
|
||||
|
||||
@@ -1,20 +1,24 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createHmac } from 'node:crypto';
|
||||
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
autoMetadata,
|
||||
type BlobOutputType,
|
||||
Config,
|
||||
EventBus,
|
||||
type GetObjectMetadata,
|
||||
OnEvent,
|
||||
PutObjectMetadata,
|
||||
type StorageProvider,
|
||||
StorageProviderFactory,
|
||||
PROXY_MULTIPART_PATH,
|
||||
PROXY_UPLOAD_PATH,
|
||||
type PutObjectMetadata,
|
||||
type R2StorageConfig,
|
||||
SIGNED_URL_EXPIRED,
|
||||
type StorageProviderConfig,
|
||||
URLHelper,
|
||||
} from '../../../base';
|
||||
import { Models } from '../../../models';
|
||||
import { BackendRuntimeProvider } from '../../backend-runtime';
|
||||
import type { StorageProviderCapabilities } from '../../../native';
|
||||
import { StorageRuntimeProvider } from '../../storage-runtime';
|
||||
|
||||
declare global {
|
||||
interface Events {
|
||||
@@ -36,52 +40,75 @@ type BlobCompleteResult =
|
||||
| 'not_found'
|
||||
| 'size_mismatch'
|
||||
| 'mime_mismatch'
|
||||
| 'checksum_mismatch';
|
||||
| 'checksum_mismatch'
|
||||
| 'size_too_large';
|
||||
};
|
||||
|
||||
type BlobGetResult = {
|
||||
redirectUrl?: string;
|
||||
body?: BlobOutputType;
|
||||
metadata?: GetObjectMetadata;
|
||||
};
|
||||
|
||||
type R2ProxyConfig = {
|
||||
signKey: string;
|
||||
urlPrefix: string;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceBlobStorage {
|
||||
private readonly logger = new Logger(WorkspaceBlobStorage.name);
|
||||
private provider!: StorageProvider;
|
||||
|
||||
get config() {
|
||||
return this.AFFiNEConfig.storages.blob;
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly AFFiNEConfig: Config,
|
||||
private readonly event: EventBus,
|
||||
private readonly storageFactory: StorageProviderFactory,
|
||||
private readonly models: Models,
|
||||
private readonly url: URLHelper,
|
||||
private readonly rt: BackendRuntimeProvider
|
||||
private readonly rt: StorageRuntimeProvider,
|
||||
private readonly config: Config
|
||||
) {}
|
||||
|
||||
@OnEvent('config.init')
|
||||
async onConfigInit() {
|
||||
this.provider = this.storageFactory.create(this.config.storage);
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
async onConfigChanged(event: Events['config.changed']) {
|
||||
if (event.updates.storages?.blob?.storage) {
|
||||
this.provider = this.storageFactory.create(this.config.storage);
|
||||
}
|
||||
}
|
||||
|
||||
async put(workspaceId: string, key: string, blob: Buffer) {
|
||||
const meta: PutObjectMetadata = autoMetadata(blob);
|
||||
|
||||
await this.provider.put(`${workspaceId}/${key}`, blob, meta);
|
||||
const metadata = await this.rt.putObject(
|
||||
'blob',
|
||||
`${workspaceId}/${key}`,
|
||||
blob
|
||||
);
|
||||
await this.upsert(workspaceId, key, {
|
||||
contentType: meta.contentType ?? 'application/octet-stream',
|
||||
contentLength: blob.length,
|
||||
lastModified: new Date(),
|
||||
contentType: metadata.contentType,
|
||||
contentLength: metadata.contentLength,
|
||||
lastModified: metadata.lastModified,
|
||||
});
|
||||
}
|
||||
|
||||
async get(workspaceId: string, key: string, signedUrl?: boolean) {
|
||||
return this.provider.get(`${workspaceId}/${key}`, signedUrl);
|
||||
async capabilities(): Promise<StorageProviderCapabilities> {
|
||||
const capabilities = await this.rt.providerCapabilities('blob');
|
||||
if (!this.r2ProxyConfig()) {
|
||||
return capabilities;
|
||||
}
|
||||
return {
|
||||
...capabilities,
|
||||
presignPut: true,
|
||||
multipartDirect: true,
|
||||
proxyUpload: true,
|
||||
serverMediatedOnly: false,
|
||||
};
|
||||
}
|
||||
|
||||
async get(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
signedUrl?: boolean
|
||||
): Promise<BlobGetResult> {
|
||||
if (signedUrl) {
|
||||
const presigned = await this.rt.presignGet(
|
||||
'blob',
|
||||
`${workspaceId}/${key}`
|
||||
);
|
||||
if (presigned) {
|
||||
return { redirectUrl: presigned.url };
|
||||
}
|
||||
}
|
||||
return this.rt.getObject('blob', `${workspaceId}/${key}`);
|
||||
}
|
||||
|
||||
async presignPut(
|
||||
@@ -89,7 +116,11 @@ export class WorkspaceBlobStorage {
|
||||
key: string,
|
||||
metadata?: PutObjectMetadata
|
||||
) {
|
||||
return this.provider.presignPut?.(`${workspaceId}/${key}`, metadata);
|
||||
const proxy = this.r2ProxyConfig();
|
||||
if (proxy) {
|
||||
return this.createProxyUploadUrl(workspaceId, key, metadata, proxy);
|
||||
}
|
||||
return this.rt.presignPut('blob', `${workspaceId}/${key}`, metadata);
|
||||
}
|
||||
|
||||
async createMultipartUpload(
|
||||
@@ -97,7 +128,8 @@ export class WorkspaceBlobStorage {
|
||||
key: string,
|
||||
metadata?: PutObjectMetadata
|
||||
) {
|
||||
return this.provider.createMultipartUpload?.(
|
||||
return this.rt.createMultipartUpload(
|
||||
'blob',
|
||||
`${workspaceId}/${key}`,
|
||||
metadata
|
||||
);
|
||||
@@ -109,7 +141,18 @@ export class WorkspaceBlobStorage {
|
||||
uploadId: string,
|
||||
partNumber: number
|
||||
) {
|
||||
return this.provider.presignUploadPart?.(
|
||||
const proxy = this.r2ProxyConfig();
|
||||
if (proxy) {
|
||||
return this.createProxyMultipartUrl(
|
||||
workspaceId,
|
||||
key,
|
||||
uploadId,
|
||||
partNumber,
|
||||
proxy
|
||||
);
|
||||
}
|
||||
return this.rt.presignUploadPart(
|
||||
'blob',
|
||||
`${workspaceId}/${key}`,
|
||||
uploadId,
|
||||
partNumber
|
||||
@@ -121,7 +164,8 @@ export class WorkspaceBlobStorage {
|
||||
key: string,
|
||||
uploadId: string
|
||||
) {
|
||||
return this.provider.listMultipartUploadParts?.(
|
||||
return this.rt.listMultipartUploadParts(
|
||||
'blob',
|
||||
`${workspaceId}/${key}`,
|
||||
uploadId
|
||||
);
|
||||
@@ -133,16 +177,12 @@ export class WorkspaceBlobStorage {
|
||||
uploadId: string,
|
||||
parts: { partNumber: number; etag: string }[]
|
||||
) {
|
||||
if (!this.provider.completeMultipartUpload) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await this.provider.completeMultipartUpload(
|
||||
return await this.rt.completeMultipartUpload(
|
||||
'blob',
|
||||
`${workspaceId}/${key}`,
|
||||
uploadId,
|
||||
parts
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
async abortMultipartUpload(
|
||||
@@ -150,16 +190,15 @@ export class WorkspaceBlobStorage {
|
||||
key: string,
|
||||
uploadId: string
|
||||
) {
|
||||
if (!this.provider.abortMultipartUpload) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await this.provider.abortMultipartUpload(`${workspaceId}/${key}`, uploadId);
|
||||
return true;
|
||||
return await this.rt.abortMultipartUpload(
|
||||
'blob',
|
||||
`${workspaceId}/${key}`,
|
||||
uploadId
|
||||
);
|
||||
}
|
||||
|
||||
async head(workspaceId: string, key: string) {
|
||||
return this.provider.head(`${workspaceId}/${key}`);
|
||||
return this.rt.headObject('blob', `${workspaceId}/${key}`);
|
||||
}
|
||||
|
||||
async complete(
|
||||
@@ -167,57 +206,28 @@ export class WorkspaceBlobStorage {
|
||||
key: string,
|
||||
expected: { size: number; mime: string }
|
||||
): Promise<BlobCompleteResult> {
|
||||
const metadata = await this.head(workspaceId, key);
|
||||
if (!metadata) {
|
||||
return { ok: false, reason: 'not_found' };
|
||||
}
|
||||
|
||||
if (metadata.contentLength !== expected.size) {
|
||||
return { ok: false, reason: 'size_mismatch' };
|
||||
}
|
||||
|
||||
if (expected.mime && metadata.contentType !== expected.mime) {
|
||||
return { ok: false, reason: 'mime_mismatch' };
|
||||
}
|
||||
|
||||
const object = await this.provider.get(`${workspaceId}/${key}`);
|
||||
if (!object.body) {
|
||||
return { ok: false, reason: 'not_found' };
|
||||
}
|
||||
|
||||
const checksum = createHash('sha256');
|
||||
try {
|
||||
for await (const chunk of object.body) {
|
||||
checksum.update(chunk as Buffer);
|
||||
}
|
||||
} catch (e) {
|
||||
this.logger.error('failed to read blob for checksum verification', e);
|
||||
return { ok: false, reason: 'checksum_mismatch' };
|
||||
}
|
||||
|
||||
const base64 = checksum.digest('base64');
|
||||
const base64urlWithPadding = base64.replace(/\+/g, '-').replace(/\//g, '_');
|
||||
|
||||
if (base64urlWithPadding !== key) {
|
||||
try {
|
||||
await this.provider.delete(`${workspaceId}/${key}`);
|
||||
} catch (e) {
|
||||
// never throw
|
||||
this.logger.error('failed to delete invalid blob', e);
|
||||
}
|
||||
return { ok: false, reason: 'checksum_mismatch' };
|
||||
}
|
||||
|
||||
await this.models.blob.upsert({
|
||||
const result = await this.rt.completeWorkspaceBlobUpload(
|
||||
workspaceId,
|
||||
key,
|
||||
mime: metadata.contentType,
|
||||
size: metadata.contentLength,
|
||||
status: 'completed',
|
||||
uploadId: null,
|
||||
});
|
||||
|
||||
return { ok: true, metadata };
|
||||
expected
|
||||
);
|
||||
if (!result.ok) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: (result.reason ?? 'checksum_mismatch') as Exclude<
|
||||
BlobCompleteResult,
|
||||
{ ok: true }
|
||||
>['reason'],
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
metadata: {
|
||||
contentType: result.contentType ?? 'application/octet-stream',
|
||||
contentLength: result.contentLength ?? expected.size,
|
||||
lastModified: new Date(result.lastModifiedMs ?? Date.now()),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async list(workspaceId: string) {
|
||||
@@ -226,7 +236,7 @@ export class WorkspaceBlobStorage {
|
||||
|
||||
async delete(workspaceId: string, key: string, permanently = false) {
|
||||
if (permanently) {
|
||||
await this.provider.delete(`${workspaceId}/${key}`);
|
||||
await this.rt.deleteObject('blob', `${workspaceId}/${key}`);
|
||||
}
|
||||
await this.models.blob.delete(workspaceId, key, permanently);
|
||||
if (!permanently) {
|
||||
@@ -297,4 +307,115 @@ export class WorkspaceBlobStorage {
|
||||
}: Events['workspace.blob.delete']) {
|
||||
await this.delete(workspaceId, key, true);
|
||||
}
|
||||
|
||||
private r2ProxyConfig() {
|
||||
const storage = this.config.storages.blob.storage as StorageProviderConfig;
|
||||
if (storage.provider !== 'cloudflare-r2') {
|
||||
return;
|
||||
}
|
||||
const r2 = storage.config as R2StorageConfig;
|
||||
const usePresignedURL = r2.usePresignedURL;
|
||||
if (
|
||||
!usePresignedURL?.enabled ||
|
||||
!usePresignedURL.urlPrefix ||
|
||||
!usePresignedURL.signKey
|
||||
) {
|
||||
return;
|
||||
}
|
||||
return {
|
||||
signKey: usePresignedURL.signKey,
|
||||
urlPrefix: usePresignedURL.urlPrefix,
|
||||
};
|
||||
}
|
||||
|
||||
private signProxy(
|
||||
path: string,
|
||||
canonicalFields: (string | number | undefined)[],
|
||||
exp: number,
|
||||
signKey: string
|
||||
) {
|
||||
const canonical = [
|
||||
path,
|
||||
...canonicalFields.map(field =>
|
||||
field === undefined ? '' : field.toString()
|
||||
),
|
||||
exp.toString(),
|
||||
].join('\n');
|
||||
return `${exp}-${createHmac('sha256', signKey).update(canonical).digest('base64')}`;
|
||||
}
|
||||
|
||||
private createProxyUploadUrl(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
metadata: PutObjectMetadata | undefined,
|
||||
proxy: R2ProxyConfig
|
||||
) {
|
||||
const contentType = metadata?.contentType ?? 'application/octet-stream';
|
||||
const contentLength = metadata?.contentLength;
|
||||
const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000);
|
||||
const exp = Math.floor(expiresAt.getTime() / 1000);
|
||||
const token = this.signProxy(
|
||||
PROXY_UPLOAD_PATH,
|
||||
[workspaceId, key, contentType, contentLength],
|
||||
exp,
|
||||
proxy.signKey
|
||||
);
|
||||
return {
|
||||
url: this.linkProxyUrl(proxy.urlPrefix, PROXY_UPLOAD_PATH, {
|
||||
workspaceId,
|
||||
key,
|
||||
contentType,
|
||||
contentLength,
|
||||
exp,
|
||||
token,
|
||||
}),
|
||||
headers: {},
|
||||
expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
private createProxyMultipartUrl(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
uploadId: string,
|
||||
partNumber: number,
|
||||
proxy: R2ProxyConfig
|
||||
) {
|
||||
const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000);
|
||||
const exp = Math.floor(expiresAt.getTime() / 1000);
|
||||
const token = this.signProxy(
|
||||
PROXY_MULTIPART_PATH,
|
||||
[workspaceId, key, uploadId, partNumber],
|
||||
exp,
|
||||
proxy.signKey
|
||||
);
|
||||
return {
|
||||
url: this.linkProxyUrl(proxy.urlPrefix, PROXY_MULTIPART_PATH, {
|
||||
workspaceId,
|
||||
key,
|
||||
uploadId,
|
||||
partNumber,
|
||||
exp,
|
||||
token,
|
||||
}),
|
||||
headers: {},
|
||||
expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
private linkProxyUrl(
|
||||
urlPrefix: string,
|
||||
path: string,
|
||||
query: Record<string, string | number | undefined>
|
||||
) {
|
||||
const url = new URL(
|
||||
`${urlPrefix.replace(/\/+$/, '')}${path.startsWith('/') ? path : `/${path}`}`
|
||||
);
|
||||
for (const [key, value] of Object.entries(query)) {
|
||||
if (value !== undefined) {
|
||||
url.searchParams.set(key, value.toString());
|
||||
}
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,11 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
autoMetadata,
|
||||
Config,
|
||||
EventBus,
|
||||
metrics,
|
||||
OnEvent,
|
||||
type StorageProvider,
|
||||
StorageProviderFactory,
|
||||
URLHelper,
|
||||
} from '../../../base';
|
||||
import { EventBus, metrics, OnEvent, URLHelper } from '../../../base';
|
||||
import { Models } from '../../../models';
|
||||
import {
|
||||
type StorageRuntimeGetObjectResult,
|
||||
StorageRuntimeProvider,
|
||||
} from '../../storage-runtime';
|
||||
|
||||
declare global {
|
||||
interface Events {
|
||||
@@ -25,32 +20,14 @@ declare global {
|
||||
@Injectable()
|
||||
export class CommentAttachmentStorage {
|
||||
private readonly logger = new Logger(CommentAttachmentStorage.name);
|
||||
private provider!: StorageProvider;
|
||||
|
||||
get config() {
|
||||
return this.AFFiNEConfig.storages.blob;
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly AFFiNEConfig: Config,
|
||||
private readonly event: EventBus,
|
||||
private readonly storageFactory: StorageProviderFactory,
|
||||
private readonly models: Models,
|
||||
private readonly url: URLHelper
|
||||
private readonly url: URLHelper,
|
||||
private readonly rt: StorageRuntimeProvider
|
||||
) {}
|
||||
|
||||
@OnEvent('config.init')
|
||||
async onConfigInit() {
|
||||
this.provider = this.storageFactory.create(this.config.storage);
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
async onConfigChanged(event: Events['config.changed']) {
|
||||
if (event.updates.storages?.blob?.storage) {
|
||||
this.provider = this.storageFactory.create(this.config.storage);
|
||||
}
|
||||
}
|
||||
|
||||
private storageKey(workspaceId: string, docId: string, key: string) {
|
||||
return `comment-attachments/${workspaceId}/${docId}/${key}`;
|
||||
}
|
||||
@@ -63,15 +40,13 @@ export class CommentAttachmentStorage {
|
||||
blob: Buffer,
|
||||
userId: string
|
||||
) {
|
||||
const meta = autoMetadata(blob);
|
||||
|
||||
await this.provider.put(
|
||||
const metadata = await this.rt.putObject(
|
||||
'blob',
|
||||
this.storageKey(workspaceId, docId, key),
|
||||
blob,
|
||||
meta
|
||||
blob
|
||||
);
|
||||
const mime = meta.contentType ?? 'application/octet-stream';
|
||||
const size = blob.length;
|
||||
const mime = metadata.contentType;
|
||||
const size = metadata.contentLength;
|
||||
await this.models.commentAttachment.upsert({
|
||||
workspaceId,
|
||||
docId,
|
||||
@@ -94,15 +69,22 @@ export class CommentAttachmentStorage {
|
||||
docId: string,
|
||||
key: string,
|
||||
signedUrl?: boolean
|
||||
) {
|
||||
return await this.provider.get(
|
||||
this.storageKey(workspaceId, docId, key),
|
||||
signedUrl
|
||||
);
|
||||
): Promise<StorageRuntimeGetObjectResult> {
|
||||
const storageKey = this.storageKey(workspaceId, docId, key);
|
||||
if (signedUrl) {
|
||||
const presigned = await this.rt.presignGet('blob', storageKey);
|
||||
if (presigned) {
|
||||
return { redirectUrl: presigned.url };
|
||||
}
|
||||
}
|
||||
return await this.rt.getObject('blob', storageKey);
|
||||
}
|
||||
|
||||
async delete(workspaceId: string, docId: string, key: string) {
|
||||
await this.provider.delete(this.storageKey(workspaceId, docId, key));
|
||||
await this.rt.deleteObject(
|
||||
'blob',
|
||||
this.storageKey(workspaceId, docId, key)
|
||||
);
|
||||
await this.models.commentAttachment.delete(workspaceId, docId, key);
|
||||
this.logger.log(
|
||||
`deleted comment attachment ${workspaceId}/${docId}/${key}`
|
||||
|
||||
@@ -16,9 +16,10 @@ export class UserAvatarController {
|
||||
|
||||
@Get('/:id')
|
||||
async getAvatar(@Res() res: Response, @Param('id') id: string) {
|
||||
if (this.storage.config.storage.provider !== 'fs') {
|
||||
const provider = this.storage.config.storage.provider;
|
||||
if (!['assetpack', 'fs'].includes(provider)) {
|
||||
throw new ActionForbidden(
|
||||
'Only available when avatar storage provider set to fs.'
|
||||
'Only available when avatar storage provider is fs or assetpack.'
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -248,11 +248,12 @@ export class WorkspaceBlobResolver {
|
||||
}
|
||||
|
||||
const metadata = { contentType: mime, contentLength: size };
|
||||
const capabilities = await this.storage.capabilities();
|
||||
let init: BlobUploadInit | null = null;
|
||||
let uploadIdForRecord: string | null = null;
|
||||
|
||||
// try to resume multipart uploads
|
||||
if (record && record.uploadId) {
|
||||
if (capabilities.multipartDirect && record && record.uploadId) {
|
||||
const uploadedParts = await this.storage.listMultipartUploadParts(
|
||||
workspaceId,
|
||||
key,
|
||||
@@ -270,7 +271,7 @@ export class WorkspaceBlobResolver {
|
||||
}
|
||||
}
|
||||
|
||||
if (size >= MULTIPART_THRESHOLD) {
|
||||
if (capabilities.multipartDirect && size >= MULTIPART_THRESHOLD) {
|
||||
const multipart = await this.storage.createMultipartUpload(
|
||||
workspaceId,
|
||||
key,
|
||||
@@ -289,7 +290,7 @@ export class WorkspaceBlobResolver {
|
||||
}
|
||||
}
|
||||
|
||||
if (!init) {
|
||||
if (!init && capabilities.presignPut) {
|
||||
const presigned = await this.storage.presignPut(
|
||||
workspaceId,
|
||||
key,
|
||||
@@ -397,6 +398,9 @@ export class WorkspaceBlobResolver {
|
||||
if (result.reason === 'mime_mismatch') {
|
||||
throw new BlobInvalid('Blob mime mismatch');
|
||||
}
|
||||
if (result.reason === 'size_too_large') {
|
||||
throw new BlobInvalid('Blob size too large');
|
||||
}
|
||||
throw new BlobInvalid('Blob key mismatch');
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user