Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-20 03:26:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): impl storage runtime

Browse Source
This commit is contained in:
DarkSky
2026-07-01 03:04:14 +08:00
parent a821f67fc9
commit 0e20823311
90 changed files with 5439 additions and 4292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts
+222 -118
View File
@@ -1,4 +1,4 @@
import { createHash } from 'node:crypto';
import { createHash, createHmac } from 'node:crypto';
import { mock } from 'node:test';
import {
@@ -6,22 +6,13 @@ import {
ConfigFactory,
PROXY_MULTIPART_PATH,
PROXY_UPLOAD_PATH,
StorageProviderConfig,
StorageProviderFactory,
toBuffer,
type R2StorageConfig,
SIGNED_URL_EXPIRED,
type StorageProviderConfig,
} from '../../../base';
import {
R2StorageConfig,
R2StorageProvider,
} from '../../../base/storage/providers/r2';
import { SIGNED_URL_EXPIRED } from '../../../base/storage/providers/utils';
import { EntitlementService } from '../../../core/entitlement';
import {
CommentAttachmentStorage,
WorkspaceBlobStorage,
} from '../../../core/storage';
import { MULTIPART_THRESHOLD } from '../../../core/storage/constants';
import { R2UploadController } from '../../../core/storage/r2-proxy';
import { StorageRuntimeProvider } from '../../../core/storage-runtime';
import {
SubscriptionPlan,
SubscriptionRecurring,
@@ -29,7 +20,7 @@ import {
} from '../../../plugins/payment/types';
import { app, e2e, Mockers } from '../test';
class MockR2Provider extends R2StorageProvider {
class MockStorageRuntime {
createMultipartCalls = 0;
putCalls: {
key: string;
@@ -46,45 +37,72 @@ class MockR2Provider extends R2StorageProvider {
contentLength?: number;
}[] = [];
constructor(config: R2StorageConfig, bucket: string) {
super(config, bucket);
async providerCapabilities() {
const storage = app.get(Config).storages.blob.storage;
const usePresignedURL = (storage.config as R2StorageConfig).usePresignedURL;
if (storage.provider !== 'cloudflare-r2') {
return {
put: true,
get: true,
head: true,
list: true,
delete: true,
presignPut: false,
presignGet: false,
multipartDirect: false,
proxyUpload: false,
assetpack: false,
serverMediatedOnly: true,
};
}
return {
put: true,
get: true,
head: true,
list: true,
delete: true,
presignPut: true,
presignGet: false,
multipartDirect: true,
proxyUpload: !!usePresignedURL?.enabled,
assetpack: false,
serverMediatedOnly: false,
};
}
destroy() {}
override async proxyPutObject(
async presignPut(
_scope: string,
key: string,
body: any,
options: { contentType?: string; contentLength?: number } = {}
metadata: { contentType?: string; contentLength?: number } = {}
) {
this.putCalls.push({
key,
body: await toBuffer(body),
contentType: options.contentType,
contentLength: options.contentLength,
});
const storage = app.get(Config).storages.blob.storage;
const r2 = storage.config as R2StorageConfig;
if (!r2.usePresignedURL?.enabled) {
return {
url: 'https://test-bucket.r2.example.com/object?X-Amz-Algorithm=AWS4-HMAC-SHA256',
headers: {},
expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000),
};
}
const [workspaceId, blobKey] = key.split('/');
return createProxyUrl(
PROXY_UPLOAD_PATH,
[
workspaceId,
blobKey,
metadata.contentType ?? 'application/octet-stream',
metadata.contentLength,
],
{
workspaceId,
key: blobKey,
contentType: metadata.contentType ?? 'application/octet-stream',
contentLength: metadata.contentLength,
}
);
}
override async proxyUploadPart(
key: string,
uploadId: string,
partNumber: number,
body: any,
options: { contentLength?: number } = {}
) {
const etag = `etag-${partNumber}`;
this.partCalls.push({
key,
uploadId,
partNumber,
etag,
body: await toBuffer(body),
contentLength: options.contentLength,
});
return etag;
}
override async createMultipartUpload() {
async createMultipartUpload() {
this.createMultipartCalls += 1;
return {
uploadId: 'upload-id',
@@ -92,7 +110,30 @@ class MockR2Provider extends R2StorageProvider {
};
}
override async listMultipartUploadParts(key: string, uploadId: string) {
async presignUploadPart(
_scope: string,
key: string,
uploadId: string,
partNumber: number
) {
const [workspaceId, blobKey] = key.split('/');
return createProxyUrl(
PROXY_MULTIPART_PATH,
[workspaceId, blobKey, uploadId, partNumber],
{
workspaceId,
key: blobKey,
uploadId,
partNumber,
}
);
}
async listMultipartUploadParts(
_scope: string,
key: string,
uploadId: string
) {
const latest = new Map<number, string>();
for (const part of this.partCalls) {
if (part.key !== key || part.uploadId !== uploadId) {
@@ -104,6 +145,45 @@ class MockR2Provider extends R2StorageProvider {
.sort((left, right) => left[0] - right[0])
.map(([partNumber, etag]) => ({ partNumber, etag }));
}
async putObject(
_scope: string,
key: string,
body: Buffer,
options: { contentType?: string; contentLength?: number } = {}
) {
this.putCalls.push({
key,
body,
contentType: options.contentType,
contentLength: options.contentLength,
});
return {
contentType: options.contentType ?? 'application/octet-stream',
contentLength: options.contentLength ?? body.length,
lastModified: new Date(),
};
}
async proxyUploadPart(
_scope: string,
key: string,
uploadId: string,
partNumber: number,
body: Buffer,
contentLength?: number
) {
const etag = `etag-${partNumber}`;
this.partCalls.push({
key,
uploadId,
partNumber,
etag,
body,
contentLength,
});
return etag;
}
}
const baseR2Storage: StorageProviderConfig = {
@@ -125,55 +205,40 @@ const baseR2Storage: StorageProviderConfig = {
};
let defaultBlobStorage: StorageProviderConfig;
let provider: MockR2Provider | null = null;
let factoryCreateUnmocked: StorageProviderFactory['create'];
let runtime: MockStorageRuntime;
e2e.before(() => {
defaultBlobStorage = structuredClone(app.get(Config).storages.blob.storage);
const factory = app.get(StorageProviderFactory);
factoryCreateUnmocked = factory.create.bind(factory);
});
e2e.beforeEach(async () => {
provider?.destroy();
provider = null;
const factory = app.get(StorageProviderFactory);
mock.method(factory, 'create', (config: StorageProviderConfig) => {
if (config.provider === 'cloudflare-r2') {
if (!provider) {
provider = new MockR2Provider(
config.config as R2StorageConfig,
config.bucket
);
}
return provider;
}
return factoryCreateUnmocked(config);
});
runtime = new MockStorageRuntime();
const rt = app.get(StorageRuntimeProvider);
for (const method of [
'providerCapabilities',
'presignPut',
'createMultipartUpload',
'presignUploadPart',
'listMultipartUploadParts',
'putObject',
'proxyUploadPart',
] as const) {
mock.method(rt, method, (...args: any[]) =>
(runtime[method] as any)(...args)
);
}
await useR2Storage();
});
e2e.afterEach.always(async () => {
await setBlobStorage(defaultBlobStorage);
provider?.destroy();
provider = null;
mock.reset();
});
async function setBlobStorage(storage: StorageProviderConfig) {
provider?.destroy();
provider = null;
const configFactory = app.get(ConfigFactory);
configFactory.override({ storages: { blob: { storage } } });
const blobStorage = app.get(WorkspaceBlobStorage);
await blobStorage.onConfigInit();
const commentAttachmentStorage = app.get(CommentAttachmentStorage);
await commentAttachmentStorage.onConfigInit();
const controller = app.get(R2UploadController);
// reset cached provider in controller
(controller as any).provider = null;
}
async function useR2Storage(
@@ -193,11 +258,8 @@ async function useR2Storage(
return storage;
}
function getProvider(): MockR2Provider {
if (!provider) {
throw new Error('R2 provider is not initialized');
}
return provider;
function getRuntime(): MockStorageRuntime {
return runtime;
}
async function createBlobUpload(
@@ -285,7 +347,7 @@ async function gql<QueryData = any>(
return res.body.data;
}
e2e('should proxy single upload with valid signature', async t => {
e2e.serial('should proxy single upload with valid signature', async t => {
const { workspace } = await setupWorkspace();
const buffer = Buffer.from('r2-proxy');
const key = sha256Base64urlWithPadding(buffer);
@@ -309,7 +371,7 @@ e2e('should proxy single upload with valid signature', async t => {
.send(buffer);
t.is(res.status, 200);
const calls = getProvider().putCalls;
const calls = getRuntime().putCalls;
t.is(calls.length, 1);
t.is(calls[0].key, `${workspace.id}/${key}`);
t.is(calls[0].contentType, 'text/plain');
@@ -317,7 +379,7 @@ e2e('should proxy single upload with valid signature', async t => {
t.deepEqual(calls[0].body, buffer);
});
e2e('should proxy multipart upload and return etag', async t => {
e2e.serial('should proxy multipart upload and return etag', async t => {
const { workspace } = await setupWorkspace();
const key = 'multipart-object';
const totalSize = MULTIPART_THRESHOLD + 1024;
@@ -340,7 +402,7 @@ e2e('should proxy multipart upload and return etag', async t => {
t.is(res.status, 200);
t.is(res.get('etag'), 'etag-1');
const calls = getProvider().partCalls;
const calls = getRuntime().partCalls;
t.is(calls.length, 1);
t.is(calls[0].key, `${workspace.id}/${key}`);
t.is(calls[0].uploadId, 'upload-id');
@@ -349,34 +411,42 @@ e2e('should proxy multipart upload and return etag', async t => {
t.deepEqual(calls[0].body, payload);
});
e2e('should resume multipart upload and return uploaded parts', async t => {
const { workspace } = await setupWorkspace();
const key = 'multipart-resume';
const totalSize = MULTIPART_THRESHOLD + 1024;
e2e.serial(
'should resume multipart upload and return uploaded parts',
async t => {
const { workspace } = await setupWorkspace();
const key = 'multipart-resume';
const totalSize = MULTIPART_THRESHOLD + 1024;
const init1 = await createBlobUpload(workspace.id, key, totalSize, 'bin');
t.is(init1.method, 'MULTIPART');
t.is(init1.uploadId, 'upload-id');
t.deepEqual(init1.uploadedParts, []);
t.is(getProvider().createMultipartCalls, 1);
const init1 = await createBlobUpload(workspace.id, key, totalSize, 'bin');
t.is(init1.method, 'MULTIPART');
t.is(init1.uploadId, 'upload-id');
t.deepEqual(init1.uploadedParts, []);
t.is(getRuntime().createMultipartCalls, 1);
const part = await getBlobUploadPartUrl(workspace.id, key, init1.uploadId, 1);
const payload = Buffer.from('part-body');
const partUrl = new URL(part.uploadUrl, app.url);
await app
.PUT(partUrl.pathname + partUrl.search)
.set('content-length', payload.length.toString())
.send(payload)
.expect(200);
const part = await getBlobUploadPartUrl(
workspace.id,
key,
init1.uploadId,
1
);
const payload = Buffer.from('part-body');
const partUrl = new URL(part.uploadUrl, app.url);
await app
.PUT(partUrl.pathname + partUrl.search)
.set('content-length', payload.length.toString())
.send(payload)
.expect(200);
const init2 = await createBlobUpload(workspace.id, key, totalSize, 'bin');
t.is(init2.method, 'MULTIPART');
t.is(init2.uploadId, 'upload-id');
t.deepEqual(init2.uploadedParts, [{ partNumber: 1, etag: 'etag-1' }]);
t.is(getProvider().createMultipartCalls, 1);
});
const init2 = await createBlobUpload(workspace.id, key, totalSize, 'bin');
t.is(init2.method, 'MULTIPART');
t.is(init2.uploadId, 'upload-id');
t.deepEqual(init2.uploadedParts, [{ partNumber: 1, etag: 'etag-1' }]);
t.is(getRuntime().createMultipartCalls, 1);
}
);
e2e('should reject upload when token is invalid', async t => {
e2e.serial('should reject upload when token is invalid', async t => {
const { workspace } = await setupWorkspace();
const buffer = Buffer.from('payload');
const init = await createBlobUpload(
@@ -396,10 +466,10 @@ e2e('should reject upload when token is invalid', async t => {
t.is(res.status, 400);
t.is(res.body.message, 'Invalid upload token');
t.is(getProvider().putCalls.length, 0);
t.is(getRuntime().putCalls.length, 0);
});
e2e('should reject upload when url is expired', async t => {
e2e.serial('should reject upload when url is expired', async t => {
const { workspace } = await setupWorkspace();
const buffer = Buffer.from('expired');
const init = await createBlobUpload(
@@ -422,10 +492,10 @@ e2e('should reject upload when url is expired', async t => {
t.is(res.status, 400);
t.is(res.body.message, 'Upload URL expired');
t.is(getProvider().putCalls.length, 0);
t.is(getRuntime().putCalls.length, 0);
});
e2e(
e2e.serial(
'should fall back to direct presign when custom domain is disabled',
async t => {
await useR2Storage({
@@ -449,7 +519,7 @@ e2e(
}
);
e2e(
e2e.serial(
'should still fallback to graphql when provider does not support presign',
async t => {
await setBlobStorage({
@@ -473,6 +543,40 @@ e2e(
}
);
function createProxyUrl(
path: string,
canonicalFields: (string | number | undefined)[],
query: Record<string, string | number | undefined>
) {
const signKey = (
app.get(Config).storages.blob.storage.config as R2StorageConfig
).usePresignedURL?.signKey;
if (!signKey) {
throw new Error('missing R2 proxy sign key');
}
const exp = Math.floor(Date.now() / 1000) + SIGNED_URL_EXPIRED;
const canonical = [
path,
...canonicalFields.map(field =>
field === undefined ? '' : field.toString()
),
exp.toString(),
].join('\n');
const token = createHmac('sha256', signKey)
.update(canonical)
.digest('base64');
const url = new URL(`http://localhost${path}`);
for (const [key, value] of Object.entries(query)) {
if (value !== undefined) {
url.searchParams.set(key, value.toString());
}
}
url.searchParams.set('exp', exp.toString());
url.searchParams.set('token', `${exp}-${token}`);
return { url: url.pathname + url.search, expiresAt: new Date(exp * 1000) };
}
function sha256Base64urlWithPadding(buffer: Buffer) {
return createHash('sha256')
.update(buffer)
packages/backend/server/src/__tests__/e2e/workspace/controller.spec.ts
+61 -4
View File
@@ -1,4 +1,5 @@
import { randomUUID } from 'node:crypto';
import { Readable } from 'node:stream';
import { mock } from 'node:test';
import {
@@ -7,6 +8,8 @@ import {
type StorageProviderConfig,
} from '../../../base';
import { CommentAttachmentStorage } from '../../../core/storage';
import { StorageRuntimeProvider } from '../../../core/storage-runtime';
import { getMime } from '../../../native';
import { Mockers } from '../../mocks';
import { app, e2e } from '../test';
@@ -26,14 +29,68 @@ e2e.afterEach.always(() => {
mock.reset();
});
const objects = new Map<
string,
{
body: Buffer;
metadata?: {
contentLength?: number;
contentType?: string;
checksumCRC32?: string;
lastModified?: Date;
};
}
>();
e2e.beforeEach(() => {
objects.clear();
const rt = app.get(StorageRuntimeProvider);
mock.method(
rt,
'putObject',
async (
_scope: string,
key: string,
body: Buffer,
metadata?: {
contentLength?: number;
contentType?: string;
checksumCRC32?: string;
}
) => {
const object = {
body,
metadata: {
...metadata,
contentType: metadata?.contentType ?? getMime(body),
contentLength: metadata?.contentLength ?? body.length,
lastModified: new Date(),
},
};
objects.set(key, object);
return object.metadata;
}
);
mock.method(rt, 'getObject', async (_scope: string, key: string) => {
const object = objects.get(key);
if (!object) {
return {};
}
return {
body: Readable.from(object.body),
metadata: object.metadata,
};
});
mock.method(rt, 'presignGet', async () => undefined);
});
async function useCommentAttachmentBlobStorage(storage: StorageProviderConfig) {
app.get(ConfigFactory).override({ storages: { blob: { storage } } });
await app.get(CommentAttachmentStorage).onConfigInit();
}
// #region comment attachment
e2e(
e2e.serial(
'should get comment attachment not found when key is not exists',
async t => {
const { owner, workspace } = await createWorkspace();
@@ -50,7 +107,7 @@ e2e(
}
);
e2e(
e2e.serial(
'should get comment attachment no permission when user is not member',
async t => {
const { workspace } = await createWorkspace();
@@ -117,7 +174,7 @@ e2e.serial('should get comment attachment body', async t => {
}
});
e2e('should get comment attachment redirect url', async t => {
e2e.serial('should get comment attachment redirect url', async t => {
const { owner, workspace } = await createWorkspace();
await app.login(owner);
packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts
+2 -2
View File
@@ -4,9 +4,9 @@ import ava, { TestFn } from 'ava';
import Sinon from 'sinon';
import { OneDay } from '../../base';
import { BackendRuntimeProvider } from '../../core/backend-runtime';
import { StorageModule, WorkspaceBlobStorage } from '../../core/storage';
import { BlobUploadCleanupJob } from '../../core/storage/job';
import { StorageRuntimeProvider } from '../../core/storage-runtime';
import { MockUser, MockWorkspace } from '../mocks';
import { createTestingModule, TestingModule } from '../utils';
@@ -28,7 +28,7 @@ test.before(async t => {
imports: [ScheduleModule.forRoot(), StorageModule],
tapModule: builder => {
builder
.overrideProvider(BackendRuntimeProvider)
.overrideProvider(StorageRuntimeProvider)
.useValue(t.context.runtime);
},
});
packages/backend/server/src/__tests__/workspace/blobs.e2e.ts
+120 -21
View File
@@ -1,12 +1,15 @@
import { createHash } from 'node:crypto';
import { Readable } from 'node:stream';
import test from 'ava';
import Sinon from 'sinon';
import { Config, ConfigFactory, StorageProviderFactory } from '../../base';
import { ConfigFactory } from '../../base';
import { QuotaStateService } from '../../core/quota/state';
import { WorkspaceBlobStorage } from '../../core/storage/wrappers/blob';
import { StorageRuntimeProvider } from '../../core/storage-runtime';
import { BlobModel, WorkspaceFeatureModel } from '../../models';
import { getMime } from '../../native';
import {
collectAllBlobSizes,
completeBlobUpload,
@@ -32,9 +35,113 @@ const RESTRICTED_QUOTA = {
let app: TestingApp;
let model: WorkspaceFeatureModel;
const objects = new Map<
string,
{
body: Buffer;
metadata: {
contentType: string;
contentLength: number;
lastModified: Date;
};
}
>();
const storageRuntime = {
providerCapabilities: async () => ({
put: true,
get: true,
head: true,
list: true,
delete: true,
presignPut: false,
presignGet: false,
multipartDirect: false,
proxyUpload: false,
assetpack: false,
serverMediatedOnly: true,
}),
putObject: async (
_scope: string,
key: string,
body: Buffer,
metadata?: { contentType?: string; contentLength?: number }
) => {
const object = {
body,
metadata: {
contentType: metadata?.contentType ?? getMime(body),
contentLength: metadata?.contentLength ?? body.length,
lastModified: new Date(),
},
};
objects.set(key, object);
return object.metadata;
},
headObject: async (_scope: string, key: string) => {
return objects.get(key)?.metadata;
},
getObject: async (_scope: string, key: string) => {
const object = objects.get(key);
return object
? { body: Readable.from(object.body), metadata: object.metadata }
: {};
},
listObjects: async (_scope: string, prefix?: string) => {
return Array.from(objects.entries())
.filter(([key]) => !prefix || key.startsWith(prefix))
.map(([key, object]) => ({ key, ...object.metadata }));
},
deleteObject: async (_scope: string, key: string) => {
objects.delete(key);
},
presignPut: async () => undefined,
presignGet: async () => undefined,
createMultipartUpload: async () => undefined,
presignUploadPart: async () => undefined,
listMultipartUploadParts: async () => undefined,
completeMultipartUpload: async () => undefined,
completeWorkspaceBlobUpload: async (
workspaceId: string,
key: string,
expected: { size: number; mime: string }
) => {
const objectKey = `${workspaceId}/${key}`;
const object = objects.get(objectKey);
if (!object) {
return { ok: false, reason: 'not_found' };
}
if (object.metadata.contentLength !== expected.size) {
return { ok: false, reason: 'size_mismatch' };
}
if (object.metadata.contentType !== expected.mime) {
return { ok: false, reason: 'mime_mismatch' };
}
if (sha256Base64urlWithPadding(object.body) !== key) {
return { ok: false, reason: 'checksum_mismatch' };
}
await app.get(BlobModel).upsert({
workspaceId,
key,
mime: object.metadata.contentType,
size: object.metadata.contentLength,
status: 'completed',
uploadId: null,
});
return {
ok: true,
contentType: object.metadata.contentType,
contentLength: object.metadata.contentLength,
lastModifiedMs: object.metadata.lastModified.getTime(),
};
},
};
test.before(async () => {
app = await createTestingApp();
app = await createTestingApp({
tapModule: builder => {
builder.overrideProvider(StorageRuntimeProvider).useValue(storageRuntime);
},
});
model = app.get(WorkspaceFeatureModel);
app.get(ConfigFactory).override({
storages: {
@@ -47,11 +154,11 @@ test.before(async () => {
},
},
});
await app.get(WorkspaceBlobStorage).onConfigInit();
});
test.beforeEach(async () => {
await app.initTestingDB();
objects.clear();
});
test.after.always(async () => {
@@ -124,22 +231,19 @@ test('should keep partial blob metadata listing on DB path without storage scan'
const workspace = await createWorkspace(app);
const storage = app.get(WorkspaceBlobStorage);
const rawProvider = (storage as any).provider;
const listSpy = Sinon.spy(rawProvider, 'list');
const rt = app.get(StorageRuntimeProvider);
const listSpy = Sinon.spy(rt, 'listObjects');
t.teardown(() => listSpy.restore());
const buffer1 = Buffer.from('with metadata');
const buffer2 = Buffer.from('without metadata');
const key1 = sha256Base64urlWithPadding(buffer1);
const key2 = sha256Base64urlWithPadding(buffer2);
const config = app.get(Config);
const factory = app.get(StorageProviderFactory);
const provider = factory.create(config.storages.blob.storage);
await provider.put(`${workspace.id}/${key1}`, buffer1, {
await rt.putObject('blob', `${workspace.id}/${key1}`, buffer1, {
contentType: 'text/plain',
contentLength: buffer1.length,
});
await provider.put(`${workspace.id}/${key2}`, buffer2, {
await rt.putObject('blob', `${workspace.id}/${key2}`, buffer2, {
contentType: 'text/plain',
contentLength: buffer2.length,
});
@@ -194,11 +298,9 @@ test('should complete pending blob upload', async t => {
await createBlobUpload(app, workspace.id, key, buffer.length, mime);
const config = app.get(Config);
const factory = app.get(StorageProviderFactory);
const provider = factory.create(config.storages.blob.storage);
const rt = app.get(StorageRuntimeProvider);
await provider.put(`${workspace.id}/${key}`, buffer, {
await rt.putObject('blob', `${workspace.id}/${key}`, buffer, {
contentType: mime,
contentLength: buffer.length,
});
@@ -225,11 +327,9 @@ test('should reject complete when blob key mismatched', async t => {
const wrongKey = sha256Base64urlWithPadding(Buffer.from('other'));
await createBlobUpload(app, workspace.id, wrongKey, buffer.length, mime);
const config = app.get(Config);
const factory = app.get(StorageProviderFactory);
const provider = factory.create(config.storages.blob.storage);
const rt = app.get(StorageRuntimeProvider);
await provider.put(`${workspace.id}/${wrongKey}`, buffer, {
await rt.putObject('blob', `${workspace.id}/${wrongKey}`, buffer, {
contentType: mime,
contentLength: buffer.length,
});
@@ -265,9 +365,8 @@ test('should auto delete blobs when workspace is deleted', async t => {
const blobs = await listBlobs(app, workspace.id);
t.is(blobs.length, 2);
const storage = app.get(WorkspaceBlobStorage);
const rawProvider = (storage as any).provider;
const listSpy = Sinon.spy(rawProvider, 'list');
const rt = app.get(StorageRuntimeProvider);
const listSpy = Sinon.spy(rt, 'listObjects');
t.teardown(() => listSpy.restore());
await deleteWorkspace(app, workspace.id);
packages/backend/server/src/__tests__/workspace/controller.spec.ts
+1 -2
View File
@@ -212,8 +212,7 @@ test('should be able to get permission granted workspace', async t => {
test('should return 404 if blob not found', async t => {
const { app, storage } = t.context;
// @ts-expect-error mock
storage.get.resolves({ body: null });
storage.get.resolves({ body: undefined });
const res = await app.GET('/api/workspaces/public/blobs/test');
t.is(res.status, HttpStatus.NOT_FOUND);
packages/backend/server/src/app.module.ts
+2 -2
View File
@@ -25,7 +25,6 @@ import { MetricsModule } from './base/metrics';
import { MutexModule } from './base/mutex';
import { PrismaModule } from './base/prisma';
import { RedisModule } from './base/redis';
import { StorageProviderModule } from './base/storage';
import { RateLimiterModule } from './base/throttler';
import { WebSocketModule } from './base/websocket';
import { AccessTokenModule } from './core/access-token';
@@ -47,6 +46,7 @@ import { RealtimeModule } from './core/realtime';
import { SelfhostModule } from './core/selfhost';
import { StaticFileModule } from './core/static-files';
import { StorageModule } from './core/storage';
import { StorageRuntimeModule } from './core/storage-runtime';
import { SyncModule } from './core/sync';
import { TelemetryModule } from './core/telemetry';
import { UserModule } from './core/user';
@@ -114,7 +114,6 @@ export const FunctionalityModules = [
MutexModule,
MetricsModule,
RateLimiterModule,
StorageProviderModule,
HelpersModule,
ErrorModule,
WebSocketModule,
@@ -122,6 +121,7 @@ export const FunctionalityModules = [
RealtimeModule,
ModelsModule,
BackendRuntimeModule,
StorageRuntimeModule,
ScheduleModule.forRoot(),
MonitorModule,
];
packages/backend/server/src/base/error/errors.gen.ts
+12 -1
View File
@@ -1059,6 +1059,16 @@ export class UnsupportedClientVersion extends UserFriendlyError {
super('action_forbidden', 'unsupported_client_version', message, args);
}
}
@ObjectType()
class UnsupportedServerVersionDataType {
@Field() requiredVersion!: string
}
export class UnsupportedServerVersion extends UserFriendlyError {
constructor(args: UnsupportedServerVersionDataType, message?: string | ((args: UnsupportedServerVersionDataType) => string)) {
super('action_forbidden', 'unsupported_server_version', message, args);
}
}
export class NotificationNotFound extends UserFriendlyError {
constructor(message?: string) {
@@ -1288,6 +1298,7 @@ export enum ErrorNames {
INVALID_LICENSE_UPDATE_PARAMS,
LICENSE_EXPIRED,
UNSUPPORTED_CLIENT_VERSION,
UNSUPPORTED_SERVER_VERSION,
NOTIFICATION_NOT_FOUND,
MENTION_USER_DOC_ACCESS_DENIED,
MENTION_USER_ONESELF_DENIED,
@@ -1308,5 +1319,5 @@ registerEnumType(ErrorNames, {
export const ErrorDataUnionType = createUnionType({
name: 'ErrorDataUnion',
types: () =>
[GraphqlBadRequestDataType, HttpRequestErrorDataType, SsrfBlockedErrorDataType, ResponseTooLargeErrorDataType, ImageFormatNotSupportedDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidOauthResponseDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CalendarProviderRequestErrorDataType, NoCopilotProviderAvailableDataType, CopilotFailedToGenerateEmbeddingDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderNotSupportedDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, MentionUserDocAccessDeniedDataType, InvalidAppConfigDataType, InvalidAppConfigInputDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const,
[GraphqlBadRequestDataType, HttpRequestErrorDataType, SsrfBlockedErrorDataType, ResponseTooLargeErrorDataType, ImageFormatNotSupportedDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidOauthResponseDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CalendarProviderRequestErrorDataType, NoCopilotProviderAvailableDataType, CopilotFailedToGenerateEmbeddingDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderNotSupportedDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, UnsupportedServerVersionDataType, MentionUserDocAccessDeniedDataType, InvalidAppConfigDataType, InvalidAppConfigInputDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const,
});
packages/backend/server/src/base/index.ts
+1 -6
View File
@@ -30,11 +30,6 @@ export { Lock, Locker, Mutex, RequestMutex } from './mutex';
export * from './nestjs';
export { type PrismaTransaction } from './prisma';
export * from './storage';
export {
autoMetadata,
type StorageProvider,
type StorageProviderConfig,
StorageProviderFactory,
} from './storage';
export { type StorageProviderConfig } from './storage';
export { CloudThrottlerGuard, SkipThrottle, Throttle } from './throttler';
export * from './utils';
packages/backend/server/src/base/storage/__tests__/fs.spec.ts
-130
View File
@@ -1,130 +0,0 @@
import { promises as fs } from 'node:fs';
import { join } from 'node:path';
import test from 'ava';
import { getStreamAsBuffer } from 'get-stream';
import { ListObjectsMetadata } from '../providers';
import { FsStorageProvider } from '../providers/fs';
const config = {
path: join(process.cwd(), 'node_modules', '.cache/affine-test-storage'),
};
function createProvider() {
return new FsStorageProvider(
config,
'test' + Math.random().toString(16).substring(2, 8)
);
}
function keys(list: ListObjectsMetadata[]) {
return list.map(i => i.key);
}
async function randomPut(
provider: FsStorageProvider,
prefix = ''
): Promise<string> {
const key = prefix + 'test-key-' + Math.random().toString(16).substring(2, 8);
const body = Buffer.from(key);
await provider.put(key, body);
return key;
}
test.after.always(() => {
fs.rm(config.path, { recursive: true }).catch(console.error);
});
test('put & get', async t => {
const provider = createProvider();
const key = 'testKey';
const body = Buffer.from('testBody');
await provider.put(key, body);
const result = await provider.get(key);
t.deepEqual(await getStreamAsBuffer(result.body!), body);
t.is(result.metadata?.contentLength, body.length);
});
test('list - one level', async t => {
const provider = createProvider();
const list = await Promise.all(
Array.from({ length: 100 }).map(() => randomPut(provider))
);
list.sort();
// random order, use set
const result = await provider.list();
t.deepEqual(keys(result), list);
const result2 = await provider.list('test-key');
t.deepEqual(keys(result2), list);
const result3 = await provider.list('testKey');
t.is(result3.length, 0);
});
test('list recursively', async t => {
const provider = createProvider();
await Promise.all([
Promise.all(Array.from({ length: 10 }).map(() => randomPut(provider))),
Promise.all(
Array.from({ length: 10 }).map(() => randomPut(provider, 'a/'))
),
Promise.all(
Array.from({ length: 10 }).map(() => randomPut(provider, 'a/b/'))
),
Promise.all(
Array.from({ length: 10 }).map(() => randomPut(provider, 'a/b/t/'))
),
]);
const r1 = await provider.list();
t.is(r1.length, 40);
// contains all `a/xxx` and `a/b/xxx` and `a/b/c/xxx`
const r2 = await provider.list('a');
t.is(r2.length, 30);
// contains only `a/b/xxx`
const r3 = await provider.list('a/b');
const r4 = await provider.list('a/b/');
t.is(r3.length, 20);
t.deepEqual(r3, r4);
// prefix is not ended with '/', it's open to all files and sub dirs
// contains all `a/b/t/xxx` and `a/b/t{xxxx}`
const r5 = await provider.list('a/b/t');
t.is(r5.length, 20);
});
test('delete', async t => {
const provider = createProvider();
const key = 'testKey';
const body = Buffer.from('testBody');
await provider.put(key, body);
await provider.delete(key);
await t.throwsAsync(() => fs.access(join(config.path, provider.bucket, key)));
});
test('rejects unsafe object keys', async t => {
const provider = createProvider();
await t.throwsAsync(() => provider.put('../escape', Buffer.from('nope')));
await t.throwsAsync(() => provider.get('nested/../escape'));
await t.throwsAsync(() => provider.head('./escape'));
t.throws(() => provider.delete('nested//escape'));
});
test('rejects unsafe list prefixes', async t => {
const provider = createProvider();
await t.throwsAsync(() => provider.list('../escape'));
await t.throwsAsync(() => provider.list('nested/../../escape'));
await t.throwsAsync(() => provider.list('/absolute'));
});
packages/backend/server/src/base/storage/__tests__/r2.spec.ts
-82
View File
@@ -1,82 +0,0 @@
import test from 'ava';
import { R2StorageProvider } from '../providers/r2';
function endpointOf(provider: R2StorageProvider) {
return provider.endpointUrl;
}
test('R2 provider should use account endpoint by default', t => {
const provider = new R2StorageProvider(
{
accountId: 'test-account',
region: 'auto',
credentials: {
accessKeyId: 'test',
secretAccessKey: 'test',
},
},
'test-bucket'
);
t.is(
endpointOf(provider),
'https://test-account.r2.cloudflarestorage.com/test-bucket'
);
});
test('R2 provider should append jurisdiction suffix for EU buckets', t => {
const provider = new R2StorageProvider(
{
accountId: 'test-account',
jurisdiction: 'eu',
region: 'auto',
credentials: {
accessKeyId: 'test',
secretAccessKey: 'test',
},
},
'test-bucket'
);
t.is(
endpointOf(provider),
'https://test-account.eu.r2.cloudflarestorage.com/test-bucket'
);
});
test('R2 provider should throw when accountId is missing', t => {
t.throws(
() =>
new R2StorageProvider(
{
region: 'auto',
credentials: {
accessKeyId: 'test',
secretAccessKey: 'test',
},
} as any,
'test-bucket'
)
);
});
test('R2 provider should use default endpoint when jurisdiction is explicitly undefined', t => {
const provider = new R2StorageProvider(
{
accountId: 'test-account',
jurisdiction: undefined,
region: 'auto',
credentials: {
accessKeyId: 'test',
secretAccessKey: 'test',
},
},
'test-bucket'
);
t.is(
endpointOf(provider),
'https://test-account.r2.cloudflarestorage.com/test-bucket'
);
});
packages/backend/server/src/base/storage/__tests__/s3-list-parts.spec.ts
-49
View File
@@ -1,49 +0,0 @@
import { parseListPartsXml } from '@affine/s3-compat';
import test from 'ava';
test('parseListPartsXml handles array parts and pagination', t => {
const xml = `<?xml version="1.0" encoding="UTF-8"?>
<ListPartsResult>
<Bucket>test</Bucket>
<Key>key</Key>
<UploadId>upload-id</UploadId>
<PartNumberMarker>0</PartNumberMarker>
<NextPartNumberMarker>3</NextPartNumberMarker>
<IsTruncated>true</IsTruncated>
<Part>
<PartNumber>1</PartNumber>
<ETag>"etag-1"</ETag>
</Part>
<Part>
<PartNumber>2</PartNumber>
<ETag>etag-2</ETag>
</Part>
</ListPartsResult>`;
const result = parseListPartsXml(xml);
t.deepEqual(result.parts, [
{ partNumber: 1, etag: 'etag-1' },
{ partNumber: 2, etag: 'etag-2' },
]);
t.true(result.isTruncated);
t.is(result.nextPartNumberMarker, '3');
});
test('parseListPartsXml handles single part', t => {
const xml = `<?xml version="1.0" encoding="UTF-8"?>
<ListPartsResult>
<Bucket>test</Bucket>
<Key>key</Key>
<UploadId>upload-id</UploadId>
<IsTruncated>false</IsTruncated>
<Part>
<PartNumber>5</PartNumber>
<ETag>"etag-5"</ETag>
</Part>
</ListPartsResult>`;
const result = parseListPartsXml(xml);
t.deepEqual(result.parts, [{ partNumber: 5, etag: 'etag-5' }]);
t.false(result.isTruncated);
t.is(result.nextPartNumberMarker, undefined);
});
packages/backend/server/src/base/storage/__tests__/s3.spec.ts
-90
View File
@@ -1,90 +0,0 @@
import test from 'ava';
import { S3StorageProvider } from '../providers/s3';
import { SIGNED_URL_EXPIRED } from '../providers/utils';
const config = {
region: 'us-east-1',
endpoint: 'https://s3.us-east-1.amazonaws.com',
credentials: {
accessKeyId: 'test',
secretAccessKey: 'test',
},
};
function createProvider() {
return new S3StorageProvider(config, 'test-bucket');
}
test('presignPut should return url and headers', async t => {
const provider = createProvider();
const result = await provider.presignPut('key', {
contentType: 'text/plain',
});
t.truthy(result);
t.true(result!.url.length > 0);
t.true(result!.url.includes('X-Amz-Algorithm=AWS4-HMAC-SHA256'));
t.true(result!.url.includes('X-Amz-SignedHeaders='));
t.true(result!.url.includes('content-type'));
t.deepEqual(result!.headers, { 'Content-Type': 'text/plain' });
const now = Date.now();
t.true(result!.expiresAt.getTime() >= now + SIGNED_URL_EXPIRED * 1000 - 2000);
t.true(result!.expiresAt.getTime() <= now + SIGNED_URL_EXPIRED * 1000 + 2000);
});
test('presignUploadPart should return url', async t => {
const provider = createProvider();
const result = await provider.presignUploadPart('key', 'upload-1', 3);
t.truthy(result);
t.true(result!.url.length > 0);
t.true(result!.url.includes('X-Amz-Algorithm=AWS4-HMAC-SHA256'));
});
test('createMultipartUpload should return uploadId', async t => {
const provider = createProvider();
let receivedKey: string | undefined;
let receivedMeta: any;
(provider as any).client = {
createMultipartUpload: async (key: string, meta: any) => {
receivedKey = key;
receivedMeta = meta;
return { uploadId: 'upload-1' };
},
};
const now = Date.now();
const result = await provider.createMultipartUpload('key', {
contentType: 'text/plain',
});
t.is(result?.uploadId, 'upload-1');
t.true(result!.expiresAt.getTime() >= now + SIGNED_URL_EXPIRED * 1000 - 2000);
t.true(result!.expiresAt.getTime() <= now + SIGNED_URL_EXPIRED * 1000 + 2000);
t.is(receivedKey, 'key');
t.is(receivedMeta.contentType, 'text/plain');
});
test('completeMultipartUpload should order parts', async t => {
const provider = createProvider();
let receivedParts: any;
(provider as any).client = {
completeMultipartUpload: async (
_key: string,
_uploadId: string,
parts: any
) => {
receivedParts = parts;
},
};
await provider.completeMultipartUpload('key', 'upload-1', [
{ partNumber: 2, etag: 'b' },
{ partNumber: 1, etag: 'a' },
]);
t.deepEqual(receivedParts, [
{ partNumber: 1, etag: 'a' },
{ partNumber: 2, etag: 'b' },
]);
});
packages/backend/server/src/base/storage/factory.ts
-20
View File
@@ -1,20 +0,0 @@
import { Injectable } from '@nestjs/common';
import {
StorageProvider,
StorageProviderConfig,
StorageProviders,
} from './providers';
@Injectable()
export class StorageProviderFactory {
create(config: StorageProviderConfig): StorageProvider {
const Provider = StorageProviders[config.provider];
if (!Provider) {
throw new Error(`Unknown storage provider type: ${config.provider}`);
}
return new Provider(config.config, config.bucket);
}
}
packages/backend/server/src/base/storage/index.ts
+2 -11
View File
@@ -1,12 +1,3 @@
import { Global, Module } from '@nestjs/common';
import { StorageProviderFactory } from './factory';
@Global()
@Module({
providers: [StorageProviderFactory],
exports: [StorageProviderFactory],
})
export class StorageProviderModule {}
export { StorageProviderFactory } from './factory';
export * from './providers';
export type * from './types';
export * from './utils';
packages/backend/server/src/base/storage/providers/fs.ts
-318
View File
@@ -1,318 +0,0 @@
import {
accessSync,
constants,
createReadStream,
Dirent,
mkdirSync,
readdirSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from 'node:fs';
import { homedir } from 'node:os';
import { join, parse } from 'node:path';
import { Readable } from 'node:stream';
import { Logger } from '@nestjs/common';
import {
BlobInputType,
GetObjectMetadata,
ListObjectsMetadata,
PutObjectMetadata,
StorageProvider,
} from './provider';
import { autoMetadata, toBuffer } from './utils';
function normalizeStorageKey(key: string): string {
const normalized = key.replaceAll('\\', '/');
const segments = normalized.split('/');
if (
!normalized ||
normalized.startsWith('/') ||
segments.some(segment => !segment || segment === '.' || segment === '..')
) {
throw new Error(`Invalid storage key: ${key}`);
}
return segments.join('/');
}
function normalizeStoragePrefix(prefix: string): string {
const normalized = prefix.replaceAll('\\', '/');
if (!normalized) {
return normalized;
}
if (normalized.startsWith('/')) {
throw new Error(`Invalid storage prefix: ${prefix}`);
}
const segments = normalized.split('/');
const lastSegment = segments.pop();
if (
lastSegment === undefined ||
segments.some(segment => !segment || segment === '.' || segment === '..') ||
lastSegment === '.' ||
lastSegment === '..'
) {
throw new Error(`Invalid storage prefix: ${prefix}`);
}
if (lastSegment === '') {
return `${segments.join('/')}/`;
}
return [...segments, lastSegment].join('/');
}
export interface FsStorageConfig {
path: string;
}
export class FsStorageProvider implements StorageProvider {
private readonly path: string;
private readonly logger: Logger;
readonly type = 'fs';
constructor(
config: FsStorageConfig,
public readonly bucket: string
) {
this.path = config.path.startsWith('~/')
? join(homedir(), config.path.slice(2), bucket)
: join(config.path, bucket);
this.ensureAvailability();
this.logger = new Logger(`${FsStorageProvider.name}:${bucket}`);
}
async put(
key: string,
body: BlobInputType,
metadata: PutObjectMetadata = {}
): Promise<void> {
key = normalizeStorageKey(key);
const blob = await toBuffer(body);
// write object
this.writeObject(key, blob);
// write metadata
await this.writeMetadata(key, blob, metadata);
this.logger.verbose(`Object \`${key}\` put`);
}
async head(key: string) {
key = normalizeStorageKey(key);
const metadata = this.readMetadata(key);
if (!metadata) {
this.logger.verbose(`Object \`${key}\` not found`);
return undefined;
}
return metadata;
}
async get(key: string): Promise<{
body?: Readable;
metadata?: GetObjectMetadata;
}> {
key = normalizeStorageKey(key);
try {
const metadata = this.readMetadata(key);
const stream = this.readObject(this.join(key));
this.logger.verbose(`Read object \`${key}\``);
return {
body: stream,
metadata,
};
} catch (e) {
this.logger.error(`Failed to read object \`${key}\``, e);
return {};
}
}
async list(prefix?: string): Promise<ListObjectsMetadata[]> {
// prefix cases:
// - `undefined`: list all objects
// - `a/b`: list objects under dir `a` with prefix `b`, `b` might be a dir under `a` as well.
// - `a/b/` list objects under dir `a/b`
// read dir recursively and filter out '.metadata.json' files
let dir = this.path;
if (prefix) {
prefix = normalizeStoragePrefix(prefix);
const parts = prefix.split(/[/\\]/);
// for prefix `a/b/c`, move `a/b` to dir and `c` to key prefix
if (parts.length > 1) {
dir = join(dir, ...parts.slice(0, -1));
prefix = parts[parts.length - 1];
}
}
const results: ListObjectsMetadata[] = [];
async function getFiles(dir: string, prefix?: string): Promise<void> {
try {
const entries: Dirent[] = readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const res = join(dir, entry.name);
if (entry.isDirectory()) {
if (!prefix || entry.name.startsWith(prefix)) {
await getFiles(res);
}
} else if (
(!prefix || entry.name.startsWith(prefix)) &&
!entry.name.endsWith('.metadata.json')
) {
const stat = statSync(res);
results.push({
key: res,
lastModified: stat.mtime,
contentLength: stat.size,
});
}
}
} catch {
// failed to read dir, stop recursion
}
}
await getFiles(dir, prefix);
// trim path with `this.path` prefix
results.forEach(r => (r.key = r.key.slice(this.path.length + 1)));
return results;
}
delete(key: string): Promise<void> {
key = normalizeStorageKey(key);
try {
rmSync(this.join(key), { force: true });
rmSync(this.join(`${key}.metadata.json`), { force: true });
} catch (e) {
throw new Error(`Failed to delete object \`${key}\``, {
cause: e,
});
}
this.logger.verbose(`Object \`${key}\` deleted`);
return Promise.resolve();
}
ensureAvailability() {
// check stats
const stats = statSync(this.path, {
throwIfNoEntry: false,
});
// not existing, create it
if (!stats) {
try {
mkdirSync(this.path, { recursive: true });
} catch (e) {
throw new Error(
`Failed to create target directory for fs storage provider: ${this.path}`,
{
cause: e,
}
);
}
} else if (stats.isDirectory()) {
// the target directory has already existed, check if it is readable & writable
try {
accessSync(this.path, constants.W_OK | constants.R_OK);
} catch (e) {
throw new Error(
`The target directory for fs storage provider has already existed, but it is not readable & writable: ${this.path}`,
{
cause: e,
}
);
}
} else if (stats.isFile()) {
throw new Error(
`The target directory for fs storage provider is a file: ${this.path}`
);
}
}
private join(...paths: string[]) {
return join(this.path, ...paths);
}
private readObject(file: string): Readable | undefined {
const state = statSync(file, { throwIfNoEntry: false });
if (state?.isFile()) {
return createReadStream(file);
}
return undefined;
}
private writeObject(key: string, blob: Buffer) {
const path = this.join(key);
mkdirSync(parse(path).dir, { recursive: true });
writeFileSync(path, blob);
}
private async writeMetadata(
key: string,
blob: Buffer,
raw: PutObjectMetadata
) {
try {
const metadata = autoMetadata(blob, raw);
if (raw.checksumCRC32 && metadata.checksumCRC32 !== raw.checksumCRC32) {
throw new Error(
'The checksum of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.'
);
}
if (raw.contentLength && metadata.contentLength !== raw.contentLength) {
throw new Error(
'The content length of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.'
);
}
writeFileSync(
this.join(`${key}.metadata.json`),
JSON.stringify({
...metadata,
lastModified: Date.now(),
})
);
} catch (e) {
this.logger.warn(`Failed to write metadata of object \`${key}\``, e);
}
}
private readMetadata(key: string): GetObjectMetadata | undefined {
try {
const raw = JSON.parse(
readFileSync(this.join(`${key}.metadata.json`), {
encoding: 'utf-8',
})
);
return {
...raw,
lastModified: new Date(raw.lastModified),
expires: raw.expires ? new Date(raw.expires) : undefined,
};
} catch (e) {
this.logger.warn(`Failed to read metadata of object \`${key}\``, e);
return;
}
}
}
packages/backend/server/src/base/storage/providers/index.ts
+66 -18
View File
@@ -1,20 +1,45 @@
import { Type } from '@nestjs/common';
import { JSONSchema } from '../../config';
import { FsStorageConfig, FsStorageProvider } from './fs';
import { StorageProvider } from './provider';
import { R2_JURISDICTIONS, R2StorageConfig, R2StorageProvider } from './r2';
import { S3StorageConfig, S3StorageProvider } from './s3';
export type StorageProviderName = 'fs' | 'aws-s3' | 'cloudflare-r2';
export const StorageProviders: Record<
StorageProviderName,
Type<StorageProvider>
> = {
fs: FsStorageProvider,
'aws-s3': S3StorageProvider,
'cloudflare-r2': R2StorageProvider,
};
export type StorageProviderName =
| 'fs'
| 'aws-s3'
| 'cloudflare-r2'
| 'assetpack';
export interface FsStorageConfig {
path: string;
}
export type AssetpackStorageConfig = FsStorageConfig;
export interface S3StorageConfig {
endpoint?: string;
region: string;
credentials?: {
accessKeyId?: string;
secretAccessKey?: string;
sessionToken?: string;
};
forcePathStyle?: boolean;
requestTimeoutMs?: number;
minPartSize?: number;
presign?: {
expiresInSeconds?: number;
signContentTypeForPut?: boolean;
};
}
export const R2_JURISDICTIONS = ['default', 'eu'] as const;
export interface R2StorageConfig extends Omit<S3StorageConfig, 'endpoint'> {
accountId: string;
jurisdiction?: (typeof R2_JURISDICTIONS)[number];
usePresignedURL?: {
enabled: boolean;
urlPrefix?: string;
signKey?: string;
};
}
export type StorageProviderConfig = { bucket: string } & (
| {
@@ -29,6 +54,10 @@ export type StorageProviderConfig = { bucket: string } & (
provider: 'cloudflare-r2';
config: R2StorageConfig;
}
| {
provider: 'assetpack';
config: AssetpackStorageConfig;
}
);
const S3ConfigSchema: JSONSchema = {
@@ -186,16 +215,35 @@ export const StorageJSONSchema: JSONSchema = {
},
},
},
{
type: 'object',
properties: {
provider: {
type: 'string',
enum: ['assetpack'],
},
bucket: {
type: 'string',
},
config: {
type: 'object',
properties: {
path: {
type: 'string',
},
},
},
},
},
],
};
export type * from './provider';
export type * from '../types';
export {
applyAttachHeaders,
autoMetadata,
PROXY_MULTIPART_PATH,
PROXY_UPLOAD_PATH,
sniffMime,
STORAGE_PROXY_ROOT,
toBuffer,
} from './utils';
} from '../utils';
packages/backend/server/src/base/storage/providers/provider.ts
-84
View File
@@ -1,84 +0,0 @@
import type { Readable } from 'node:stream';
export interface GetObjectMetadata {
/**
* @default 'application/octet-stream'
*/
contentType: string;
contentLength: number;
lastModified: Date;
checksumCRC32?: string;
}
export interface PutObjectMetadata {
contentType?: string;
contentLength?: number;
checksumCRC32?: string;
}
export interface ListObjectsMetadata {
key: string;
lastModified: Date;
contentLength: number;
}
export type BlobInputType = Buffer | Readable | string;
export type BlobOutputType = Readable;
export interface PresignedUpload {
url: string;
headers?: Record<string, string>;
expiresAt: Date;
}
export interface MultipartUploadInit {
uploadId: string;
expiresAt: Date;
}
export interface MultipartUploadPart {
partNumber: number;
etag: string;
}
export interface StorageProvider {
put(
key: string,
body: BlobInputType,
metadata?: PutObjectMetadata
): Promise<void>;
presignPut?(
key: string,
metadata?: PutObjectMetadata
): Promise<PresignedUpload | undefined>;
createMultipartUpload?(
key: string,
metadata?: PutObjectMetadata
): Promise<MultipartUploadInit | undefined>;
presignUploadPart?(
key: string,
uploadId: string,
partNumber: number
): Promise<PresignedUpload | undefined>;
listMultipartUploadParts?(
key: string,
uploadId: string
): Promise<MultipartUploadPart[] | undefined>;
completeMultipartUpload?(
key: string,
uploadId: string,
parts: MultipartUploadPart[]
): Promise<void>;
abortMultipartUpload?(key: string, uploadId: string): Promise<void>;
head(key: string): Promise<GetObjectMetadata | undefined>;
get(
key: string,
signedUrl?: boolean
): Promise<{
redirectUrl?: string;
body?: BlobOutputType;
metadata?: GetObjectMetadata;
}>;
list(prefix?: string): Promise<ListObjectsMetadata[]>;
delete(key: string): Promise<void>;
}
packages/backend/server/src/base/storage/providers/r2.ts
-251
View File
@@ -1,251 +0,0 @@
import assert from 'node:assert';
import { Readable } from 'node:stream';
import { Logger } from '@nestjs/common';
import {
GetObjectMetadata,
PresignedUpload,
PutObjectMetadata,
} from './provider';
import { S3StorageConfig, S3StorageProvider } from './s3';
import {
PROXY_MULTIPART_PATH,
PROXY_UPLOAD_PATH,
SIGNED_URL_EXPIRED,
} from './utils';
export const R2_JURISDICTIONS = ['eu'] as const;
type R2Jurisdiction = (typeof R2_JURISDICTIONS)[number];
export interface R2StorageConfig extends Omit<
S3StorageConfig,
'endpoint' | 'forcePathStyle'
> {
accountId: string;
jurisdiction?: R2Jurisdiction;
usePresignedURL?: {
enabled: boolean;
urlPrefix?: string;
signKey?: string;
};
}
export class R2StorageProvider extends S3StorageProvider {
private readonly encoder = new TextEncoder();
private readonly key: Uint8Array;
constructor(
private readonly config: R2StorageConfig,
bucket: string
) {
assert(config.accountId, 'accountId is required for R2 storage provider');
const account = config.jurisdiction
? `${config.accountId}.${config.jurisdiction}`
: config.accountId;
const endpoint = `https://${account}.r2.cloudflarestorage.com`;
super(
{
...config,
forcePathStyle: true,
endpoint,
},
bucket
);
this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`);
this.key = this.encoder.encode(config.usePresignedURL?.signKey ?? '');
}
private get shouldUseProxyUpload() {
const { usePresignedURL } = this.config;
return (
!!usePresignedURL?.enabled &&
!!usePresignedURL.signKey &&
this.key.length > 0
);
}
private parseWorkspaceKey(fullKey: string) {
const [workspaceId, ...rest] = fullKey.split('/');
if (!workspaceId || rest.length !== 1) {
return null;
}
return { workspaceId, key: rest.join('/') };
}
private async signPayload(payload: string) {
const key = await crypto.subtle.importKey(
'raw',
this.key,
{ name: 'HMAC', hash: 'SHA-256' },
false,
['sign', 'verify']
);
const mac = await crypto.subtle.sign(
'HMAC',
key,
this.encoder.encode(payload)
);
return Buffer.from(mac).toString('base64');
}
private async signUrl(url: URL): Promise<string> {
const timestamp = Math.floor(Date.now() / 1000);
const base64Mac = await this.signPayload(`${url.pathname}${timestamp}`);
url.searchParams.set('sign', `${timestamp}-${base64Mac}`);
return url.toString();
}
private async createProxyUrl(
path: string,
canonicalFields: (string | number | undefined)[],
query: Record<string, string | number | undefined>
) {
const exp = Math.floor(Date.now() / 1000) + SIGNED_URL_EXPIRED;
const canonical = [
path,
...canonicalFields.map(field =>
field === undefined ? '' : field.toString()
),
exp.toString(),
].join('\n');
const token = await this.signPayload(canonical);
const url = new URL(`http://localhost${path}`);
for (const [key, value] of Object.entries(query)) {
if (value === undefined) continue;
url.searchParams.set(key, value.toString());
}
url.searchParams.set('exp', exp.toString());
url.searchParams.set('token', `${exp}-${token}`);
return { url: url.pathname + url.search, expiresAt: new Date(exp * 1000) };
}
override async presignPut(
key: string,
metadata: PutObjectMetadata = {}
): Promise<PresignedUpload | undefined> {
if (!this.shouldUseProxyUpload) {
return super.presignPut(key, metadata);
}
const parsed = this.parseWorkspaceKey(key);
if (!parsed) {
return super.presignPut(key, metadata);
}
const contentType = metadata.contentType ?? 'application/octet-stream';
const { url, expiresAt } = await this.createProxyUrl(
PROXY_UPLOAD_PATH,
[parsed.workspaceId, parsed.key, contentType, metadata.contentLength],
{
workspaceId: parsed.workspaceId,
key: parsed.key,
contentType,
contentLength: metadata.contentLength,
}
);
return {
url,
headers: { 'Content-Type': contentType },
expiresAt,
};
}
override async presignUploadPart(
key: string,
uploadId: string,
partNumber: number
): Promise<PresignedUpload | undefined> {
if (!this.shouldUseProxyUpload) {
return super.presignUploadPart(key, uploadId, partNumber);
}
const parsed = this.parseWorkspaceKey(key);
if (!parsed) {
return super.presignUploadPart(key, uploadId, partNumber);
}
return this.createProxyUrl(
PROXY_MULTIPART_PATH,
[parsed.workspaceId, parsed.key, uploadId, partNumber],
{
workspaceId: parsed.workspaceId,
key: parsed.key,
uploadId,
partNumber,
}
);
}
async proxyPutObject(
key: string,
body: Readable | Buffer | Uint8Array | string,
options: { contentType?: string; contentLength?: number } = {}
) {
return this.client.putObject(key, this.normalizeBody(body), {
contentType: options.contentType,
contentLength: options.contentLength,
});
}
async proxyUploadPart(
key: string,
uploadId: string,
partNumber: number,
body: Readable | Buffer | Uint8Array | string,
options: { contentLength?: number } = {}
) {
const result = await this.client.uploadPart(
key,
uploadId,
partNumber,
this.normalizeBody(body),
{ contentLength: options.contentLength }
);
return result.etag;
}
private normalizeBody(body: Readable | Buffer | Uint8Array | string) {
// s3mini does not accept Node.js Readable directly.
// Convert it to Web ReadableStream for compatibility.
if (body instanceof Readable) {
return Readable.toWeb(body);
} else if (typeof body === 'string') {
return this.encoder.encode(body);
}
return body;
}
override async get(
key: string,
signedUrl?: boolean
): Promise<{
body?: Readable;
metadata?: GetObjectMetadata;
redirectUrl?: string;
}> {
const { usePresignedURL: { enabled, urlPrefix } = {} } = this.config;
if (signedUrl && enabled && urlPrefix) {
const metadata = await this.head(key);
const url = await this.signUrl(new URL(`/${key}`, urlPrefix));
if (metadata) {
return {
redirectUrl: url.toString(),
metadata,
};
}
// object not found
return {};
}
// fallback to s3 get
return super.get(key, signedUrl);
}
}
packages/backend/server/src/base/storage/providers/s3.ts
-363
View File
@@ -1,363 +0,0 @@
/* oxlint-disable @typescript-eslint/no-non-null-assertion */
import { Readable } from 'node:stream';
import type {
S3CompatClient,
S3CompatConfig,
S3CompatCredentials,
} from '@affine/s3-compat';
import { createS3CompatClient } from '@affine/s3-compat';
import { Logger } from '@nestjs/common';
import {
BlobInputType,
GetObjectMetadata,
ListObjectsMetadata,
MultipartUploadInit,
MultipartUploadPart,
PresignedUpload,
PutObjectMetadata,
StorageProvider,
} from './provider';
import { autoMetadata, SIGNED_URL_EXPIRED, toBuffer } from './utils';
export interface S3StorageConfig {
endpoint?: string;
region: string;
credentials: S3CompatCredentials;
forcePathStyle?: boolean;
requestTimeoutMs?: number;
minPartSize?: number;
presign?: {
expiresInSeconds?: number;
signContentTypeForPut?: boolean;
};
usePresignedURL?: {
enabled: boolean;
};
}
function resolveEndpoint(config: S3StorageConfig) {
if (config.endpoint) {
return config.endpoint;
}
if (config.region === 'us-east-1') {
return 'https://s3.amazonaws.com';
}
return `https://s3.${config.region}.amazonaws.com`;
}
function joinPath(basePath: string, suffix: string) {
const trimmedBase = basePath.endsWith('/') ? basePath.slice(0, -1) : basePath;
const trimmedSuffix = suffix.startsWith('/') ? suffix.slice(1) : suffix;
if (!trimmedBase) {
return `/${trimmedSuffix}`;
}
if (!trimmedSuffix) {
return trimmedBase;
}
return `${trimmedBase}/${trimmedSuffix}`;
}
function composeEndpointUrl(config: S3CompatConfig) {
const url = new URL(config.endpoint);
if (config.forcePathStyle) {
const firstSegment = url.pathname.split('/').find(Boolean);
if (firstSegment !== config.bucket) {
url.pathname = joinPath(url.pathname, config.bucket);
}
return url.toString();
}
const firstSegment = url.pathname.split('/').find(Boolean);
const hostHasBucket = url.hostname.startsWith(`${config.bucket}.`);
const pathHasBucket = firstSegment === config.bucket;
if (!hostHasBucket && !pathHasBucket) {
url.hostname = `${config.bucket}.${url.hostname}`;
}
return url.toString();
}
export class S3StorageProvider implements StorageProvider {
protected logger: Logger;
protected client: S3CompatClient;
private readonly usePresignedURL: boolean;
private readonly endpoint: string;
get endpointUrl() {
return this.endpoint;
}
constructor(
config: S3StorageConfig,
public readonly bucket: string
) {
const { usePresignedURL, presign, credentials, ...clientConfig } = config;
const compatConfig: S3CompatConfig = {
...clientConfig,
endpoint: resolveEndpoint(config),
bucket,
requestTimeoutMs: clientConfig.requestTimeoutMs ?? 60_000,
presign: {
expiresInSeconds: presign?.expiresInSeconds ?? SIGNED_URL_EXPIRED,
signContentTypeForPut: presign?.signContentTypeForPut ?? true,
},
};
this.endpoint = composeEndpointUrl(compatConfig);
this.client = createS3CompatClient(compatConfig, credentials);
this.usePresignedURL = usePresignedURL?.enabled ?? false;
this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`);
}
async put(
key: string,
body: BlobInputType,
metadata: PutObjectMetadata = {}
): Promise<void> {
const blob = await toBuffer(body);
metadata = autoMetadata(blob, metadata);
try {
await this.client.putObject(key, blob, {
contentType: metadata.contentType,
contentLength: metadata.contentLength,
});
this.logger.verbose(`Object \`${key}\` put`);
} catch (e) {
this.logger.error(
`Failed to put object (${JSON.stringify({
key,
bucket: this.bucket,
metadata,
})})`
);
throw e;
}
}
async presignPut(
key: string,
metadata: PutObjectMetadata = {}
): Promise<PresignedUpload | undefined> {
try {
const contentType = metadata.contentType ?? 'application/octet-stream';
const result = await this.client.presignPutObject(key, { contentType });
return {
url: result.url,
headers: result.headers,
expiresAt: result.expiresAt,
};
} catch (e) {
this.logger.error(
`Failed to presign put object (${JSON.stringify({
key,
bucket: this.bucket,
metadata,
})}`
);
throw e;
}
}
async createMultipartUpload(
key: string,
metadata: PutObjectMetadata = {}
): Promise<MultipartUploadInit | undefined> {
try {
const contentType = metadata.contentType ?? 'application/octet-stream';
const response = await this.client.createMultipartUpload(key, {
contentType,
});
if (!response.uploadId) {
return;
}
return {
uploadId: response.uploadId,
expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000),
};
} catch (e) {
this.logger.error(
`Failed to create multipart upload (${JSON.stringify({
key,
bucket: this.bucket,
metadata,
})}`
);
throw e;
}
}
async presignUploadPart(
key: string,
uploadId: string,
partNumber: number
): Promise<PresignedUpload | undefined> {
try {
const result = await this.client.presignUploadPart(
key,
uploadId,
partNumber
);
return {
url: result.url,
expiresAt: result.expiresAt,
};
} catch (e) {
this.logger.error(
`Failed to presign upload part (${JSON.stringify({ key, bucket: this.bucket, uploadId, partNumber })}`
);
throw e;
}
}
async listMultipartUploadParts(
key: string,
uploadId: string
): Promise<MultipartUploadPart[] | undefined> {
try {
return await this.client.listParts(key, uploadId);
} catch (e) {
this.logger.error(`Failed to list multipart upload parts for \`${key}\``);
throw e;
}
}
async completeMultipartUpload(
key: string,
uploadId: string,
parts: MultipartUploadPart[]
): Promise<void> {
try {
const orderedParts = [...parts].sort(
(left, right) => left.partNumber - right.partNumber
);
await this.client.completeMultipartUpload(key, uploadId, orderedParts);
} catch (e) {
this.logger.error(`Failed to complete multipart upload for \`${key}\``);
throw e;
}
}
async abortMultipartUpload(key: string, uploadId: string): Promise<void> {
try {
await this.client.abortMultipartUpload(key, uploadId);
} catch (e) {
this.logger.error(`Failed to abort multipart upload for \`${key}\``);
throw e;
}
}
async head(key: string) {
try {
const obj = await this.client.headObject(key);
if (!obj) {
this.logger.verbose(`Object \`${key}\` not found`);
return undefined;
}
return {
contentType: obj.contentType ?? 'application/octet-stream',
contentLength: obj.contentLength ?? 0,
lastModified: obj.lastModified ?? new Date(0),
checksumCRC32: obj.checksumCRC32,
};
} catch (e) {
this.logger.error(`Failed to head object \`${key}\``);
throw e;
}
}
async get(
key: string,
signedUrl?: boolean
): Promise<{
body?: Readable;
metadata?: GetObjectMetadata;
redirectUrl?: string;
}> {
try {
if (this.usePresignedURL && signedUrl) {
const metadata = await this.head(key);
if (metadata) {
const result = await this.client.presignGetObject(key);
return {
redirectUrl: result.url,
metadata,
};
}
// object not found
return {};
}
const obj = await this.client.getObjectResponse(key);
if (!obj || !obj.body) {
this.logger.verbose(`Object \`${key}\` not found`);
return {};
}
const contentType = obj.headers.get('content-type') ?? undefined;
const contentLengthHeader = obj.headers.get('content-length');
const contentLength = contentLengthHeader
? Number(contentLengthHeader)
: undefined;
const lastModifiedHeader = obj.headers.get('last-modified');
const lastModified = lastModifiedHeader
? new Date(lastModifiedHeader)
: undefined;
this.logger.verbose(`Read object \`${key}\``);
return {
body: Readable.fromWeb(obj.body),
metadata: {
contentType: contentType ?? 'application/octet-stream',
contentLength: contentLength ?? 0,
lastModified: lastModified ?? new Date(0),
checksumCRC32: obj.headers.get('x-amz-checksum-crc32') ?? undefined,
},
};
} catch (e) {
this.logger.error(`Failed to read object \`${key}\``);
throw e;
}
}
async list(prefix?: string): Promise<ListObjectsMetadata[]> {
try {
const result = await this.client.listObjectsV2(prefix);
this.logger.verbose(
`List ${result.length} objects with prefix \`${prefix}\``
);
return result;
} catch (e) {
this.logger.error(`Failed to list objects with prefix \`${prefix}\``);
throw e;
}
}
async delete(key: string): Promise<void> {
try {
await this.client.deleteObject(key);
this.logger.verbose(`Deleted object \`${key}\``);
} catch (e) {
this.logger.error(`Failed to delete object \`${key}\``, {
bucket: this.bucket,
key,
cause: e,
});
throw e;
}
}
}
packages/backend/server/src/base/storage/types.ts
+32
View File
@@ -0,0 +1,32 @@
import type { Readable } from 'node:stream';
export interface GetObjectMetadata {
/**
* @default 'application/octet-stream'
*/
contentType: string;
contentLength: number;
lastModified: Date;
checksumCRC32?: string;
}
export interface PutObjectMetadata {
contentType?: string;
contentLength?: number;
checksumCRC32?: string;
}
export interface ListObjectsMetadata {
key: string;
lastModified: Date;
contentLength: number;
}
export type BlobInputType = Buffer | Readable | string;
export type BlobOutputType = Readable;
export interface PresignedUpload {
url: string;
headers?: Record<string, string>;
expiresAt: Date;
}
packages/backend/server/src/base/storage/providers/utils.ts → packages/backend/server/src/base/storage/utils.ts
+2 -32
View File
@@ -1,11 +1,10 @@
import { Readable } from 'node:stream';
import { crc32 } from '@node-rs/crc32';
import type { Response } from 'express';
import { getStreamAsBuffer } from 'get-stream';
import { getMime } from '../../../native';
import { BlobInputType, PutObjectMetadata } from './provider';
import { getMime } from '../../native';
import type { BlobInputType } from './types';
export async function toBuffer(input: BlobInputType): Promise<Buffer> {
return input instanceof Readable
@@ -15,35 +14,6 @@ export async function toBuffer(input: BlobInputType): Promise<Buffer> {
: Buffer.from(input as string);
}
export function autoMetadata(
blob: Buffer,
raw: PutObjectMetadata = {}
): PutObjectMetadata {
const metadata = {
...raw,
};
if (!metadata.contentLength) {
metadata.contentLength = blob.byteLength;
}
try {
// checksum
if (!metadata.checksumCRC32) {
metadata.checksumCRC32 = crc32(blob).toString(16);
}
// mime type
if (!metadata.contentType) {
metadata.contentType = getMime(blob);
}
} catch {
// noop
}
return metadata;
}
const DANGEROUS_INLINE_MIME_PREFIXES = [
'text/html',
'application/xhtml+xml',
packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts
-2
View File
@@ -12,7 +12,6 @@ test('backend-runtime provider starts once, runs migrations once, and reports he
health: Sinon.stub().resolves({
started: true,
databaseConnected: true,
objectStorageConfigured: true,
}),
};
(provider as any).runtime = runtime;
@@ -25,7 +24,6 @@ test('backend-runtime provider starts once, runs migrations once, and reports he
t.is(runtime.start.callCount, 2);
t.is(runtime.runMigrations.callCount, 1);
t.true(health.databaseConnected);
t.true(health.objectStorageConfigured);
t.is(runtime.stop.callCount, 1);
});
packages/backend/server/src/core/backend-runtime/index.ts
+2 -7
View File
@@ -1,17 +1,12 @@
import { Global, Module } from '@nestjs/common';
import { BackendRuntimeBlobJob } from './blob-job';
import { BackendRuntimeHousekeepingJob } from './job';
import { BackendRuntimeProvider } from './provider';
@Global()
@Module({
providers: [
BackendRuntimeProvider,
BackendRuntimeBlobJob,
BackendRuntimeHousekeepingJob,
],
exports: [BackendRuntimeProvider, BackendRuntimeBlobJob],
providers: [BackendRuntimeProvider, BackendRuntimeHousekeepingJob],
exports: [BackendRuntimeProvider],
})
export class BackendRuntimeModule {}
packages/backend/server/src/core/backend-runtime/provider.ts
+1 -50
View File
@@ -30,9 +30,7 @@ export class BackendRuntimeProvider
await this.runtime.start();
await this.runMigrationsOnce();
const health = await this.runtime.health();
this.logger.log(
`backend runtime started: db=${health.databaseConnected} objectStorage=${health.objectStorageConfigured}`
);
this.logger.log(`backend runtime started: db=${health.databaseConnected}`);
}
async stop() {
@@ -44,18 +42,6 @@ export class BackendRuntimeProvider
return await this.runtime.health();
}
async cleanupExpiredPendingBlobs(cutoffMs: number, limit: number) {
return await this.measured('cleanupExpiredPendingBlobs', rt =>
rt.cleanupExpiredPendingBlobs(cutoffMs, limit)
);
}
async releaseDeletedBlobs(workspaceId: string, limit: number) {
return await this.measured('releaseDeletedBlobs', rt =>
rt.releaseDeletedBlobs(workspaceId, limit)
);
}
async cleanupExpiredSnapshotHistories(limit: number) {
return await this.measured('cleanupExpiredSnapshotHistories', rt =>
rt.cleanupExpiredSnapshotHistories(limit)
@@ -80,41 +66,6 @@ export class BackendRuntimeProvider
);
}
async backfillMissingBlobMetadata(
workspaceId: string | null | undefined,
limit: number
) {
return await this.measured('backfillMissingBlobMetadata', rt =>
rt.backfillMissingBlobMetadata(workspaceId, limit)
);
}
async rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number) {
return await this.measured('rebuildWorkspaceDocBlobRefs', rt =>
rt.rebuildWorkspaceDocBlobRefs(workspaceId, limit)
);
}
async planUnreferencedWorkspaceBlobs(
workspaceId: string,
gracePeriodDays: number,
limit: number
) {
return await this.measured('planUnreferencedWorkspaceBlobs', rt =>
rt.planUnreferencedWorkspaceBlobs(workspaceId, gracePeriodDays, limit)
);
}
async executeBlobCleanupCandidates(
runId: string,
gracePeriodDays: number,
limit: number
) {
return await this.measured('executeBlobCleanupCandidates', rt =>
rt.executeBlobCleanupCandidates(runId, gracePeriodDays, limit)
);
}
private async measured<T>(
method: string,
fn: (runtime: RuntimeInstance) => Promise<T>
packages/backend/server/src/core/storage-runtime/__tests__/provider.spec.ts
+42
View File
@@ -0,0 +1,42 @@
import test from 'ava';
import Sinon from 'sinon';
import { StorageRuntimeProvider } from '../provider';
function createProvider() {
const provider = new StorageRuntimeProvider();
const runtime = {
start: Sinon.stub().resolves(),
stop: Sinon.stub().resolves(),
runMigrations: Sinon.stub().resolves(),
health: Sinon.stub().resolves({
started: true,
databaseConnected: true,
provider: 'fs',
}),
};
(provider as any).runtime = runtime;
return { provider, runtime };
}
test('storage-runtime provider restarts on storage config changes', async t => {
const { provider, runtime } = createProvider();
await provider.start();
await provider.onConfigChanged({ updates: { storages: {} } });
t.is(runtime.stop.callCount, 1);
t.is(runtime.start.callCount, 2);
t.is(runtime.runMigrations.callCount, 2);
});
test('storage-runtime provider ignores unrelated config changes', async t => {
const { provider, runtime } = createProvider();
await provider.start();
await provider.onConfigChanged({ updates: { flags: {} } });
t.is(runtime.stop.callCount, 0);
t.is(runtime.start.callCount, 1);
t.is(runtime.runMigrations.callCount, 1);
});
packages/backend/server/src/core/storage-runtime/index.ts
+15
View File
@@ -0,0 +1,15 @@
import { Global, Module } from '@nestjs/common';
import { StorageRuntimeProvider } from './provider';
@Global()
@Module({
providers: [StorageRuntimeProvider],
exports: [StorageRuntimeProvider],
})
export class StorageRuntimeModule {}
export {
type StorageRuntimeGetObjectResult,
StorageRuntimeProvider,
} from './provider';
packages/backend/server/src/core/storage-runtime/provider.ts
+350
View File
@@ -0,0 +1,350 @@
import { Readable } from 'node:stream';
import {
Injectable,
Logger,
type OnApplicationBootstrap,
type OnApplicationShutdown,
} from '@nestjs/common';
import type {
BlobOutputType,
GetObjectMetadata,
ListObjectsMetadata,
PresignedUpload,
PutObjectMetadata,
} from '../../base';
import { OnEvent } from '../../base';
import { wrapCallMetric } from '../../base/metrics';
import {
type RuntimeObjectGetResult,
type RuntimeObjectListEntry,
type RuntimeObjectMetadata,
type RuntimePresignedObjectRequest,
type StorageProviderCapabilities,
StorageRuntime,
type StorageRuntimeHealth,
} from '../../native';
type RuntimeInstance = InstanceType<typeof StorageRuntime>;
@Injectable()
export class StorageRuntimeProvider
implements OnApplicationBootstrap, OnApplicationShutdown
{
private readonly logger = new Logger(StorageRuntimeProvider.name);
private readonly runtime: RuntimeInstance = new StorageRuntime();
private migrationsStarted = false;
async onApplicationBootstrap() {
await this.start();
}
async onApplicationShutdown() {
await this.stop();
}
async start() {
await this.runtime.start();
await this.runMigrationsOnce();
const health = await this.runtime.health();
this.logger.log(
`storage runtime started: db=${health.databaseConnected} provider=${health.provider ?? 'none'}`
);
}
async stop() {
await this.runtime.stop();
this.logger.log('storage runtime stopped');
}
@OnEvent('config.init')
async onConfigInit() {
await this.start();
}
@OnEvent('config.changed')
async onConfigChanged({ updates }: Events['config.changed']) {
if (!('storages' in updates) && !('db' in updates)) {
return;
}
await this.restart();
}
async health(): Promise<StorageRuntimeHealth> {
return await this.runtime.health();
}
async providerCapabilities(
scope: string
): Promise<StorageProviderCapabilities> {
return await this.measured('providerCapabilities', rt =>
rt.providerCapabilities(scope)
);
}
async putObject(
scope: string,
key: string,
body: Buffer,
metadata?: PutObjectMetadata
) {
const result = await this.measured('putObject', rt =>
rt.putObject(scope, key, body, toRuntimeMetadata(metadata))
);
return fromRuntimeMetadata(result);
}
async headObject(scope: string, key: string) {
const metadata = await this.measured('headObject', rt =>
rt.headObject(scope, key)
);
return metadata ? fromRuntimeMetadata(metadata) : undefined;
}
async getObject(
scope: string,
key: string
): Promise<StorageRuntimeGetObjectResult> {
const result = await this.measured('getObject', rt =>
rt.getObject(scope, key)
);
return result ? fromRuntimeGetResult(result) : {};
}
async listObjects(scope: string, prefix?: string) {
const entries = await this.measured('listObjects', rt =>
rt.listObjects(scope, prefix)
);
return entries.map(fromRuntimeListEntry);
}
async deleteObject(scope: string, key: string) {
await this.measured('deleteObject', rt => rt.deleteObject(scope, key));
}
async presignPut(scope: string, key: string, metadata?: PutObjectMetadata) {
const result = await this.measured('presignPut', rt =>
rt.presignPut(scope, key, toRuntimeMetadata(metadata))
);
return result ? fromRuntimePresigned(result) : undefined;
}
async presignGet(scope: string, key: string) {
const result = await this.measured('presignGet', rt =>
rt.presignGet(scope, key)
);
return result ? fromRuntimePresigned(result) : undefined;
}
async createMultipartUpload(
scope: string,
key: string,
metadata?: PutObjectMetadata
) {
const result = await this.measured('createMultipartUpload', rt =>
rt.createMultipartUpload(scope, key, toRuntimeMetadata(metadata))
);
return result
? { uploadId: result.uploadId, expiresAt: new Date(result.expiresAtMs) }
: undefined;
}
async presignUploadPart(
scope: string,
key: string,
uploadId: string,
partNumber: number
) {
const result = await this.measured('presignUploadPart', rt =>
rt.presignUploadPart(scope, key, uploadId, partNumber)
);
return result ? fromRuntimePresigned(result) : undefined;
}
async listMultipartUploadParts(scope: string, key: string, uploadId: string) {
return (
(await this.measured('listMultipartUploadParts', rt =>
rt.listMultipartUploadParts(scope, key, uploadId)
)) ?? undefined
);
}
async proxyUploadPart(
scope: string,
key: string,
uploadId: string,
partNumber: number,
body: Buffer,
contentLength?: number
) {
return (
(await this.measured('proxyUploadPart', rt =>
rt.proxyUploadPart(
scope,
key,
uploadId,
partNumber,
body,
contentLength
)
)) ?? undefined
);
}
async completeMultipartUpload(
scope: string,
key: string,
uploadId: string,
parts: { partNumber: number; etag: string }[]
) {
return await this.measured('completeMultipartUpload', rt =>
rt.completeMultipartUpload(scope, key, uploadId, parts)
);
}
async abortMultipartUpload(scope: string, key: string, uploadId: string) {
return await this.measured('abortMultipartUpload', rt =>
rt.abortMultipartUpload(scope, key, uploadId)
);
}
async completeWorkspaceBlobUpload(
workspaceId: string,
key: string,
expected: { size: number; mime: string }
) {
return await this.measured('completeWorkspaceBlobUpload', rt =>
rt.completeWorkspaceBlobUpload(
workspaceId,
key,
expected.size,
expected.mime
)
);
}
async cleanupExpiredPendingBlobs(cutoffMs: number, limit: number) {
return await this.measured('cleanupExpiredPendingBlobs', rt =>
rt.cleanupExpiredPendingBlobs(cutoffMs, limit)
);
}
async releaseDeletedBlobs(workspaceId: string, limit: number) {
return await this.measured('releaseDeletedBlobs', rt =>
rt.releaseDeletedBlobs(workspaceId, limit)
);
}
async backfillMissingBlobMetadata(
workspaceId: string | null | undefined,
limit: number
) {
return await this.measured('backfillMissingBlobMetadata', rt =>
rt.backfillMissingBlobMetadata(workspaceId, limit)
);
}
async rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number) {
return await this.measured('rebuildWorkspaceDocBlobRefs', rt =>
rt.rebuildWorkspaceDocBlobRefs(workspaceId, limit)
);
}
async planUnreferencedWorkspaceBlobs(
workspaceId: string,
gracePeriodDays: number,
limit: number
) {
return await this.measured('planUnreferencedWorkspaceBlobs', rt =>
rt.planUnreferencedWorkspaceBlobs(workspaceId, gracePeriodDays, limit)
);
}
async executeBlobCleanupCandidates(
runId: string,
gracePeriodDays: number,
limit: number
) {
return await this.measured('executeBlobCleanupCandidates', rt =>
rt.executeBlobCleanupCandidates(runId, gracePeriodDays, limit)
);
}
private async measured<T>(
method: string,
fn: (runtime: RuntimeInstance) => Promise<T>
): Promise<T> {
return await wrapCallMetric(() => fn(this.runtime), 'storage', 'runtime', {
method,
})();
}
private async runMigrationsOnce() {
if (this.migrationsStarted) {
return;
}
await this.runtime.runMigrations();
this.migrationsStarted = true;
}
private async restart() {
await this.runtime.stop();
this.migrationsStarted = false;
await this.start();
}
}
function toRuntimeMetadata(metadata?: PutObjectMetadata) {
return metadata
? {
contentType: metadata.contentType,
contentLength: metadata.contentLength,
checksumCrc32: metadata.checksumCRC32,
}
: undefined;
}
function fromRuntimeMetadata(
metadata: RuntimeObjectMetadata
): GetObjectMetadata {
return {
contentType: metadata.contentType,
contentLength: metadata.contentLength,
lastModified: new Date(metadata.lastModifiedMs),
checksumCRC32: metadata.checksumCrc32,
};
}
function fromRuntimeGetResult(result: RuntimeObjectGetResult) {
return {
body: Readable.from(result.body),
metadata: fromRuntimeMetadata(result.metadata),
};
}
function fromRuntimeListEntry(
entry: RuntimeObjectListEntry
): ListObjectsMetadata {
return {
key: entry.key,
contentLength: entry.contentLength,
lastModified: new Date(entry.lastModifiedMs),
};
}
function fromRuntimePresigned(
request: RuntimePresignedObjectRequest
): PresignedUpload {
return {
url: request.url,
headers: JSON.parse(request.headersJson) as Record<string, string>,
expiresAt: new Date(request.expiresAtMs),
};
}
export type StorageRuntimeGetObjectResult = {
redirectUrl?: string;
body?: BlobOutputType;
metadata?: GetObjectMetadata;
};
packages/backend/server/src/core/backend-runtime/__tests__/blob-job.spec.ts → packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts
+5 -5
View File
@@ -1,7 +1,7 @@
import ava, { TestFn } from 'ava';
import Sinon from 'sinon';
import { BackendRuntimeBlobJob } from '../blob-job';
import { StorageBlobJob } from '../blob-job';
interface Context {
runtime: {
@@ -22,7 +22,7 @@ interface Context {
findMany: Sinon.SinonStub;
};
};
job: BackendRuntimeBlobJob;
job: StorageBlobJob;
}
const test = ava as TestFn<Context>;
@@ -31,7 +31,7 @@ test.beforeEach(t => {
t.context.runtime = {
health: Sinon.stub().resolves({
databaseConnected: true,
objectStorageConfigured: true,
providerConfigured: true,
}),
backfillMissingBlobMetadata: Sinon.stub(),
rebuildWorkspaceDocBlobRefs: Sinon.stub(),
@@ -49,7 +49,7 @@ test.beforeEach(t => {
findMany: Sinon.stub(),
},
};
t.context.job = new BackendRuntimeBlobJob(
t.context.job = new StorageBlobJob(
t.context.runtime as any,
t.context.event as any,
t.context.queue as any,
@@ -103,7 +103,7 @@ for (const scenario of objectStorageRequiredCases) {
test(`${scenario.name} skips when object storage is not configured`, async t => {
t.context.runtime.health.resolves({
databaseConnected: true,
objectStorageConfigured: false,
providerConfigured: false,
});
await scenario.run(t.context);
packages/backend/server/src/core/storage/__tests__/comment-attachment.spec.ts
+51 -2
View File
@@ -5,17 +5,66 @@ import test from 'ava';
import { createModule } from '../../../__tests__/create-module';
import { Mockers } from '../../../__tests__/mocks';
import { initTestingDB } from '../../../__tests__/utils';
import { Models } from '../../../models';
import { getMime } from '../../../native';
import { StorageRuntimeProvider } from '../../storage-runtime';
import { CommentAttachmentStorage, StorageModule } from '../index';
const objects = new Map<
string,
{
body: Buffer;
metadata: {
contentType: string;
contentLength: number;
lastModified: Date;
};
}
>();
const storageRuntime = {
putObject: async (
_scope: string,
key: string,
body: Buffer,
_metadata?: { contentType?: string; contentLength?: number }
) => {
const object = {
body,
metadata: {
contentType: getMime(body),
contentLength: body.length,
lastModified: new Date(),
},
};
objects.set(key, object);
return object.metadata;
},
headObject: async (_scope: string, key: string) => objects.get(key)?.metadata,
getObject: async (_scope: string, key: string) => {
const object = objects.get(key);
return object
? { body: Readable.from(object.body), metadata: object.metadata }
: {};
},
deleteObject: async (_scope: string, key: string) => {
objects.delete(key);
},
presignGet: async () => undefined,
};
const module = await createModule({
imports: [StorageModule],
tapModule: builder => {
builder.overrideProvider(StorageRuntimeProvider).useValue(storageRuntime);
},
});
const storage = module.get(CommentAttachmentStorage);
const models = module.get(Models);
test.before(async () => {
await storage.onConfigInit();
test.beforeEach(async () => {
await initTestingDB(module);
objects.clear();
});
test.after.always(async () => {
packages/backend/server/src/core/backend-runtime/blob-job.ts → packages/backend/server/src/core/storage/blob-job.ts
+10 -8
View File
@@ -3,8 +3,10 @@ import { Cron, CronExpression } from '@nestjs/schedule';
import { PrismaClient } from '@prisma/client';
import { EventBus, JobQueue, OnJob } from '../../base';
import { BackendRuntimeProvider } from './provider';
import { StorageRuntimeProvider } from '../storage-runtime';
// Queue keys are persisted API; keep the legacy backendRuntime.* names while
// StorageBlobJob and StorageRuntimeProvider own the implementation.
declare global {
interface Jobs {
'backendRuntime.backfillMissingBlobMetadata': {
@@ -45,11 +47,11 @@ declare global {
}
@Injectable()
export class BackendRuntimeBlobJob {
private readonly logger = new Logger(BackendRuntimeBlobJob.name);
export class StorageBlobJob {
private readonly logger = new Logger(StorageBlobJob.name);
constructor(
private readonly rt: BackendRuntimeProvider,
private readonly rt: StorageRuntimeProvider,
private readonly event: EventBus,
private readonly queue: JobQueue,
private readonly db: PrismaClient
@@ -355,7 +357,7 @@ export class BackendRuntimeBlobJob {
limit
);
await Promise.all(
result.workspaceIds.map(workspaceId =>
result.workspaceIds.map((workspaceId: string) =>
this.event.emitAsync('workspace.blobs.updated', { workspaceId })
)
);
@@ -375,7 +377,7 @@ export class BackendRuntimeBlobJob {
limit
);
await Promise.all(
result.workspaceIds.map(workspaceId =>
result.workspaceIds.map((workspaceId: string) =>
this.event.emitAsync('workspace.blobs.updated', { workspaceId })
)
);
@@ -409,12 +411,12 @@ export class BackendRuntimeBlobJob {
private async hasObjectStorage(operation: string) {
const health = await this.rt.health();
if (health.objectStorageConfigured) {
if (health.providerConfigured) {
return true;
}
this.logger.warn(
`skip ${operation}: BackendRuntime object storage is not configured`
`skip ${operation}: StorageRuntime provider is not configured`
);
return false;
}
packages/backend/server/src/core/storage/index.ts
+11 -3
View File
@@ -2,7 +2,8 @@ import './config';
import { Module } from '@nestjs/common';
import { BackendRuntimeModule } from '../backend-runtime';
import { StorageRuntimeModule } from '../storage-runtime';
import { StorageBlobJob } from './blob-job';
import { BlobUploadCleanupJob } from './job';
import { R2UploadController } from './r2-proxy';
import {
@@ -12,16 +13,23 @@ import {
} from './wrappers';
@Module({
imports: [BackendRuntimeModule],
imports: [StorageRuntimeModule],
controllers: [R2UploadController],
providers: [
WorkspaceBlobStorage,
AvatarStorage,
CommentAttachmentStorage,
StorageBlobJob,
BlobUploadCleanupJob,
],
exports: [WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage],
exports: [
WorkspaceBlobStorage,
AvatarStorage,
CommentAttachmentStorage,
StorageBlobJob,
],
})
export class StorageModule {}
export { StorageBlobJob } from './blob-job';
export { AvatarStorage, CommentAttachmentStorage, WorkspaceBlobStorage };
packages/backend/server/src/core/storage/job.ts
+2 -2
View File
@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { EventBus, JobQueue, OneDay, OnJob } from '../../base';
import { BackendRuntimeProvider } from '../backend-runtime';
import { StorageRuntimeProvider } from '../storage-runtime';
declare global {
interface Jobs {
@@ -15,7 +15,7 @@ export class BlobUploadCleanupJob {
private readonly logger = new Logger(BlobUploadCleanupJob.name);
constructor(
private readonly rt: BackendRuntimeProvider,
private readonly rt: StorageRuntimeProvider,
private readonly event: EventBus,
private readonly queue: JobQueue
) {}
packages/backend/server/src/core/storage/r2-proxy.ts
+20 -45
View File
@@ -7,19 +7,16 @@ import {
BlobInvalid,
CallMetric,
Config,
OnEvent,
PROXY_MULTIPART_PATH,
PROXY_UPLOAD_PATH,
type R2StorageConfig,
STORAGE_PROXY_ROOT,
StorageProviderConfig,
StorageProviderFactory,
type StorageProviderConfig,
toBuffer,
} from '../../base';
import {
R2StorageConfig,
R2StorageProvider,
} from '../../base/storage/providers/r2';
import { Models } from '../../models';
import { Public } from '../auth/guard';
import { StorageRuntimeProvider } from '../storage-runtime';
import { MULTIPART_PART_SIZE } from './constants';
type R2BlobStorageConfig = StorageProviderConfig & {
@@ -37,21 +34,13 @@ type R2Config = {
@Controller(STORAGE_PROXY_ROOT)
export class R2UploadController {
private readonly logger = new Logger(R2UploadController.name);
private provider: R2StorageProvider | null = null;
constructor(
private readonly config: Config,
private readonly models: Models,
private readonly storageFactory: StorageProviderFactory
private readonly rt: StorageRuntimeProvider
) {}
@OnEvent('config.changed')
onConfigChanged(event: Events['config.changed']) {
if (event.updates.storages?.blob?.storage) {
this.provider = null;
}
}
private getR2Config(): R2Config {
const storage = this.config.storages.blob.storage as StorageProviderConfig;
if (storage.provider !== 'cloudflare-r2') {
@@ -69,16 +58,6 @@ export class R2UploadController {
return { storage: storage as R2BlobStorageConfig, signKey };
}
private getProvider(storage: R2BlobStorageConfig) {
if (!this.provider) {
const candidate = this.storageFactory.create(storage);
if (candidate instanceof R2StorageProvider) {
this.provider = candidate;
}
}
return this.provider;
}
private sign(canonical: string, signKey: string) {
return createHmac('sha256', signKey).update(canonical).digest('base64');
}
@@ -173,7 +152,7 @@ export class R2UploadController {
@Put('upload')
@CallMetric('controllers', 'r2_proxy_upload')
async upload(@Req() req: Request, @Res() res: Response) {
const { storage, signKey } = this.getR2Config();
const { signKey } = this.getR2Config();
const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId');
const key = this.expectString(req.query.key, 'key');
@@ -229,16 +208,16 @@ export class R2UploadController {
throw new BlobInvalid('Mime type mismatch');
}
const provider = this.getProvider(storage);
if (!provider) {
throw new BlobInvalid('R2 provider is not available');
}
try {
await provider.proxyPutObject(`${workspaceId}/${key}`, req, {
contentType: mime,
contentLength,
});
await this.rt.putObject(
'blob',
`${workspaceId}/${key}`,
await toBuffer(req),
{
contentType: mime,
contentLength,
}
);
} catch (error) {
this.logger.error('Failed to proxy upload', error as Error);
throw new BlobInvalid('Upload failed');
@@ -251,7 +230,7 @@ export class R2UploadController {
@Put('multipart')
@CallMetric('controllers', 'r2_proxy_multipart')
async uploadPart(@Req() req: Request, @Res() res: Response) {
const { storage, signKey } = this.getR2Config();
const { signKey } = this.getR2Config();
const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId');
const key = this.expectString(req.query.key, 'key');
@@ -305,18 +284,14 @@ export class R2UploadController {
throw new BlobInvalid('Part size exceeds upload metadata');
}
const provider = this.getProvider(storage);
if (!provider) {
throw new BlobInvalid('R2 provider is not available');
}
try {
const etag = await provider.proxyUploadPart(
const etag = await this.rt.proxyUploadPart(
'blob',
`${workspaceId}/${key}`,
uploadId,
partNumber,
req,
{ contentLength }
await toBuffer(req),
contentLength
);
if (etag) {
res.setHeader('etag', etag);
packages/backend/server/src/core/storage/wrappers/avatar.ts
+7 -29
View File
@@ -1,21 +1,11 @@
import { Injectable } from '@nestjs/common';
import type {
BlobInputType,
PutObjectMetadata,
StorageProvider,
} from '../../../base';
import {
Config,
OnEvent,
StorageProviderFactory,
URLHelper,
} from '../../../base';
import type { BlobInputType, PutObjectMetadata } from '../../../base';
import { Config, OnEvent, toBuffer, URLHelper } from '../../../base';
import { StorageRuntimeProvider } from '../../storage-runtime';
@Injectable()
export class AvatarStorage {
private provider!: StorageProvider;
get config() {
return this.AFFiNEConfig.storages.avatar;
}
@@ -23,23 +13,11 @@ export class AvatarStorage {
constructor(
private readonly AFFiNEConfig: Config,
private readonly url: URLHelper,
private readonly storageFactory: StorageProviderFactory
private readonly rt: StorageRuntimeProvider
) {}
@OnEvent('config.init')
async onConfigInit() {
this.provider = this.storageFactory.create(this.config.storage);
}
@OnEvent('config.changed')
async onConfigChanged(event: Events['config.changed']) {
if (event.updates.storages?.avatar?.storage) {
this.provider = this.storageFactory.create(this.config.storage);
}
}
async put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) {
await this.provider.put(key, blob, metadata);
await this.rt.putObject('avatar', key, await toBuffer(blob), metadata);
let link = this.config.publicPath + key;
if (link.startsWith('/')) {
@@ -50,11 +28,11 @@ export class AvatarStorage {
}
get(key: string) {
return this.provider.get(key);
return this.rt.getObject('avatar', key);
}
delete(link: string) {
return this.provider.delete(link.split('/').pop() as string);
return this.rt.deleteObject('avatar', link.split('/').pop() as string);
}
@OnEvent('user.deleted')
packages/backend/server/src/core/storage/wrappers/blob.ts
+199 -102
View File
@@ -1,20 +1,24 @@
import { createHash } from 'node:crypto';
import { createHmac } from 'node:crypto';
import { Injectable, Logger } from '@nestjs/common';
import {
autoMetadata,
type BlobOutputType,
Config,
EventBus,
type GetObjectMetadata,
OnEvent,
PutObjectMetadata,
type StorageProvider,
StorageProviderFactory,
PROXY_MULTIPART_PATH,
PROXY_UPLOAD_PATH,
type PutObjectMetadata,
type R2StorageConfig,
SIGNED_URL_EXPIRED,
type StorageProviderConfig,
URLHelper,
} from '../../../base';
import { Models } from '../../../models';
import { BackendRuntimeProvider } from '../../backend-runtime';
import type { StorageProviderCapabilities } from '../../../native';
import { StorageRuntimeProvider } from '../../storage-runtime';
declare global {
interface Events {
@@ -36,52 +40,70 @@ type BlobCompleteResult =
| 'not_found'
| 'size_mismatch'
| 'mime_mismatch'
| 'checksum_mismatch';
| 'checksum_mismatch'
| 'size_too_large';
};
type BlobGetResult = {
redirectUrl?: string;
body?: BlobOutputType;
metadata?: GetObjectMetadata;
};
@Injectable()
export class WorkspaceBlobStorage {
private readonly logger = new Logger(WorkspaceBlobStorage.name);
private provider!: StorageProvider;
get config() {
return this.AFFiNEConfig.storages.blob;
}
constructor(
private readonly AFFiNEConfig: Config,
private readonly event: EventBus,
private readonly storageFactory: StorageProviderFactory,
private readonly models: Models,
private readonly url: URLHelper,
private readonly rt: BackendRuntimeProvider
private readonly rt: StorageRuntimeProvider,
private readonly config: Config
) {}
@OnEvent('config.init')
async onConfigInit() {
this.provider = this.storageFactory.create(this.config.storage);
}
@OnEvent('config.changed')
async onConfigChanged(event: Events['config.changed']) {
if (event.updates.storages?.blob?.storage) {
this.provider = this.storageFactory.create(this.config.storage);
}
}
async put(workspaceId: string, key: string, blob: Buffer) {
const meta: PutObjectMetadata = autoMetadata(blob);
await this.provider.put(`${workspaceId}/${key}`, blob, meta);
const metadata = await this.rt.putObject(
'blob',
`${workspaceId}/${key}`,
blob
);
await this.upsert(workspaceId, key, {
contentType: meta.contentType ?? 'application/octet-stream',
contentLength: blob.length,
lastModified: new Date(),
contentType: metadata.contentType,
contentLength: metadata.contentLength,
lastModified: metadata.lastModified,
});
}
async get(workspaceId: string, key: string, signedUrl?: boolean) {
return this.provider.get(`${workspaceId}/${key}`, signedUrl);
async capabilities(): Promise<StorageProviderCapabilities> {
const capabilities = await this.rt.providerCapabilities('blob');
if (!this.r2ProxyConfig()) {
return capabilities;
}
return {
...capabilities,
presignPut: true,
multipartDirect: true,
proxyUpload: true,
serverMediatedOnly: false,
};
}
async get(
workspaceId: string,
key: string,
signedUrl?: boolean
): Promise<BlobGetResult> {
if (signedUrl) {
const presigned = await this.rt.presignGet(
'blob',
`${workspaceId}/${key}`
);
if (presigned) {
return { redirectUrl: presigned.url };
}
}
return this.rt.getObject('blob', `${workspaceId}/${key}`);
}
async presignPut(
@@ -89,7 +111,11 @@ export class WorkspaceBlobStorage {
key: string,
metadata?: PutObjectMetadata
) {
return this.provider.presignPut?.(`${workspaceId}/${key}`, metadata);
const proxy = this.r2ProxyConfig();
if (proxy) {
return this.createProxyUploadUrl(workspaceId, key, metadata, proxy);
}
return this.rt.presignPut('blob', `${workspaceId}/${key}`, metadata);
}
async createMultipartUpload(
@@ -97,7 +123,8 @@ export class WorkspaceBlobStorage {
key: string,
metadata?: PutObjectMetadata
) {
return this.provider.createMultipartUpload?.(
return this.rt.createMultipartUpload(
'blob',
`${workspaceId}/${key}`,
metadata
);
@@ -109,7 +136,18 @@ export class WorkspaceBlobStorage {
uploadId: string,
partNumber: number
) {
return this.provider.presignUploadPart?.(
const proxy = this.r2ProxyConfig();
if (proxy) {
return this.createProxyMultipartUrl(
workspaceId,
key,
uploadId,
partNumber,
proxy
);
}
return this.rt.presignUploadPart(
'blob',
`${workspaceId}/${key}`,
uploadId,
partNumber
@@ -121,7 +159,8 @@ export class WorkspaceBlobStorage {
key: string,
uploadId: string
) {
return this.provider.listMultipartUploadParts?.(
return this.rt.listMultipartUploadParts(
'blob',
`${workspaceId}/${key}`,
uploadId
);
@@ -133,16 +172,12 @@ export class WorkspaceBlobStorage {
uploadId: string,
parts: { partNumber: number; etag: string }[]
) {
if (!this.provider.completeMultipartUpload) {
return false;
}
await this.provider.completeMultipartUpload(
return await this.rt.completeMultipartUpload(
'blob',
`${workspaceId}/${key}`,
uploadId,
parts
);
return true;
}
async abortMultipartUpload(
@@ -150,16 +185,15 @@ export class WorkspaceBlobStorage {
key: string,
uploadId: string
) {
if (!this.provider.abortMultipartUpload) {
return false;
}
await this.provider.abortMultipartUpload(`${workspaceId}/${key}`, uploadId);
return true;
return await this.rt.abortMultipartUpload(
'blob',
`${workspaceId}/${key}`,
uploadId
);
}
async head(workspaceId: string, key: string) {
return this.provider.head(`${workspaceId}/${key}`);
return this.rt.headObject('blob', `${workspaceId}/${key}`);
}
async complete(
@@ -167,57 +201,28 @@ export class WorkspaceBlobStorage {
key: string,
expected: { size: number; mime: string }
): Promise<BlobCompleteResult> {
const metadata = await this.head(workspaceId, key);
if (!metadata) {
return { ok: false, reason: 'not_found' };
}
if (metadata.contentLength !== expected.size) {
return { ok: false, reason: 'size_mismatch' };
}
if (expected.mime && metadata.contentType !== expected.mime) {
return { ok: false, reason: 'mime_mismatch' };
}
const object = await this.provider.get(`${workspaceId}/${key}`);
if (!object.body) {
return { ok: false, reason: 'not_found' };
}
const checksum = createHash('sha256');
try {
for await (const chunk of object.body) {
checksum.update(chunk as Buffer);
}
} catch (e) {
this.logger.error('failed to read blob for checksum verification', e);
return { ok: false, reason: 'checksum_mismatch' };
}
const base64 = checksum.digest('base64');
const base64urlWithPadding = base64.replace(/\+/g, '-').replace(/\//g, '_');
if (base64urlWithPadding !== key) {
try {
await this.provider.delete(`${workspaceId}/${key}`);
} catch (e) {
// never throw
this.logger.error('failed to delete invalid blob', e);
}
return { ok: false, reason: 'checksum_mismatch' };
}
await this.models.blob.upsert({
const result = await this.rt.completeWorkspaceBlobUpload(
workspaceId,
key,
mime: metadata.contentType,
size: metadata.contentLength,
status: 'completed',
uploadId: null,
});
return { ok: true, metadata };
expected
);
if (!result.ok) {
return {
ok: false,
reason: (result.reason ?? 'checksum_mismatch') as Exclude<
BlobCompleteResult,
{ ok: true }
>['reason'],
};
}
return {
ok: true,
metadata: {
contentType: result.contentType ?? 'application/octet-stream',
contentLength: result.contentLength ?? expected.size,
lastModified: new Date(result.lastModifiedMs ?? Date.now()),
},
};
}
async list(workspaceId: string) {
@@ -226,7 +231,7 @@ export class WorkspaceBlobStorage {
async delete(workspaceId: string, key: string, permanently = false) {
if (permanently) {
await this.provider.delete(`${workspaceId}/${key}`);
await this.rt.deleteObject('blob', `${workspaceId}/${key}`);
}
await this.models.blob.delete(workspaceId, key, permanently);
if (!permanently) {
@@ -297,4 +302,96 @@ export class WorkspaceBlobStorage {
}: Events['workspace.blob.delete']) {
await this.delete(workspaceId, key, true);
}
private r2ProxyConfig() {
const storage = this.config.storages.blob.storage as StorageProviderConfig;
if (storage.provider !== 'cloudflare-r2') {
return;
}
const r2 = storage.config as R2StorageConfig;
const usePresignedURL = r2.usePresignedURL;
if (
!usePresignedURL?.enabled ||
!usePresignedURL.urlPrefix ||
!usePresignedURL.signKey
) {
return;
}
return { signKey: usePresignedURL.signKey };
}
private signProxy(
path: string,
canonicalFields: (string | number | undefined)[],
exp: number,
signKey: string
) {
const canonical = [
path,
...canonicalFields.map(field =>
field === undefined ? '' : field.toString()
),
exp.toString(),
].join('\n');
return `${exp}-${createHmac('sha256', signKey).update(canonical).digest('base64')}`;
}
private createProxyUploadUrl(
workspaceId: string,
key: string,
metadata: PutObjectMetadata | undefined,
proxy: { signKey: string }
) {
const contentType = metadata?.contentType ?? 'application/octet-stream';
const contentLength = metadata?.contentLength;
const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000);
const exp = Math.floor(expiresAt.getTime() / 1000);
const token = this.signProxy(
PROXY_UPLOAD_PATH,
[workspaceId, key, contentType, contentLength],
exp,
proxy.signKey
);
return {
url: this.url.link(PROXY_UPLOAD_PATH, {
workspaceId,
key,
contentType,
contentLength,
exp,
token,
}),
headers: {},
expiresAt,
};
}
private createProxyMultipartUrl(
workspaceId: string,
key: string,
uploadId: string,
partNumber: number,
proxy: { signKey: string }
) {
const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000);
const exp = Math.floor(expiresAt.getTime() / 1000);
const token = this.signProxy(
PROXY_MULTIPART_PATH,
[workspaceId, key, uploadId, partNumber],
exp,
proxy.signKey
);
return {
url: this.url.link(PROXY_MULTIPART_PATH, {
workspaceId,
key,
uploadId,
partNumber,
exp,
token,
}),
headers: {},
expiresAt,
};
}
}
packages/backend/server/src/core/storage/wrappers/comment-attachment.ts
+25 -43
View File
@@ -1,16 +1,11 @@
import { Injectable, Logger } from '@nestjs/common';
import {
autoMetadata,
Config,
EventBus,
metrics,
OnEvent,
type StorageProvider,
StorageProviderFactory,
URLHelper,
} from '../../../base';
import { EventBus, metrics, OnEvent, URLHelper } from '../../../base';
import { Models } from '../../../models';
import {
type StorageRuntimeGetObjectResult,
StorageRuntimeProvider,
} from '../../storage-runtime';
declare global {
interface Events {
@@ -25,32 +20,14 @@ declare global {
@Injectable()
export class CommentAttachmentStorage {
private readonly logger = new Logger(CommentAttachmentStorage.name);
private provider!: StorageProvider;
get config() {
return this.AFFiNEConfig.storages.blob;
}
constructor(
private readonly AFFiNEConfig: Config,
private readonly event: EventBus,
private readonly storageFactory: StorageProviderFactory,
private readonly models: Models,
private readonly url: URLHelper
private readonly url: URLHelper,
private readonly rt: StorageRuntimeProvider
) {}
@OnEvent('config.init')
async onConfigInit() {
this.provider = this.storageFactory.create(this.config.storage);
}
@OnEvent('config.changed')
async onConfigChanged(event: Events['config.changed']) {
if (event.updates.storages?.blob?.storage) {
this.provider = this.storageFactory.create(this.config.storage);
}
}
private storageKey(workspaceId: string, docId: string, key: string) {
return `comment-attachments/${workspaceId}/${docId}/${key}`;
}
@@ -63,15 +40,13 @@ export class CommentAttachmentStorage {
blob: Buffer,
userId: string
) {
const meta = autoMetadata(blob);
await this.provider.put(
const metadata = await this.rt.putObject(
'blob',
this.storageKey(workspaceId, docId, key),
blob,
meta
blob
);
const mime = meta.contentType ?? 'application/octet-stream';
const size = blob.length;
const mime = metadata.contentType;
const size = metadata.contentLength;
await this.models.commentAttachment.upsert({
workspaceId,
docId,
@@ -94,15 +69,22 @@ export class CommentAttachmentStorage {
docId: string,
key: string,
signedUrl?: boolean
) {
return await this.provider.get(
this.storageKey(workspaceId, docId, key),
signedUrl
);
): Promise<StorageRuntimeGetObjectResult> {
const storageKey = this.storageKey(workspaceId, docId, key);
if (signedUrl) {
const presigned = await this.rt.presignGet('blob', storageKey);
if (presigned) {
return { redirectUrl: presigned.url };
}
}
return await this.rt.getObject('blob', storageKey);
}
async delete(workspaceId: string, docId: string, key: string) {
await this.provider.delete(this.storageKey(workspaceId, docId, key));
await this.rt.deleteObject(
'blob',
this.storageKey(workspaceId, docId, key)
);
await this.models.commentAttachment.delete(workspaceId, docId, key);
this.logger.log(
`deleted comment attachment ${workspaceId}/${docId}/${key}`
packages/backend/server/src/core/workspaces/resolvers/blob.ts
+4 -3
View File
@@ -248,11 +248,12 @@ export class WorkspaceBlobResolver {
}
const metadata = { contentType: mime, contentLength: size };
const capabilities = await this.storage.capabilities();
let init: BlobUploadInit | null = null;
let uploadIdForRecord: string | null = null;
// try to resume multipart uploads
if (record && record.uploadId) {
if (capabilities.multipartDirect && record && record.uploadId) {
const uploadedParts = await this.storage.listMultipartUploadParts(
workspaceId,
key,
@@ -270,7 +271,7 @@ export class WorkspaceBlobResolver {
}
}
if (size >= MULTIPART_THRESHOLD) {
if (capabilities.multipartDirect && size >= MULTIPART_THRESHOLD) {
const multipart = await this.storage.createMultipartUpload(
workspaceId,
key,
@@ -289,7 +290,7 @@ export class WorkspaceBlobResolver {
}
}
if (!init) {
if (!init && capabilities.presignPut) {
const presigned = await this.storage.presignPut(
workspaceId,
key,
packages/backend/server/src/native.ts
+5 -2
View File
@@ -60,7 +60,6 @@ import serverNativeModule, {
type RuntimeObjectGetResult,
type RuntimeObjectListEntry,
type RuntimeObjectMetadata,
type RuntimeObjectStorageHealth,
type RuntimeObjectStoragePutOptions,
type RuntimePresignedObjectRequest,
type RuntimeVerificationTokenRecord,
@@ -68,6 +67,8 @@ import serverNativeModule, {
type RuntimeWorkspaceStatsDailyRecalibrationResult,
type SafeFetchRequest,
type SafeFetchResponse,
type StorageProviderCapabilities,
type StorageRuntimeHealth,
type Tokenizer,
} from '@affine/server-native';
@@ -109,7 +110,6 @@ export type {
RuntimeObjectGetResult,
RuntimeObjectListEntry,
RuntimeObjectMetadata,
RuntimeObjectStorageHealth,
RuntimeObjectStoragePutOptions,
RuntimePresignedObjectRequest,
RuntimeVerificationTokenRecord,
@@ -117,6 +117,8 @@ export type {
RuntimeWorkspaceStatsDailyRecalibrationResult,
SafeFetchRequest,
SafeFetchResponse,
StorageProviderCapabilities,
StorageRuntimeHealth,
};
export type ActionEventType =
@@ -274,6 +276,7 @@ export const AFFINE_PRO_PUBLIC_KEY = serverNativeModule.AFFINE_PRO_PUBLIC_KEY;
export const AFFINE_PRO_LICENSE_AES_KEY =
serverNativeModule.AFFINE_PRO_LICENSE_AES_KEY;
export const BackendRuntime = serverNativeModule.BackendRuntime;
export const StorageRuntime = serverNativeModule.StorageRuntime;
export type PermissionWorkspaceRole = 'external' | 'member' | 'admin' | 'owner';
export type PermissionDocRole =
packages/backend/server/src/plugins/copilot/storage.ts
+19 -25
View File
@@ -6,43 +6,29 @@ import {
type BlobInputType,
BlobQuotaExceeded,
CallMetric,
Config,
type FileUpload,
OneMB,
OnEvent,
readBuffer,
type StorageProvider,
StorageProviderFactory,
toBuffer,
URLHelper,
} from '../../base';
import { QuotaService } from '../../core/quota';
import {
type StorageRuntimeGetObjectResult,
StorageRuntimeProvider,
} from '../../core/storage-runtime';
import { fetchRemoteAttachment } from '../../native';
const REMOTE_BLOB_MAX_BYTES = 20 * OneMB;
@Injectable()
export class CopilotStorage {
public provider!: StorageProvider;
constructor(
private readonly config: Config,
private readonly url: URLHelper,
private readonly storageFactory: StorageProviderFactory,
private readonly rt: StorageRuntimeProvider,
private readonly quota: QuotaService
) {}
@OnEvent('config.init')
async onConfigInit() {
this.provider = this.storageFactory.create(this.config.copilot.storage);
}
@OnEvent('config.changed')
async onConfigChanged(event: Events['config.changed']) {
if (event.updates?.copilot?.storage) {
this.provider = this.storageFactory.create(this.config.copilot.storage);
}
}
@CallMetric('ai', 'blob_put')
async put(
userId: string,
@@ -52,10 +38,11 @@ export class CopilotStorage {
mimeType = 'image/png'
) {
const name = `${userId}/${workspaceId}/${key}`;
await this.provider.put(name, blob);
const buffer = await toBuffer(blob);
await this.rt.putObject('copilot', name, buffer);
if (!env.prod) {
// return image base64url for dev environment
return `data:${mimeType};base64,${blob.toString('base64')}`;
return `data:${mimeType};base64,${buffer.toString('base64')}`;
}
return this.url.link(`/api/copilot/blob/${name}`);
}
@@ -66,13 +53,20 @@ export class CopilotStorage {
workspaceId: string,
key: string,
signedUrl?: boolean
) {
return this.provider.get(`${userId}/${workspaceId}/${key}`, signedUrl);
): Promise<StorageRuntimeGetObjectResult> {
const name = `${userId}/${workspaceId}/${key}`;
if (signedUrl) {
const presigned = await this.rt.presignGet('copilot', name);
if (presigned) {
return { redirectUrl: presigned.url };
}
}
return this.rt.getObject('copilot', name);
}
@CallMetric('ai', 'blob_delete')
async delete(userId: string, workspaceId: string, key: string) {
await this.provider.delete(`${userId}/${workspaceId}/${key}`);
await this.rt.deleteObject('copilot', `${userId}/${workspaceId}/${key}`);
}
@CallMetric('ai', 'blob_upload')