mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-08-02 01:49:51 +08:00
feat(server): impl storage runtime (#15181)
#### PR Dependency Tree * **PR #15181** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added an additional storage backend option: asset-pack based storage (provider for avatar, blob, and copilot). * Introduced a dedicated storage runtime with provider capability reporting and expanded object operations (put/head/get/list/delete), including presigned and multipart flows where supported. * Cloudflare R2 `jurisdiction` now uses an explicit default when omitted. * **Bug Fixes** * Broadened avatar access to allow both fs and asset-pack providers. * Improved workspace blob upload completion validation and handling when stored objects are missing or mismatched. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
@@ -1,130 +0,0 @@
|
||||
import { promises as fs } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import test from 'ava';
|
||||
import { getStreamAsBuffer } from 'get-stream';
|
||||
|
||||
import { ListObjectsMetadata } from '../providers';
|
||||
import { FsStorageProvider } from '../providers/fs';
|
||||
|
||||
const config = {
|
||||
path: join(process.cwd(), 'node_modules', '.cache/affine-test-storage'),
|
||||
};
|
||||
|
||||
function createProvider() {
|
||||
return new FsStorageProvider(
|
||||
config,
|
||||
'test' + Math.random().toString(16).substring(2, 8)
|
||||
);
|
||||
}
|
||||
|
||||
function keys(list: ListObjectsMetadata[]) {
|
||||
return list.map(i => i.key);
|
||||
}
|
||||
|
||||
async function randomPut(
|
||||
provider: FsStorageProvider,
|
||||
prefix = ''
|
||||
): Promise<string> {
|
||||
const key = prefix + 'test-key-' + Math.random().toString(16).substring(2, 8);
|
||||
const body = Buffer.from(key);
|
||||
await provider.put(key, body);
|
||||
return key;
|
||||
}
|
||||
|
||||
test.after.always(() => {
|
||||
fs.rm(config.path, { recursive: true }).catch(console.error);
|
||||
});
|
||||
|
||||
test('put & get', async t => {
|
||||
const provider = createProvider();
|
||||
const key = 'testKey';
|
||||
const body = Buffer.from('testBody');
|
||||
await provider.put(key, body);
|
||||
|
||||
const result = await provider.get(key);
|
||||
|
||||
t.deepEqual(await getStreamAsBuffer(result.body!), body);
|
||||
t.is(result.metadata?.contentLength, body.length);
|
||||
});
|
||||
|
||||
test('list - one level', async t => {
|
||||
const provider = createProvider();
|
||||
const list = await Promise.all(
|
||||
Array.from({ length: 100 }).map(() => randomPut(provider))
|
||||
);
|
||||
list.sort();
|
||||
// random order, use set
|
||||
const result = await provider.list();
|
||||
t.deepEqual(keys(result), list);
|
||||
|
||||
const result2 = await provider.list('test-key');
|
||||
t.deepEqual(keys(result2), list);
|
||||
|
||||
const result3 = await provider.list('testKey');
|
||||
t.is(result3.length, 0);
|
||||
});
|
||||
|
||||
test('list recursively', async t => {
|
||||
const provider = createProvider();
|
||||
|
||||
await Promise.all([
|
||||
Promise.all(Array.from({ length: 10 }).map(() => randomPut(provider))),
|
||||
Promise.all(
|
||||
Array.from({ length: 10 }).map(() => randomPut(provider, 'a/'))
|
||||
),
|
||||
Promise.all(
|
||||
Array.from({ length: 10 }).map(() => randomPut(provider, 'a/b/'))
|
||||
),
|
||||
Promise.all(
|
||||
Array.from({ length: 10 }).map(() => randomPut(provider, 'a/b/t/'))
|
||||
),
|
||||
]);
|
||||
|
||||
const r1 = await provider.list();
|
||||
t.is(r1.length, 40);
|
||||
|
||||
// contains all `a/xxx` and `a/b/xxx` and `a/b/c/xxx`
|
||||
const r2 = await provider.list('a');
|
||||
t.is(r2.length, 30);
|
||||
|
||||
// contains only `a/b/xxx`
|
||||
const r3 = await provider.list('a/b');
|
||||
const r4 = await provider.list('a/b/');
|
||||
t.is(r3.length, 20);
|
||||
t.deepEqual(r3, r4);
|
||||
|
||||
// prefix is not ended with '/', it's open to all files and sub dirs
|
||||
// contains all `a/b/t/xxx` and `a/b/t{xxxx}`
|
||||
const r5 = await provider.list('a/b/t');
|
||||
|
||||
t.is(r5.length, 20);
|
||||
});
|
||||
|
||||
test('delete', async t => {
|
||||
const provider = createProvider();
|
||||
const key = 'testKey';
|
||||
const body = Buffer.from('testBody');
|
||||
await provider.put(key, body);
|
||||
|
||||
await provider.delete(key);
|
||||
|
||||
await t.throwsAsync(() => fs.access(join(config.path, provider.bucket, key)));
|
||||
});
|
||||
|
||||
test('rejects unsafe object keys', async t => {
|
||||
const provider = createProvider();
|
||||
|
||||
await t.throwsAsync(() => provider.put('../escape', Buffer.from('nope')));
|
||||
await t.throwsAsync(() => provider.get('nested/../escape'));
|
||||
await t.throwsAsync(() => provider.head('./escape'));
|
||||
t.throws(() => provider.delete('nested//escape'));
|
||||
});
|
||||
|
||||
test('rejects unsafe list prefixes', async t => {
|
||||
const provider = createProvider();
|
||||
|
||||
await t.throwsAsync(() => provider.list('../escape'));
|
||||
await t.throwsAsync(() => provider.list('nested/../../escape'));
|
||||
await t.throwsAsync(() => provider.list('/absolute'));
|
||||
});
|
||||
@@ -1,82 +0,0 @@
|
||||
import test from 'ava';
|
||||
|
||||
import { R2StorageProvider } from '../providers/r2';
|
||||
|
||||
function endpointOf(provider: R2StorageProvider) {
|
||||
return provider.endpointUrl;
|
||||
}
|
||||
|
||||
test('R2 provider should use account endpoint by default', t => {
|
||||
const provider = new R2StorageProvider(
|
||||
{
|
||||
accountId: 'test-account',
|
||||
region: 'auto',
|
||||
credentials: {
|
||||
accessKeyId: 'test',
|
||||
secretAccessKey: 'test',
|
||||
},
|
||||
},
|
||||
'test-bucket'
|
||||
);
|
||||
|
||||
t.is(
|
||||
endpointOf(provider),
|
||||
'https://test-account.r2.cloudflarestorage.com/test-bucket'
|
||||
);
|
||||
});
|
||||
|
||||
test('R2 provider should append jurisdiction suffix for EU buckets', t => {
|
||||
const provider = new R2StorageProvider(
|
||||
{
|
||||
accountId: 'test-account',
|
||||
jurisdiction: 'eu',
|
||||
region: 'auto',
|
||||
credentials: {
|
||||
accessKeyId: 'test',
|
||||
secretAccessKey: 'test',
|
||||
},
|
||||
},
|
||||
'test-bucket'
|
||||
);
|
||||
|
||||
t.is(
|
||||
endpointOf(provider),
|
||||
'https://test-account.eu.r2.cloudflarestorage.com/test-bucket'
|
||||
);
|
||||
});
|
||||
|
||||
test('R2 provider should throw when accountId is missing', t => {
|
||||
t.throws(
|
||||
() =>
|
||||
new R2StorageProvider(
|
||||
{
|
||||
region: 'auto',
|
||||
credentials: {
|
||||
accessKeyId: 'test',
|
||||
secretAccessKey: 'test',
|
||||
},
|
||||
} as any,
|
||||
'test-bucket'
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
test('R2 provider should use default endpoint when jurisdiction is explicitly undefined', t => {
|
||||
const provider = new R2StorageProvider(
|
||||
{
|
||||
accountId: 'test-account',
|
||||
jurisdiction: undefined,
|
||||
region: 'auto',
|
||||
credentials: {
|
||||
accessKeyId: 'test',
|
||||
secretAccessKey: 'test',
|
||||
},
|
||||
},
|
||||
'test-bucket'
|
||||
);
|
||||
|
||||
t.is(
|
||||
endpointOf(provider),
|
||||
'https://test-account.r2.cloudflarestorage.com/test-bucket'
|
||||
);
|
||||
});
|
||||
@@ -1,49 +0,0 @@
|
||||
import { parseListPartsXml } from '@affine/s3-compat';
|
||||
import test from 'ava';
|
||||
|
||||
test('parseListPartsXml handles array parts and pagination', t => {
|
||||
const xml = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListPartsResult>
|
||||
<Bucket>test</Bucket>
|
||||
<Key>key</Key>
|
||||
<UploadId>upload-id</UploadId>
|
||||
<PartNumberMarker>0</PartNumberMarker>
|
||||
<NextPartNumberMarker>3</NextPartNumberMarker>
|
||||
<IsTruncated>true</IsTruncated>
|
||||
<Part>
|
||||
<PartNumber>1</PartNumber>
|
||||
<ETag>"etag-1"</ETag>
|
||||
</Part>
|
||||
<Part>
|
||||
<PartNumber>2</PartNumber>
|
||||
<ETag>etag-2</ETag>
|
||||
</Part>
|
||||
</ListPartsResult>`;
|
||||
|
||||
const result = parseListPartsXml(xml);
|
||||
t.deepEqual(result.parts, [
|
||||
{ partNumber: 1, etag: 'etag-1' },
|
||||
{ partNumber: 2, etag: 'etag-2' },
|
||||
]);
|
||||
t.true(result.isTruncated);
|
||||
t.is(result.nextPartNumberMarker, '3');
|
||||
});
|
||||
|
||||
test('parseListPartsXml handles single part', t => {
|
||||
const xml = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListPartsResult>
|
||||
<Bucket>test</Bucket>
|
||||
<Key>key</Key>
|
||||
<UploadId>upload-id</UploadId>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Part>
|
||||
<PartNumber>5</PartNumber>
|
||||
<ETag>"etag-5"</ETag>
|
||||
</Part>
|
||||
</ListPartsResult>`;
|
||||
|
||||
const result = parseListPartsXml(xml);
|
||||
t.deepEqual(result.parts, [{ partNumber: 5, etag: 'etag-5' }]);
|
||||
t.false(result.isTruncated);
|
||||
t.is(result.nextPartNumberMarker, undefined);
|
||||
});
|
||||
@@ -1,90 +0,0 @@
|
||||
import test from 'ava';
|
||||
|
||||
import { S3StorageProvider } from '../providers/s3';
|
||||
import { SIGNED_URL_EXPIRED } from '../providers/utils';
|
||||
|
||||
const config = {
|
||||
region: 'us-east-1',
|
||||
endpoint: 'https://s3.us-east-1.amazonaws.com',
|
||||
credentials: {
|
||||
accessKeyId: 'test',
|
||||
secretAccessKey: 'test',
|
||||
},
|
||||
};
|
||||
|
||||
function createProvider() {
|
||||
return new S3StorageProvider(config, 'test-bucket');
|
||||
}
|
||||
|
||||
test('presignPut should return url and headers', async t => {
|
||||
const provider = createProvider();
|
||||
const result = await provider.presignPut('key', {
|
||||
contentType: 'text/plain',
|
||||
});
|
||||
|
||||
t.truthy(result);
|
||||
t.true(result!.url.length > 0);
|
||||
t.true(result!.url.includes('X-Amz-Algorithm=AWS4-HMAC-SHA256'));
|
||||
t.true(result!.url.includes('X-Amz-SignedHeaders='));
|
||||
t.true(result!.url.includes('content-type'));
|
||||
t.deepEqual(result!.headers, { 'Content-Type': 'text/plain' });
|
||||
const now = Date.now();
|
||||
t.true(result!.expiresAt.getTime() >= now + SIGNED_URL_EXPIRED * 1000 - 2000);
|
||||
t.true(result!.expiresAt.getTime() <= now + SIGNED_URL_EXPIRED * 1000 + 2000);
|
||||
});
|
||||
|
||||
test('presignUploadPart should return url', async t => {
|
||||
const provider = createProvider();
|
||||
const result = await provider.presignUploadPart('key', 'upload-1', 3);
|
||||
|
||||
t.truthy(result);
|
||||
t.true(result!.url.length > 0);
|
||||
t.true(result!.url.includes('X-Amz-Algorithm=AWS4-HMAC-SHA256'));
|
||||
});
|
||||
|
||||
test('createMultipartUpload should return uploadId', async t => {
|
||||
const provider = createProvider();
|
||||
let receivedKey: string | undefined;
|
||||
let receivedMeta: any;
|
||||
(provider as any).client = {
|
||||
createMultipartUpload: async (key: string, meta: any) => {
|
||||
receivedKey = key;
|
||||
receivedMeta = meta;
|
||||
return { uploadId: 'upload-1' };
|
||||
},
|
||||
};
|
||||
|
||||
const now = Date.now();
|
||||
const result = await provider.createMultipartUpload('key', {
|
||||
contentType: 'text/plain',
|
||||
});
|
||||
|
||||
t.is(result?.uploadId, 'upload-1');
|
||||
t.true(result!.expiresAt.getTime() >= now + SIGNED_URL_EXPIRED * 1000 - 2000);
|
||||
t.true(result!.expiresAt.getTime() <= now + SIGNED_URL_EXPIRED * 1000 + 2000);
|
||||
t.is(receivedKey, 'key');
|
||||
t.is(receivedMeta.contentType, 'text/plain');
|
||||
});
|
||||
|
||||
test('completeMultipartUpload should order parts', async t => {
|
||||
const provider = createProvider();
|
||||
let receivedParts: any;
|
||||
(provider as any).client = {
|
||||
completeMultipartUpload: async (
|
||||
_key: string,
|
||||
_uploadId: string,
|
||||
parts: any
|
||||
) => {
|
||||
receivedParts = parts;
|
||||
},
|
||||
};
|
||||
|
||||
await provider.completeMultipartUpload('key', 'upload-1', [
|
||||
{ partNumber: 2, etag: 'b' },
|
||||
{ partNumber: 1, etag: 'a' },
|
||||
]);
|
||||
t.deepEqual(receivedParts, [
|
||||
{ partNumber: 1, etag: 'a' },
|
||||
{ partNumber: 2, etag: 'b' },
|
||||
]);
|
||||
});
|
||||
@@ -1,20 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
StorageProvider,
|
||||
StorageProviderConfig,
|
||||
StorageProviders,
|
||||
} from './providers';
|
||||
|
||||
@Injectable()
|
||||
export class StorageProviderFactory {
|
||||
create(config: StorageProviderConfig): StorageProvider {
|
||||
const Provider = StorageProviders[config.provider];
|
||||
|
||||
if (!Provider) {
|
||||
throw new Error(`Unknown storage provider type: ${config.provider}`);
|
||||
}
|
||||
|
||||
return new Provider(config.config, config.bucket);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,3 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
|
||||
import { StorageProviderFactory } from './factory';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [StorageProviderFactory],
|
||||
exports: [StorageProviderFactory],
|
||||
})
|
||||
export class StorageProviderModule {}
|
||||
export { StorageProviderFactory } from './factory';
|
||||
export * from './providers';
|
||||
export type * from './types';
|
||||
export * from './utils';
|
||||
|
||||
@@ -1,318 +0,0 @@
|
||||
import {
|
||||
accessSync,
|
||||
constants,
|
||||
createReadStream,
|
||||
Dirent,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { join, parse } from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
BlobInputType,
|
||||
GetObjectMetadata,
|
||||
ListObjectsMetadata,
|
||||
PutObjectMetadata,
|
||||
StorageProvider,
|
||||
} from './provider';
|
||||
import { autoMetadata, toBuffer } from './utils';
|
||||
|
||||
function normalizeStorageKey(key: string): string {
|
||||
const normalized = key.replaceAll('\\', '/');
|
||||
const segments = normalized.split('/');
|
||||
|
||||
if (
|
||||
!normalized ||
|
||||
normalized.startsWith('/') ||
|
||||
segments.some(segment => !segment || segment === '.' || segment === '..')
|
||||
) {
|
||||
throw new Error(`Invalid storage key: ${key}`);
|
||||
}
|
||||
|
||||
return segments.join('/');
|
||||
}
|
||||
|
||||
function normalizeStoragePrefix(prefix: string): string {
|
||||
const normalized = prefix.replaceAll('\\', '/');
|
||||
if (!normalized) {
|
||||
return normalized;
|
||||
}
|
||||
if (normalized.startsWith('/')) {
|
||||
throw new Error(`Invalid storage prefix: ${prefix}`);
|
||||
}
|
||||
|
||||
const segments = normalized.split('/');
|
||||
const lastSegment = segments.pop();
|
||||
|
||||
if (
|
||||
lastSegment === undefined ||
|
||||
segments.some(segment => !segment || segment === '.' || segment === '..') ||
|
||||
lastSegment === '.' ||
|
||||
lastSegment === '..'
|
||||
) {
|
||||
throw new Error(`Invalid storage prefix: ${prefix}`);
|
||||
}
|
||||
|
||||
if (lastSegment === '') {
|
||||
return `${segments.join('/')}/`;
|
||||
}
|
||||
|
||||
return [...segments, lastSegment].join('/');
|
||||
}
|
||||
|
||||
export interface FsStorageConfig {
|
||||
path: string;
|
||||
}
|
||||
|
||||
export class FsStorageProvider implements StorageProvider {
|
||||
private readonly path: string;
|
||||
private readonly logger: Logger;
|
||||
|
||||
readonly type = 'fs';
|
||||
|
||||
constructor(
|
||||
config: FsStorageConfig,
|
||||
public readonly bucket: string
|
||||
) {
|
||||
this.path = config.path.startsWith('~/')
|
||||
? join(homedir(), config.path.slice(2), bucket)
|
||||
: join(config.path, bucket);
|
||||
this.ensureAvailability();
|
||||
|
||||
this.logger = new Logger(`${FsStorageProvider.name}:${bucket}`);
|
||||
}
|
||||
|
||||
async put(
|
||||
key: string,
|
||||
body: BlobInputType,
|
||||
metadata: PutObjectMetadata = {}
|
||||
): Promise<void> {
|
||||
key = normalizeStorageKey(key);
|
||||
const blob = await toBuffer(body);
|
||||
|
||||
// write object
|
||||
this.writeObject(key, blob);
|
||||
// write metadata
|
||||
await this.writeMetadata(key, blob, metadata);
|
||||
this.logger.verbose(`Object \`${key}\` put`);
|
||||
}
|
||||
|
||||
async head(key: string) {
|
||||
key = normalizeStorageKey(key);
|
||||
const metadata = this.readMetadata(key);
|
||||
if (!metadata) {
|
||||
this.logger.verbose(`Object \`${key}\` not found`);
|
||||
return undefined;
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
async get(key: string): Promise<{
|
||||
body?: Readable;
|
||||
metadata?: GetObjectMetadata;
|
||||
}> {
|
||||
key = normalizeStorageKey(key);
|
||||
|
||||
try {
|
||||
const metadata = this.readMetadata(key);
|
||||
const stream = this.readObject(this.join(key));
|
||||
this.logger.verbose(`Read object \`${key}\``);
|
||||
return {
|
||||
body: stream,
|
||||
metadata,
|
||||
};
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to read object \`${key}\``, e);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async list(prefix?: string): Promise<ListObjectsMetadata[]> {
|
||||
// prefix cases:
|
||||
// - `undefined`: list all objects
|
||||
// - `a/b`: list objects under dir `a` with prefix `b`, `b` might be a dir under `a` as well.
|
||||
// - `a/b/` list objects under dir `a/b`
|
||||
|
||||
// read dir recursively and filter out '.metadata.json' files
|
||||
let dir = this.path;
|
||||
if (prefix) {
|
||||
prefix = normalizeStoragePrefix(prefix);
|
||||
const parts = prefix.split(/[/\\]/);
|
||||
// for prefix `a/b/c`, move `a/b` to dir and `c` to key prefix
|
||||
if (parts.length > 1) {
|
||||
dir = join(dir, ...parts.slice(0, -1));
|
||||
prefix = parts[parts.length - 1];
|
||||
}
|
||||
}
|
||||
|
||||
const results: ListObjectsMetadata[] = [];
|
||||
async function getFiles(dir: string, prefix?: string): Promise<void> {
|
||||
try {
|
||||
const entries: Dirent[] = readdirSync(dir, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const res = join(dir, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
if (!prefix || entry.name.startsWith(prefix)) {
|
||||
await getFiles(res);
|
||||
}
|
||||
} else if (
|
||||
(!prefix || entry.name.startsWith(prefix)) &&
|
||||
!entry.name.endsWith('.metadata.json')
|
||||
) {
|
||||
const stat = statSync(res);
|
||||
results.push({
|
||||
key: res,
|
||||
lastModified: stat.mtime,
|
||||
contentLength: stat.size,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// failed to read dir, stop recursion
|
||||
}
|
||||
}
|
||||
|
||||
await getFiles(dir, prefix);
|
||||
|
||||
// trim path with `this.path` prefix
|
||||
results.forEach(r => (r.key = r.key.slice(this.path.length + 1)));
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
delete(key: string): Promise<void> {
|
||||
key = normalizeStorageKey(key);
|
||||
|
||||
try {
|
||||
rmSync(this.join(key), { force: true });
|
||||
rmSync(this.join(`${key}.metadata.json`), { force: true });
|
||||
} catch (e) {
|
||||
throw new Error(`Failed to delete object \`${key}\``, {
|
||||
cause: e,
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.verbose(`Object \`${key}\` deleted`);
|
||||
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
ensureAvailability() {
|
||||
// check stats
|
||||
const stats = statSync(this.path, {
|
||||
throwIfNoEntry: false,
|
||||
});
|
||||
|
||||
// not existing, create it
|
||||
if (!stats) {
|
||||
try {
|
||||
mkdirSync(this.path, { recursive: true });
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Failed to create target directory for fs storage provider: ${this.path}`,
|
||||
{
|
||||
cause: e,
|
||||
}
|
||||
);
|
||||
}
|
||||
} else if (stats.isDirectory()) {
|
||||
// the target directory has already existed, check if it is readable & writable
|
||||
try {
|
||||
accessSync(this.path, constants.W_OK | constants.R_OK);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`The target directory for fs storage provider has already existed, but it is not readable & writable: ${this.path}`,
|
||||
{
|
||||
cause: e,
|
||||
}
|
||||
);
|
||||
}
|
||||
} else if (stats.isFile()) {
|
||||
throw new Error(
|
||||
`The target directory for fs storage provider is a file: ${this.path}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private join(...paths: string[]) {
|
||||
return join(this.path, ...paths);
|
||||
}
|
||||
|
||||
private readObject(file: string): Readable | undefined {
|
||||
const state = statSync(file, { throwIfNoEntry: false });
|
||||
|
||||
if (state?.isFile()) {
|
||||
return createReadStream(file);
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private writeObject(key: string, blob: Buffer) {
|
||||
const path = this.join(key);
|
||||
mkdirSync(parse(path).dir, { recursive: true });
|
||||
writeFileSync(path, blob);
|
||||
}
|
||||
|
||||
private async writeMetadata(
|
||||
key: string,
|
||||
blob: Buffer,
|
||||
raw: PutObjectMetadata
|
||||
) {
|
||||
try {
|
||||
const metadata = autoMetadata(blob, raw);
|
||||
|
||||
if (raw.checksumCRC32 && metadata.checksumCRC32 !== raw.checksumCRC32) {
|
||||
throw new Error(
|
||||
'The checksum of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.'
|
||||
);
|
||||
}
|
||||
|
||||
if (raw.contentLength && metadata.contentLength !== raw.contentLength) {
|
||||
throw new Error(
|
||||
'The content length of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.'
|
||||
);
|
||||
}
|
||||
|
||||
writeFileSync(
|
||||
this.join(`${key}.metadata.json`),
|
||||
JSON.stringify({
|
||||
...metadata,
|
||||
lastModified: Date.now(),
|
||||
})
|
||||
);
|
||||
} catch (e) {
|
||||
this.logger.warn(`Failed to write metadata of object \`${key}\``, e);
|
||||
}
|
||||
}
|
||||
|
||||
private readMetadata(key: string): GetObjectMetadata | undefined {
|
||||
try {
|
||||
const raw = JSON.parse(
|
||||
readFileSync(this.join(`${key}.metadata.json`), {
|
||||
encoding: 'utf-8',
|
||||
})
|
||||
);
|
||||
|
||||
return {
|
||||
...raw,
|
||||
lastModified: new Date(raw.lastModified),
|
||||
expires: raw.expires ? new Date(raw.expires) : undefined,
|
||||
};
|
||||
} catch (e) {
|
||||
this.logger.warn(`Failed to read metadata of object \`${key}\``, e);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,20 +1,45 @@
|
||||
import { Type } from '@nestjs/common';
|
||||
|
||||
import { JSONSchema } from '../../config';
|
||||
import { FsStorageConfig, FsStorageProvider } from './fs';
|
||||
import { StorageProvider } from './provider';
|
||||
import { R2_JURISDICTIONS, R2StorageConfig, R2StorageProvider } from './r2';
|
||||
import { S3StorageConfig, S3StorageProvider } from './s3';
|
||||
|
||||
export type StorageProviderName = 'fs' | 'aws-s3' | 'cloudflare-r2';
|
||||
export const StorageProviders: Record<
|
||||
StorageProviderName,
|
||||
Type<StorageProvider>
|
||||
> = {
|
||||
fs: FsStorageProvider,
|
||||
'aws-s3': S3StorageProvider,
|
||||
'cloudflare-r2': R2StorageProvider,
|
||||
};
|
||||
export type StorageProviderName =
|
||||
| 'fs'
|
||||
| 'aws-s3'
|
||||
| 'cloudflare-r2'
|
||||
| 'assetpack';
|
||||
|
||||
export interface FsStorageConfig {
|
||||
path: string;
|
||||
}
|
||||
|
||||
export type AssetpackStorageConfig = FsStorageConfig;
|
||||
|
||||
export interface S3StorageConfig {
|
||||
endpoint?: string;
|
||||
region: string;
|
||||
credentials?: {
|
||||
accessKeyId?: string;
|
||||
secretAccessKey?: string;
|
||||
sessionToken?: string;
|
||||
};
|
||||
forcePathStyle?: boolean;
|
||||
requestTimeoutMs?: number;
|
||||
minPartSize?: number;
|
||||
presign?: {
|
||||
expiresInSeconds?: number;
|
||||
signContentTypeForPut?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export const R2_JURISDICTIONS = ['default', 'eu'] as const;
|
||||
|
||||
export interface R2StorageConfig extends Omit<S3StorageConfig, 'endpoint'> {
|
||||
accountId: string;
|
||||
jurisdiction?: (typeof R2_JURISDICTIONS)[number];
|
||||
usePresignedURL?: {
|
||||
enabled: boolean;
|
||||
urlPrefix?: string;
|
||||
signKey?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export type StorageProviderConfig = { bucket: string } & (
|
||||
| {
|
||||
@@ -29,6 +54,10 @@ export type StorageProviderConfig = { bucket: string } & (
|
||||
provider: 'cloudflare-r2';
|
||||
config: R2StorageConfig;
|
||||
}
|
||||
| {
|
||||
provider: 'assetpack';
|
||||
config: AssetpackStorageConfig;
|
||||
}
|
||||
);
|
||||
|
||||
const S3ConfigSchema: JSONSchema = {
|
||||
@@ -186,16 +215,37 @@ export const StorageJSONSchema: JSONSchema = {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
properties: {
|
||||
provider: {
|
||||
type: 'string',
|
||||
enum: ['assetpack'],
|
||||
},
|
||||
bucket: {
|
||||
type: 'string',
|
||||
},
|
||||
config: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string',
|
||||
},
|
||||
},
|
||||
required: ['path'],
|
||||
},
|
||||
},
|
||||
required: ['provider', 'bucket', 'config'],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export type * from './provider';
|
||||
export type * from '../types';
|
||||
export {
|
||||
applyAttachHeaders,
|
||||
autoMetadata,
|
||||
PROXY_MULTIPART_PATH,
|
||||
PROXY_UPLOAD_PATH,
|
||||
sniffMime,
|
||||
STORAGE_PROXY_ROOT,
|
||||
toBuffer,
|
||||
} from './utils';
|
||||
} from '../utils';
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
import type { Readable } from 'node:stream';
|
||||
|
||||
export interface GetObjectMetadata {
|
||||
/**
|
||||
* @default 'application/octet-stream'
|
||||
*/
|
||||
contentType: string;
|
||||
contentLength: number;
|
||||
lastModified: Date;
|
||||
checksumCRC32?: string;
|
||||
}
|
||||
|
||||
export interface PutObjectMetadata {
|
||||
contentType?: string;
|
||||
contentLength?: number;
|
||||
checksumCRC32?: string;
|
||||
}
|
||||
|
||||
export interface ListObjectsMetadata {
|
||||
key: string;
|
||||
lastModified: Date;
|
||||
contentLength: number;
|
||||
}
|
||||
|
||||
export type BlobInputType = Buffer | Readable | string;
|
||||
export type BlobOutputType = Readable;
|
||||
|
||||
export interface PresignedUpload {
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
expiresAt: Date;
|
||||
}
|
||||
|
||||
export interface MultipartUploadInit {
|
||||
uploadId: string;
|
||||
expiresAt: Date;
|
||||
}
|
||||
|
||||
export interface MultipartUploadPart {
|
||||
partNumber: number;
|
||||
etag: string;
|
||||
}
|
||||
|
||||
export interface StorageProvider {
|
||||
put(
|
||||
key: string,
|
||||
body: BlobInputType,
|
||||
metadata?: PutObjectMetadata
|
||||
): Promise<void>;
|
||||
presignPut?(
|
||||
key: string,
|
||||
metadata?: PutObjectMetadata
|
||||
): Promise<PresignedUpload | undefined>;
|
||||
createMultipartUpload?(
|
||||
key: string,
|
||||
metadata?: PutObjectMetadata
|
||||
): Promise<MultipartUploadInit | undefined>;
|
||||
presignUploadPart?(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
partNumber: number
|
||||
): Promise<PresignedUpload | undefined>;
|
||||
listMultipartUploadParts?(
|
||||
key: string,
|
||||
uploadId: string
|
||||
): Promise<MultipartUploadPart[] | undefined>;
|
||||
completeMultipartUpload?(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: MultipartUploadPart[]
|
||||
): Promise<void>;
|
||||
abortMultipartUpload?(key: string, uploadId: string): Promise<void>;
|
||||
head(key: string): Promise<GetObjectMetadata | undefined>;
|
||||
get(
|
||||
key: string,
|
||||
signedUrl?: boolean
|
||||
): Promise<{
|
||||
redirectUrl?: string;
|
||||
body?: BlobOutputType;
|
||||
metadata?: GetObjectMetadata;
|
||||
}>;
|
||||
list(prefix?: string): Promise<ListObjectsMetadata[]>;
|
||||
delete(key: string): Promise<void>;
|
||||
}
|
||||
@@ -1,251 +0,0 @@
|
||||
import assert from 'node:assert';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
GetObjectMetadata,
|
||||
PresignedUpload,
|
||||
PutObjectMetadata,
|
||||
} from './provider';
|
||||
import { S3StorageConfig, S3StorageProvider } from './s3';
|
||||
import {
|
||||
PROXY_MULTIPART_PATH,
|
||||
PROXY_UPLOAD_PATH,
|
||||
SIGNED_URL_EXPIRED,
|
||||
} from './utils';
|
||||
|
||||
export const R2_JURISDICTIONS = ['eu'] as const;
|
||||
type R2Jurisdiction = (typeof R2_JURISDICTIONS)[number];
|
||||
|
||||
export interface R2StorageConfig extends Omit<
|
||||
S3StorageConfig,
|
||||
'endpoint' | 'forcePathStyle'
|
||||
> {
|
||||
accountId: string;
|
||||
jurisdiction?: R2Jurisdiction;
|
||||
usePresignedURL?: {
|
||||
enabled: boolean;
|
||||
urlPrefix?: string;
|
||||
signKey?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export class R2StorageProvider extends S3StorageProvider {
|
||||
private readonly encoder = new TextEncoder();
|
||||
private readonly key: Uint8Array;
|
||||
|
||||
constructor(
|
||||
private readonly config: R2StorageConfig,
|
||||
bucket: string
|
||||
) {
|
||||
assert(config.accountId, 'accountId is required for R2 storage provider');
|
||||
const account = config.jurisdiction
|
||||
? `${config.accountId}.${config.jurisdiction}`
|
||||
: config.accountId;
|
||||
const endpoint = `https://${account}.r2.cloudflarestorage.com`;
|
||||
|
||||
super(
|
||||
{
|
||||
...config,
|
||||
forcePathStyle: true,
|
||||
endpoint,
|
||||
},
|
||||
bucket
|
||||
);
|
||||
this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`);
|
||||
this.key = this.encoder.encode(config.usePresignedURL?.signKey ?? '');
|
||||
}
|
||||
|
||||
private get shouldUseProxyUpload() {
|
||||
const { usePresignedURL } = this.config;
|
||||
return (
|
||||
!!usePresignedURL?.enabled &&
|
||||
!!usePresignedURL.signKey &&
|
||||
this.key.length > 0
|
||||
);
|
||||
}
|
||||
|
||||
private parseWorkspaceKey(fullKey: string) {
|
||||
const [workspaceId, ...rest] = fullKey.split('/');
|
||||
if (!workspaceId || rest.length !== 1) {
|
||||
return null;
|
||||
}
|
||||
return { workspaceId, key: rest.join('/') };
|
||||
}
|
||||
|
||||
private async signPayload(payload: string) {
|
||||
const key = await crypto.subtle.importKey(
|
||||
'raw',
|
||||
this.key,
|
||||
{ name: 'HMAC', hash: 'SHA-256' },
|
||||
false,
|
||||
['sign', 'verify']
|
||||
);
|
||||
const mac = await crypto.subtle.sign(
|
||||
'HMAC',
|
||||
key,
|
||||
this.encoder.encode(payload)
|
||||
);
|
||||
|
||||
return Buffer.from(mac).toString('base64');
|
||||
}
|
||||
|
||||
private async signUrl(url: URL): Promise<string> {
|
||||
const timestamp = Math.floor(Date.now() / 1000);
|
||||
const base64Mac = await this.signPayload(`${url.pathname}${timestamp}`);
|
||||
url.searchParams.set('sign', `${timestamp}-${base64Mac}`);
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
private async createProxyUrl(
|
||||
path: string,
|
||||
canonicalFields: (string | number | undefined)[],
|
||||
query: Record<string, string | number | undefined>
|
||||
) {
|
||||
const exp = Math.floor(Date.now() / 1000) + SIGNED_URL_EXPIRED;
|
||||
const canonical = [
|
||||
path,
|
||||
...canonicalFields.map(field =>
|
||||
field === undefined ? '' : field.toString()
|
||||
),
|
||||
exp.toString(),
|
||||
].join('\n');
|
||||
const token = await this.signPayload(canonical);
|
||||
|
||||
const url = new URL(`http://localhost${path}`);
|
||||
for (const [key, value] of Object.entries(query)) {
|
||||
if (value === undefined) continue;
|
||||
url.searchParams.set(key, value.toString());
|
||||
}
|
||||
url.searchParams.set('exp', exp.toString());
|
||||
url.searchParams.set('token', `${exp}-${token}`);
|
||||
|
||||
return { url: url.pathname + url.search, expiresAt: new Date(exp * 1000) };
|
||||
}
|
||||
|
||||
override async presignPut(
|
||||
key: string,
|
||||
metadata: PutObjectMetadata = {}
|
||||
): Promise<PresignedUpload | undefined> {
|
||||
if (!this.shouldUseProxyUpload) {
|
||||
return super.presignPut(key, metadata);
|
||||
}
|
||||
|
||||
const parsed = this.parseWorkspaceKey(key);
|
||||
if (!parsed) {
|
||||
return super.presignPut(key, metadata);
|
||||
}
|
||||
|
||||
const contentType = metadata.contentType ?? 'application/octet-stream';
|
||||
const { url, expiresAt } = await this.createProxyUrl(
|
||||
PROXY_UPLOAD_PATH,
|
||||
[parsed.workspaceId, parsed.key, contentType, metadata.contentLength],
|
||||
{
|
||||
workspaceId: parsed.workspaceId,
|
||||
key: parsed.key,
|
||||
contentType,
|
||||
contentLength: metadata.contentLength,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
url,
|
||||
headers: { 'Content-Type': contentType },
|
||||
expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
override async presignUploadPart(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
partNumber: number
|
||||
): Promise<PresignedUpload | undefined> {
|
||||
if (!this.shouldUseProxyUpload) {
|
||||
return super.presignUploadPart(key, uploadId, partNumber);
|
||||
}
|
||||
|
||||
const parsed = this.parseWorkspaceKey(key);
|
||||
if (!parsed) {
|
||||
return super.presignUploadPart(key, uploadId, partNumber);
|
||||
}
|
||||
|
||||
return this.createProxyUrl(
|
||||
PROXY_MULTIPART_PATH,
|
||||
[parsed.workspaceId, parsed.key, uploadId, partNumber],
|
||||
{
|
||||
workspaceId: parsed.workspaceId,
|
||||
key: parsed.key,
|
||||
uploadId,
|
||||
partNumber,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async proxyPutObject(
|
||||
key: string,
|
||||
body: Readable | Buffer | Uint8Array | string,
|
||||
options: { contentType?: string; contentLength?: number } = {}
|
||||
) {
|
||||
return this.client.putObject(key, this.normalizeBody(body), {
|
||||
contentType: options.contentType,
|
||||
contentLength: options.contentLength,
|
||||
});
|
||||
}
|
||||
|
||||
async proxyUploadPart(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
partNumber: number,
|
||||
body: Readable | Buffer | Uint8Array | string,
|
||||
options: { contentLength?: number } = {}
|
||||
) {
|
||||
const result = await this.client.uploadPart(
|
||||
key,
|
||||
uploadId,
|
||||
partNumber,
|
||||
this.normalizeBody(body),
|
||||
{ contentLength: options.contentLength }
|
||||
);
|
||||
|
||||
return result.etag;
|
||||
}
|
||||
|
||||
private normalizeBody(body: Readable | Buffer | Uint8Array | string) {
|
||||
// s3mini does not accept Node.js Readable directly.
|
||||
// Convert it to Web ReadableStream for compatibility.
|
||||
if (body instanceof Readable) {
|
||||
return Readable.toWeb(body);
|
||||
} else if (typeof body === 'string') {
|
||||
return this.encoder.encode(body);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
override async get(
|
||||
key: string,
|
||||
signedUrl?: boolean
|
||||
): Promise<{
|
||||
body?: Readable;
|
||||
metadata?: GetObjectMetadata;
|
||||
redirectUrl?: string;
|
||||
}> {
|
||||
const { usePresignedURL: { enabled, urlPrefix } = {} } = this.config;
|
||||
if (signedUrl && enabled && urlPrefix) {
|
||||
const metadata = await this.head(key);
|
||||
const url = await this.signUrl(new URL(`/${key}`, urlPrefix));
|
||||
if (metadata) {
|
||||
return {
|
||||
redirectUrl: url.toString(),
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
|
||||
// object not found
|
||||
return {};
|
||||
}
|
||||
|
||||
// fallback to s3 get
|
||||
return super.get(key, signedUrl);
|
||||
}
|
||||
}
|
||||
@@ -1,363 +0,0 @@
|
||||
/* oxlint-disable @typescript-eslint/no-non-null-assertion */
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import type {
|
||||
S3CompatClient,
|
||||
S3CompatConfig,
|
||||
S3CompatCredentials,
|
||||
} from '@affine/s3-compat';
|
||||
import { createS3CompatClient } from '@affine/s3-compat';
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
BlobInputType,
|
||||
GetObjectMetadata,
|
||||
ListObjectsMetadata,
|
||||
MultipartUploadInit,
|
||||
MultipartUploadPart,
|
||||
PresignedUpload,
|
||||
PutObjectMetadata,
|
||||
StorageProvider,
|
||||
} from './provider';
|
||||
import { autoMetadata, SIGNED_URL_EXPIRED, toBuffer } from './utils';
|
||||
|
||||
export interface S3StorageConfig {
|
||||
endpoint?: string;
|
||||
region: string;
|
||||
credentials: S3CompatCredentials;
|
||||
forcePathStyle?: boolean;
|
||||
requestTimeoutMs?: number;
|
||||
minPartSize?: number;
|
||||
presign?: {
|
||||
expiresInSeconds?: number;
|
||||
signContentTypeForPut?: boolean;
|
||||
};
|
||||
usePresignedURL?: {
|
||||
enabled: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
function resolveEndpoint(config: S3StorageConfig) {
|
||||
if (config.endpoint) {
|
||||
return config.endpoint;
|
||||
}
|
||||
if (config.region === 'us-east-1') {
|
||||
return 'https://s3.amazonaws.com';
|
||||
}
|
||||
return `https://s3.${config.region}.amazonaws.com`;
|
||||
}
|
||||
|
||||
function joinPath(basePath: string, suffix: string) {
|
||||
const trimmedBase = basePath.endsWith('/') ? basePath.slice(0, -1) : basePath;
|
||||
const trimmedSuffix = suffix.startsWith('/') ? suffix.slice(1) : suffix;
|
||||
if (!trimmedBase) {
|
||||
return `/${trimmedSuffix}`;
|
||||
}
|
||||
if (!trimmedSuffix) {
|
||||
return trimmedBase;
|
||||
}
|
||||
return `${trimmedBase}/${trimmedSuffix}`;
|
||||
}
|
||||
|
||||
function composeEndpointUrl(config: S3CompatConfig) {
|
||||
const url = new URL(config.endpoint);
|
||||
if (config.forcePathStyle) {
|
||||
const firstSegment = url.pathname.split('/').find(Boolean);
|
||||
if (firstSegment !== config.bucket) {
|
||||
url.pathname = joinPath(url.pathname, config.bucket);
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
const firstSegment = url.pathname.split('/').find(Boolean);
|
||||
const hostHasBucket = url.hostname.startsWith(`${config.bucket}.`);
|
||||
const pathHasBucket = firstSegment === config.bucket;
|
||||
if (!hostHasBucket && !pathHasBucket) {
|
||||
url.hostname = `${config.bucket}.${url.hostname}`;
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export class S3StorageProvider implements StorageProvider {
|
||||
protected logger: Logger;
|
||||
protected client: S3CompatClient;
|
||||
private readonly usePresignedURL: boolean;
|
||||
private readonly endpoint: string;
|
||||
|
||||
get endpointUrl() {
|
||||
return this.endpoint;
|
||||
}
|
||||
|
||||
constructor(
|
||||
config: S3StorageConfig,
|
||||
public readonly bucket: string
|
||||
) {
|
||||
const { usePresignedURL, presign, credentials, ...clientConfig } = config;
|
||||
|
||||
const compatConfig: S3CompatConfig = {
|
||||
...clientConfig,
|
||||
endpoint: resolveEndpoint(config),
|
||||
bucket,
|
||||
requestTimeoutMs: clientConfig.requestTimeoutMs ?? 60_000,
|
||||
presign: {
|
||||
expiresInSeconds: presign?.expiresInSeconds ?? SIGNED_URL_EXPIRED,
|
||||
signContentTypeForPut: presign?.signContentTypeForPut ?? true,
|
||||
},
|
||||
};
|
||||
|
||||
this.endpoint = composeEndpointUrl(compatConfig);
|
||||
this.client = createS3CompatClient(compatConfig, credentials);
|
||||
this.usePresignedURL = usePresignedURL?.enabled ?? false;
|
||||
this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`);
|
||||
}
|
||||
|
||||
async put(
|
||||
key: string,
|
||||
body: BlobInputType,
|
||||
metadata: PutObjectMetadata = {}
|
||||
): Promise<void> {
|
||||
const blob = await toBuffer(body);
|
||||
|
||||
metadata = autoMetadata(blob, metadata);
|
||||
|
||||
try {
|
||||
await this.client.putObject(key, blob, {
|
||||
contentType: metadata.contentType,
|
||||
contentLength: metadata.contentLength,
|
||||
});
|
||||
|
||||
this.logger.verbose(`Object \`${key}\` put`);
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
`Failed to put object (${JSON.stringify({
|
||||
key,
|
||||
bucket: this.bucket,
|
||||
metadata,
|
||||
})})`
|
||||
);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async presignPut(
|
||||
key: string,
|
||||
metadata: PutObjectMetadata = {}
|
||||
): Promise<PresignedUpload | undefined> {
|
||||
try {
|
||||
const contentType = metadata.contentType ?? 'application/octet-stream';
|
||||
const result = await this.client.presignPutObject(key, { contentType });
|
||||
|
||||
return {
|
||||
url: result.url,
|
||||
headers: result.headers,
|
||||
expiresAt: result.expiresAt,
|
||||
};
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
`Failed to presign put object (${JSON.stringify({
|
||||
key,
|
||||
bucket: this.bucket,
|
||||
metadata,
|
||||
})}`
|
||||
);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async createMultipartUpload(
|
||||
key: string,
|
||||
metadata: PutObjectMetadata = {}
|
||||
): Promise<MultipartUploadInit | undefined> {
|
||||
try {
|
||||
const contentType = metadata.contentType ?? 'application/octet-stream';
|
||||
const response = await this.client.createMultipartUpload(key, {
|
||||
contentType,
|
||||
});
|
||||
|
||||
if (!response.uploadId) {
|
||||
return;
|
||||
}
|
||||
|
||||
return {
|
||||
uploadId: response.uploadId,
|
||||
expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000),
|
||||
};
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
`Failed to create multipart upload (${JSON.stringify({
|
||||
key,
|
||||
bucket: this.bucket,
|
||||
metadata,
|
||||
})}`
|
||||
);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async presignUploadPart(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
partNumber: number
|
||||
): Promise<PresignedUpload | undefined> {
|
||||
try {
|
||||
const result = await this.client.presignUploadPart(
|
||||
key,
|
||||
uploadId,
|
||||
partNumber
|
||||
);
|
||||
|
||||
return {
|
||||
url: result.url,
|
||||
expiresAt: result.expiresAt,
|
||||
};
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
`Failed to presign upload part (${JSON.stringify({ key, bucket: this.bucket, uploadId, partNumber })}`
|
||||
);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async listMultipartUploadParts(
|
||||
key: string,
|
||||
uploadId: string
|
||||
): Promise<MultipartUploadPart[] | undefined> {
|
||||
try {
|
||||
return await this.client.listParts(key, uploadId);
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to list multipart upload parts for \`${key}\``);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async completeMultipartUpload(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: MultipartUploadPart[]
|
||||
): Promise<void> {
|
||||
try {
|
||||
const orderedParts = [...parts].sort(
|
||||
(left, right) => left.partNumber - right.partNumber
|
||||
);
|
||||
|
||||
await this.client.completeMultipartUpload(key, uploadId, orderedParts);
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to complete multipart upload for \`${key}\``);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async abortMultipartUpload(key: string, uploadId: string): Promise<void> {
|
||||
try {
|
||||
await this.client.abortMultipartUpload(key, uploadId);
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to abort multipart upload for \`${key}\``);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async head(key: string) {
|
||||
try {
|
||||
const obj = await this.client.headObject(key);
|
||||
if (!obj) {
|
||||
this.logger.verbose(`Object \`${key}\` not found`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
contentType: obj.contentType ?? 'application/octet-stream',
|
||||
contentLength: obj.contentLength ?? 0,
|
||||
lastModified: obj.lastModified ?? new Date(0),
|
||||
checksumCRC32: obj.checksumCRC32,
|
||||
};
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to head object \`${key}\``);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async get(
|
||||
key: string,
|
||||
signedUrl?: boolean
|
||||
): Promise<{
|
||||
body?: Readable;
|
||||
metadata?: GetObjectMetadata;
|
||||
redirectUrl?: string;
|
||||
}> {
|
||||
try {
|
||||
if (this.usePresignedURL && signedUrl) {
|
||||
const metadata = await this.head(key);
|
||||
if (metadata) {
|
||||
const result = await this.client.presignGetObject(key);
|
||||
|
||||
return {
|
||||
redirectUrl: result.url,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
|
||||
// object not found
|
||||
return {};
|
||||
}
|
||||
|
||||
const obj = await this.client.getObjectResponse(key);
|
||||
if (!obj || !obj.body) {
|
||||
this.logger.verbose(`Object \`${key}\` not found`);
|
||||
return {};
|
||||
}
|
||||
|
||||
const contentType = obj.headers.get('content-type') ?? undefined;
|
||||
const contentLengthHeader = obj.headers.get('content-length');
|
||||
const contentLength = contentLengthHeader
|
||||
? Number(contentLengthHeader)
|
||||
: undefined;
|
||||
const lastModifiedHeader = obj.headers.get('last-modified');
|
||||
const lastModified = lastModifiedHeader
|
||||
? new Date(lastModifiedHeader)
|
||||
: undefined;
|
||||
|
||||
this.logger.verbose(`Read object \`${key}\``);
|
||||
return {
|
||||
body: Readable.fromWeb(obj.body),
|
||||
metadata: {
|
||||
contentType: contentType ?? 'application/octet-stream',
|
||||
contentLength: contentLength ?? 0,
|
||||
lastModified: lastModified ?? new Date(0),
|
||||
checksumCRC32: obj.headers.get('x-amz-checksum-crc32') ?? undefined,
|
||||
},
|
||||
};
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to read object \`${key}\``);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async list(prefix?: string): Promise<ListObjectsMetadata[]> {
|
||||
try {
|
||||
const result = await this.client.listObjectsV2(prefix);
|
||||
|
||||
this.logger.verbose(
|
||||
`List ${result.length} objects with prefix \`${prefix}\``
|
||||
);
|
||||
return result;
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to list objects with prefix \`${prefix}\``);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
try {
|
||||
await this.client.deleteObject(key);
|
||||
|
||||
this.logger.verbose(`Deleted object \`${key}\``);
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to delete object \`${key}\``, {
|
||||
bucket: this.bucket,
|
||||
key,
|
||||
cause: e,
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { Readable } from 'node:stream';
|
||||
|
||||
export interface GetObjectMetadata {
|
||||
/**
|
||||
* @default 'application/octet-stream'
|
||||
*/
|
||||
contentType: string;
|
||||
contentLength: number;
|
||||
lastModified: Date;
|
||||
checksumCRC32?: string;
|
||||
}
|
||||
|
||||
export interface PutObjectMetadata {
|
||||
contentType?: string;
|
||||
contentLength?: number;
|
||||
checksumCRC32?: string;
|
||||
}
|
||||
|
||||
export interface ListObjectsMetadata {
|
||||
key: string;
|
||||
lastModified: Date;
|
||||
contentLength: number;
|
||||
}
|
||||
|
||||
export type BlobInputType = Buffer | Readable | string;
|
||||
export type BlobOutputType = Readable;
|
||||
|
||||
export interface PresignedUpload {
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
expiresAt: Date;
|
||||
}
|
||||
+2
-32
@@ -1,11 +1,10 @@
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import { crc32 } from '@node-rs/crc32';
|
||||
import type { Response } from 'express';
|
||||
import { getStreamAsBuffer } from 'get-stream';
|
||||
|
||||
import { getMime } from '../../../native';
|
||||
import { BlobInputType, PutObjectMetadata } from './provider';
|
||||
import { getMime } from '../../native';
|
||||
import type { BlobInputType } from './types';
|
||||
|
||||
export async function toBuffer(input: BlobInputType): Promise<Buffer> {
|
||||
return input instanceof Readable
|
||||
@@ -15,35 +14,6 @@ export async function toBuffer(input: BlobInputType): Promise<Buffer> {
|
||||
: Buffer.from(input as string);
|
||||
}
|
||||
|
||||
export function autoMetadata(
|
||||
blob: Buffer,
|
||||
raw: PutObjectMetadata = {}
|
||||
): PutObjectMetadata {
|
||||
const metadata = {
|
||||
...raw,
|
||||
};
|
||||
|
||||
if (!metadata.contentLength) {
|
||||
metadata.contentLength = blob.byteLength;
|
||||
}
|
||||
|
||||
try {
|
||||
// checksum
|
||||
if (!metadata.checksumCRC32) {
|
||||
metadata.checksumCRC32 = crc32(blob).toString(16);
|
||||
}
|
||||
|
||||
// mime type
|
||||
if (!metadata.contentType) {
|
||||
metadata.contentType = getMime(blob);
|
||||
}
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
const DANGEROUS_INLINE_MIME_PREFIXES = [
|
||||
'text/html',
|
||||
'application/xhtml+xml',
|
||||
Reference in New Issue
Block a user