mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-21 12:06:35 +08:00
feat: multipart blob sync support (#14138)
<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit
* **New Features**
* Flexible blob uploads: GRAPHQL, presigned, and multipart flows with
per‑part URLs, abort/complete operations, presigned proxy endpoints, and
nightly cleanup of expired pending uploads.
* **API / Schema**
* GraphQL additions: new types, mutations, enum and error to manage
upload lifecycle (create, complete, abort, get part URL).
* **Database**
* New blob status enum and columns (status, upload_id); listing now
defaults to completed blobs.
* **Localization**
* Added user-facing message: "Blob is invalid."
* **Tests**
* Expanded unit and end‑to‑end coverage for upload flows, proxy
behavior, multipart and provider integrations.
<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
import { OneMB } from '../../base';
|
||||
|
||||
export const MULTIPART_PART_SIZE = 5 * OneMB;
|
||||
export const MULTIPART_THRESHOLD = 10 * OneMB;
|
||||
@@ -2,6 +2,8 @@ import './config';
|
||||
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { BlobUploadCleanupJob } from './job';
|
||||
import { R2UploadController } from './r2-proxy';
|
||||
import {
|
||||
AvatarStorage,
|
||||
CommentAttachmentStorage,
|
||||
@@ -9,7 +11,13 @@ import {
|
||||
} from './wrappers';
|
||||
|
||||
@Module({
|
||||
providers: [WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage],
|
||||
controllers: [R2UploadController],
|
||||
providers: [
|
||||
WorkspaceBlobStorage,
|
||||
AvatarStorage,
|
||||
CommentAttachmentStorage,
|
||||
BlobUploadCleanupJob,
|
||||
],
|
||||
exports: [WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage],
|
||||
})
|
||||
export class StorageModule {}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
|
||||
import { JobQueue, OneDay, OnJob } from '../../base';
|
||||
import { Models } from '../../models';
|
||||
import { WorkspaceBlobStorage } from './wrappers/blob';
|
||||
|
||||
declare global {
|
||||
interface Jobs {
|
||||
'nightly.cleanExpiredPendingBlobs': {};
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class BlobUploadCleanupJob {
|
||||
private readonly logger = new Logger(BlobUploadCleanupJob.name);
|
||||
|
||||
constructor(
|
||||
private readonly models: Models,
|
||||
private readonly storage: WorkspaceBlobStorage,
|
||||
private readonly queue: JobQueue
|
||||
) {}
|
||||
|
||||
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
|
||||
async nightlyJob() {
|
||||
await this.queue.add(
|
||||
'nightly.cleanExpiredPendingBlobs',
|
||||
{},
|
||||
{
|
||||
jobId: 'nightly-blob-clean-expired-pending',
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@OnJob('nightly.cleanExpiredPendingBlobs')
|
||||
async cleanExpiredPendingBlobs() {
|
||||
const cutoff = new Date(Date.now() - OneDay);
|
||||
const pending = await this.models.blob.listPendingExpired(cutoff);
|
||||
|
||||
for (const blob of pending) {
|
||||
if (blob.uploadId) {
|
||||
await this.storage.abortMultipartUpload(
|
||||
blob.workspaceId,
|
||||
blob.key,
|
||||
blob.uploadId
|
||||
);
|
||||
}
|
||||
|
||||
await this.storage.delete(blob.workspaceId, blob.key, true);
|
||||
}
|
||||
|
||||
this.logger.log(`cleaned ${pending.length} expired pending blobs`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
import { createHmac, timingSafeEqual } from 'node:crypto';
|
||||
|
||||
import { Controller, Logger, Put, Req, Res } from '@nestjs/common';
|
||||
import type { Request, Response } from 'express';
|
||||
|
||||
import {
|
||||
BlobInvalid,
|
||||
CallMetric,
|
||||
Config,
|
||||
OnEvent,
|
||||
PROXY_MULTIPART_PATH,
|
||||
PROXY_UPLOAD_PATH,
|
||||
STORAGE_PROXY_ROOT,
|
||||
StorageProviderConfig,
|
||||
StorageProviderFactory,
|
||||
} from '../../base';
|
||||
import {
|
||||
R2StorageConfig,
|
||||
R2StorageProvider,
|
||||
} from '../../base/storage/providers/r2';
|
||||
import { Models } from '../../models';
|
||||
import { Public } from '../auth/guard';
|
||||
import { MULTIPART_PART_SIZE } from './constants';
|
||||
|
||||
type R2BlobStorageConfig = StorageProviderConfig & {
|
||||
provider: 'cloudflare-r2';
|
||||
config: R2StorageConfig;
|
||||
};
|
||||
|
||||
type QueryValue = Request['query'][string];
|
||||
|
||||
type R2Config = {
|
||||
storage: R2BlobStorageConfig;
|
||||
signKey: string;
|
||||
};
|
||||
|
||||
@Controller(STORAGE_PROXY_ROOT)
|
||||
export class R2UploadController {
|
||||
private readonly logger = new Logger(R2UploadController.name);
|
||||
private provider: R2StorageProvider | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly models: Models,
|
||||
private readonly storageFactory: StorageProviderFactory
|
||||
) {}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
onConfigChanged(event: Events['config.changed']) {
|
||||
if (event.updates.storages?.blob?.storage) {
|
||||
this.provider = null;
|
||||
}
|
||||
}
|
||||
|
||||
private getR2Config(): R2Config {
|
||||
const storage = this.config.storages.blob.storage as StorageProviderConfig;
|
||||
if (storage.provider !== 'cloudflare-r2') {
|
||||
throw new BlobInvalid('Invalid endpoint');
|
||||
}
|
||||
const r2Config = storage.config as R2StorageConfig;
|
||||
const signKey = r2Config.usePresignedURL?.signKey;
|
||||
if (
|
||||
!r2Config.usePresignedURL?.enabled ||
|
||||
!r2Config.usePresignedURL.urlPrefix ||
|
||||
!signKey
|
||||
) {
|
||||
throw new BlobInvalid('Invalid endpoint');
|
||||
}
|
||||
return { storage: storage as R2BlobStorageConfig, signKey };
|
||||
}
|
||||
|
||||
private getProvider(storage: R2BlobStorageConfig) {
|
||||
if (!this.provider) {
|
||||
const candidate = this.storageFactory.create(storage);
|
||||
if (candidate instanceof R2StorageProvider) {
|
||||
this.provider = candidate;
|
||||
}
|
||||
}
|
||||
return this.provider;
|
||||
}
|
||||
|
||||
private sign(canonical: string, signKey: string) {
|
||||
return createHmac('sha256', signKey).update(canonical).digest('base64');
|
||||
}
|
||||
|
||||
private safeEqual(expected: string, actual: string) {
|
||||
const a = Buffer.from(expected);
|
||||
const b = Buffer.from(actual);
|
||||
|
||||
if (a.length !== b.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return timingSafeEqual(a, b);
|
||||
}
|
||||
|
||||
private verifyToken(
|
||||
path: string,
|
||||
canonicalFields: (string | number | undefined)[],
|
||||
exp: number,
|
||||
token: string,
|
||||
signKey: string
|
||||
) {
|
||||
const canonical = [
|
||||
path,
|
||||
...canonicalFields.map(field =>
|
||||
field === undefined ? '' : field.toString()
|
||||
),
|
||||
exp.toString(),
|
||||
].join('\n');
|
||||
const expected = `${exp}-${this.sign(canonical, signKey)}`;
|
||||
|
||||
return this.safeEqual(expected, token);
|
||||
}
|
||||
|
||||
private expectString(value: QueryValue, field: string): string {
|
||||
if (Array.isArray(value)) {
|
||||
return String(value[0]);
|
||||
}
|
||||
if (typeof value === 'string' && value.length > 0) {
|
||||
return value;
|
||||
}
|
||||
throw new BlobInvalid(`Missing ${field}.`);
|
||||
}
|
||||
|
||||
private optionalString(value: QueryValue) {
|
||||
if (Array.isArray(value)) {
|
||||
return String(value[0]);
|
||||
}
|
||||
return typeof value === 'string' && value.length > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
private number(value: QueryValue, field: string): number {
|
||||
const str = this.expectString(value, field);
|
||||
const num = Number(str);
|
||||
if (!Number.isFinite(num)) {
|
||||
throw new BlobInvalid(`Invalid ${field}.`);
|
||||
}
|
||||
return num;
|
||||
}
|
||||
|
||||
private optionalNumber(value: QueryValue, field: string): number | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const num = Number(Array.isArray(value) ? value[0] : value);
|
||||
if (!Number.isFinite(num)) {
|
||||
throw new BlobInvalid(`Invalid ${field}.`);
|
||||
}
|
||||
return num;
|
||||
}
|
||||
|
||||
private parseContentLength(req: Request) {
|
||||
const raw = req.header('content-length');
|
||||
if (!raw) {
|
||||
return undefined;
|
||||
}
|
||||
const num = Number(raw);
|
||||
if (!Number.isFinite(num) || num < 0) {
|
||||
throw new BlobInvalid('Invalid Content-Length header');
|
||||
}
|
||||
return num;
|
||||
}
|
||||
|
||||
private ensureNotExpired(exp: number) {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
if (exp < now) {
|
||||
throw new BlobInvalid('Upload URL expired');
|
||||
}
|
||||
}
|
||||
|
||||
@Public()
|
||||
@Put('upload')
|
||||
@CallMetric('controllers', 'r2_proxy_upload')
|
||||
async upload(@Req() req: Request, @Res() res: Response) {
|
||||
const { storage, signKey } = this.getR2Config();
|
||||
|
||||
const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId');
|
||||
const key = this.expectString(req.query.key, 'key');
|
||||
const token = this.expectString(req.query.token, 'token');
|
||||
const exp = this.number(req.query.exp, 'exp');
|
||||
const contentType = this.optionalString(req.query.contentType);
|
||||
const contentLengthFromQuery = this.optionalNumber(
|
||||
req.query.contentLength,
|
||||
'contentLength'
|
||||
);
|
||||
|
||||
this.ensureNotExpired(exp);
|
||||
|
||||
if (
|
||||
!this.verifyToken(
|
||||
PROXY_UPLOAD_PATH,
|
||||
[workspaceId, key, contentType, contentLengthFromQuery],
|
||||
exp,
|
||||
token,
|
||||
signKey
|
||||
)
|
||||
) {
|
||||
throw new BlobInvalid('Invalid upload token');
|
||||
}
|
||||
|
||||
const record = await this.models.blob.get(workspaceId, key);
|
||||
if (!record) {
|
||||
throw new BlobInvalid('Blob upload is not initialized');
|
||||
}
|
||||
if (record.status === 'completed') {
|
||||
throw new BlobInvalid('Blob upload is already completed');
|
||||
}
|
||||
|
||||
const contentLengthHeader = this.parseContentLength(req);
|
||||
if (
|
||||
contentLengthFromQuery !== undefined &&
|
||||
contentLengthHeader !== undefined &&
|
||||
contentLengthFromQuery !== contentLengthHeader
|
||||
) {
|
||||
throw new BlobInvalid('Content length mismatch');
|
||||
}
|
||||
|
||||
const contentLength = contentLengthHeader ?? contentLengthFromQuery;
|
||||
if (contentLength === undefined) {
|
||||
throw new BlobInvalid('Missing Content-Length header');
|
||||
}
|
||||
if (record.size && contentLength !== record.size) {
|
||||
throw new BlobInvalid('Content length does not match upload metadata');
|
||||
}
|
||||
|
||||
const mime = contentType ?? record.mime;
|
||||
if (record.mime && mime && record.mime !== mime) {
|
||||
throw new BlobInvalid('Mime type mismatch');
|
||||
}
|
||||
|
||||
const provider = this.getProvider(storage);
|
||||
if (!provider) {
|
||||
throw new BlobInvalid('R2 provider is not available');
|
||||
}
|
||||
|
||||
try {
|
||||
await provider.proxyPutObject(`${workspaceId}/${key}`, req, {
|
||||
contentType: mime,
|
||||
contentLength,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to proxy upload', error as Error);
|
||||
throw new BlobInvalid('Upload failed');
|
||||
}
|
||||
|
||||
res.status(200).end();
|
||||
}
|
||||
|
||||
@Public()
|
||||
@Put('multipart')
|
||||
@CallMetric('controllers', 'r2_proxy_multipart')
|
||||
async uploadPart(@Req() req: Request, @Res() res: Response) {
|
||||
const { storage, signKey } = this.getR2Config();
|
||||
|
||||
const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId');
|
||||
const key = this.expectString(req.query.key, 'key');
|
||||
const uploadId = this.expectString(req.query.uploadId, 'uploadId');
|
||||
const token = this.expectString(req.query.token, 'token');
|
||||
const exp = this.number(req.query.exp, 'exp');
|
||||
const partNumber = this.number(req.query.partNumber, 'partNumber');
|
||||
|
||||
if (partNumber < 1) {
|
||||
throw new BlobInvalid('Invalid part number');
|
||||
}
|
||||
|
||||
this.ensureNotExpired(exp);
|
||||
|
||||
if (
|
||||
!this.verifyToken(
|
||||
PROXY_MULTIPART_PATH,
|
||||
[workspaceId, key, uploadId, partNumber],
|
||||
exp,
|
||||
token,
|
||||
signKey
|
||||
)
|
||||
) {
|
||||
throw new BlobInvalid('Invalid upload token');
|
||||
}
|
||||
|
||||
const record = await this.models.blob.get(workspaceId, key);
|
||||
if (!record) {
|
||||
throw new BlobInvalid('Multipart upload is not initialized');
|
||||
}
|
||||
if (record.status === 'completed') {
|
||||
throw new BlobInvalid('Blob upload is already completed');
|
||||
}
|
||||
if (record.uploadId !== uploadId) {
|
||||
throw new BlobInvalid('Upload id mismatch');
|
||||
}
|
||||
|
||||
const contentLength = this.parseContentLength(req);
|
||||
if (contentLength === undefined || contentLength === 0) {
|
||||
throw new BlobInvalid('Missing Content-Length header');
|
||||
}
|
||||
|
||||
const maxPartNumber = Math.ceil(record.size / MULTIPART_PART_SIZE);
|
||||
if (partNumber > maxPartNumber) {
|
||||
throw new BlobInvalid('Part number exceeds upload size');
|
||||
}
|
||||
if (
|
||||
record.size &&
|
||||
(partNumber - 1) * MULTIPART_PART_SIZE + contentLength > record.size
|
||||
) {
|
||||
throw new BlobInvalid('Part size exceeds upload metadata');
|
||||
}
|
||||
|
||||
const provider = this.getProvider(storage);
|
||||
if (!provider) {
|
||||
throw new BlobInvalid('R2 provider is not available');
|
||||
}
|
||||
|
||||
try {
|
||||
const etag = await provider.proxyUploadPart(
|
||||
`${workspaceId}/${key}`,
|
||||
uploadId,
|
||||
partNumber,
|
||||
req,
|
||||
{ contentLength }
|
||||
);
|
||||
if (etag) {
|
||||
res.setHeader('etag', etag);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to proxy multipart upload', error as Error);
|
||||
throw new BlobInvalid('Upload failed');
|
||||
}
|
||||
|
||||
res.status(200).end();
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
@@ -27,6 +29,17 @@ declare global {
|
||||
}
|
||||
}
|
||||
|
||||
type BlobCompleteResult =
|
||||
| { ok: true; metadata: GetObjectMetadata }
|
||||
| {
|
||||
ok: false;
|
||||
reason:
|
||||
| 'not_found'
|
||||
| 'size_mismatch'
|
||||
| 'mime_mismatch'
|
||||
| 'checksum_mismatch';
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceBlobStorage {
|
||||
private readonly logger = new Logger(WorkspaceBlobStorage.name);
|
||||
@@ -71,6 +84,142 @@ export class WorkspaceBlobStorage {
|
||||
return this.provider.get(`${workspaceId}/${key}`, signedUrl);
|
||||
}
|
||||
|
||||
async presignPut(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
metadata?: PutObjectMetadata
|
||||
) {
|
||||
return this.provider.presignPut?.(`${workspaceId}/${key}`, metadata);
|
||||
}
|
||||
|
||||
async createMultipartUpload(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
metadata?: PutObjectMetadata
|
||||
) {
|
||||
return this.provider.createMultipartUpload?.(
|
||||
`${workspaceId}/${key}`,
|
||||
metadata
|
||||
);
|
||||
}
|
||||
|
||||
async presignUploadPart(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
uploadId: string,
|
||||
partNumber: number
|
||||
) {
|
||||
return this.provider.presignUploadPart?.(
|
||||
`${workspaceId}/${key}`,
|
||||
uploadId,
|
||||
partNumber
|
||||
);
|
||||
}
|
||||
|
||||
async listMultipartUploadParts(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
uploadId: string
|
||||
) {
|
||||
return this.provider.listMultipartUploadParts?.(
|
||||
`${workspaceId}/${key}`,
|
||||
uploadId
|
||||
);
|
||||
}
|
||||
|
||||
async completeMultipartUpload(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: { partNumber: number; etag: string }[]
|
||||
) {
|
||||
if (!this.provider.completeMultipartUpload) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await this.provider.completeMultipartUpload(
|
||||
`${workspaceId}/${key}`,
|
||||
uploadId,
|
||||
parts
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
async abortMultipartUpload(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
uploadId: string
|
||||
) {
|
||||
if (!this.provider.abortMultipartUpload) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await this.provider.abortMultipartUpload(`${workspaceId}/${key}`, uploadId);
|
||||
return true;
|
||||
}
|
||||
|
||||
async head(workspaceId: string, key: string) {
|
||||
return this.provider.head(`${workspaceId}/${key}`);
|
||||
}
|
||||
|
||||
async complete(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
expected: { size: number; mime: string }
|
||||
): Promise<BlobCompleteResult> {
|
||||
const metadata = await this.head(workspaceId, key);
|
||||
if (!metadata) {
|
||||
return { ok: false, reason: 'not_found' };
|
||||
}
|
||||
|
||||
if (metadata.contentLength !== expected.size) {
|
||||
return { ok: false, reason: 'size_mismatch' };
|
||||
}
|
||||
|
||||
if (expected.mime && metadata.contentType !== expected.mime) {
|
||||
return { ok: false, reason: 'mime_mismatch' };
|
||||
}
|
||||
|
||||
const object = await this.provider.get(`${workspaceId}/${key}`);
|
||||
if (!object.body) {
|
||||
return { ok: false, reason: 'not_found' };
|
||||
}
|
||||
|
||||
const checksum = createHash('sha256');
|
||||
try {
|
||||
for await (const chunk of object.body) {
|
||||
checksum.update(chunk as Buffer);
|
||||
}
|
||||
} catch (e) {
|
||||
this.logger.error('failed to read blob for checksum verification', e);
|
||||
return { ok: false, reason: 'checksum_mismatch' };
|
||||
}
|
||||
|
||||
const base64 = checksum.digest('base64');
|
||||
const base64urlWithPadding = base64.replace(/\+/g, '-').replace(/\//g, '_');
|
||||
|
||||
if (base64urlWithPadding !== key) {
|
||||
try {
|
||||
await this.provider.delete(`${workspaceId}/${key}`);
|
||||
} catch (e) {
|
||||
// never throw
|
||||
this.logger.error('failed to delete invalid blob', e);
|
||||
}
|
||||
return { ok: false, reason: 'checksum_mismatch' };
|
||||
}
|
||||
|
||||
await this.models.blob.upsert({
|
||||
workspaceId,
|
||||
key,
|
||||
mime: metadata.contentType,
|
||||
size: metadata.contentLength,
|
||||
status: 'completed',
|
||||
uploadId: null,
|
||||
});
|
||||
|
||||
return { ok: true, metadata };
|
||||
}
|
||||
|
||||
async list(workspaceId: string, syncBlobMeta = true) {
|
||||
const blobsInDb = await this.models.blob.list(workspaceId);
|
||||
|
||||
@@ -78,6 +227,12 @@ export class WorkspaceBlobStorage {
|
||||
return blobsInDb;
|
||||
}
|
||||
|
||||
// all blobs are uploading but not completed yet
|
||||
const hasDbBlobs = await this.models.blob.hasAny(workspaceId);
|
||||
if (hasDbBlobs) {
|
||||
return blobsInDb;
|
||||
}
|
||||
|
||||
const blobs = await this.provider.list(workspaceId + '/');
|
||||
blobs.forEach(blob => {
|
||||
blob.key = blob.key.slice(workspaceId.length + 1);
|
||||
@@ -147,6 +302,8 @@ export class WorkspaceBlobStorage {
|
||||
key,
|
||||
mime: meta.contentType,
|
||||
size: meta.contentLength,
|
||||
status: 'completed',
|
||||
uploadId: null,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -2,29 +2,110 @@ import { Logger, UseGuards } from '@nestjs/common';
|
||||
import {
|
||||
Args,
|
||||
Field,
|
||||
InputType,
|
||||
Int,
|
||||
Mutation,
|
||||
ObjectType,
|
||||
Parent,
|
||||
Query,
|
||||
registerEnumType,
|
||||
ResolveField,
|
||||
Resolver,
|
||||
} from '@nestjs/graphql';
|
||||
import { GraphQLJSONObject } from 'graphql-scalars';
|
||||
import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';
|
||||
|
||||
import type { FileUpload } from '../../../base';
|
||||
import {
|
||||
BlobInvalid,
|
||||
BlobNotFound,
|
||||
BlobQuotaExceeded,
|
||||
CloudThrottlerGuard,
|
||||
readBuffer,
|
||||
StorageQuotaExceeded,
|
||||
} from '../../../base';
|
||||
import { Models } from '../../../models';
|
||||
import { CurrentUser } from '../../auth';
|
||||
import { AccessController } from '../../permission';
|
||||
import { QuotaService } from '../../quota';
|
||||
import { WorkspaceBlobStorage } from '../../storage';
|
||||
import {
|
||||
MULTIPART_PART_SIZE,
|
||||
MULTIPART_THRESHOLD,
|
||||
} from '../../storage/constants';
|
||||
import { WorkspaceBlobSizes, WorkspaceType } from '../types';
|
||||
|
||||
enum BlobUploadMethod {
|
||||
GRAPHQL = 'GRAPHQL',
|
||||
PRESIGNED = 'PRESIGNED',
|
||||
MULTIPART = 'MULTIPART',
|
||||
}
|
||||
|
||||
registerEnumType(BlobUploadMethod, {
|
||||
name: 'BlobUploadMethod',
|
||||
description: 'Blob upload method',
|
||||
});
|
||||
|
||||
@ObjectType()
|
||||
class BlobUploadedPart {
|
||||
@Field(() => Int)
|
||||
partNumber!: number;
|
||||
|
||||
@Field()
|
||||
etag!: string;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class BlobUploadInit {
|
||||
@Field(() => BlobUploadMethod)
|
||||
method!: BlobUploadMethod;
|
||||
|
||||
@Field()
|
||||
blobKey!: string;
|
||||
|
||||
@Field(() => Boolean, { nullable: true })
|
||||
alreadyUploaded?: boolean;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
uploadUrl?: string;
|
||||
|
||||
@Field(() => GraphQLJSONObject, { nullable: true })
|
||||
headers?: Record<string, string>;
|
||||
|
||||
@Field(() => Date, { nullable: true })
|
||||
expiresAt?: Date;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
uploadId?: string;
|
||||
|
||||
@Field(() => Int, { nullable: true })
|
||||
partSize?: number;
|
||||
|
||||
@Field(() => [BlobUploadedPart], { nullable: true })
|
||||
uploadedParts?: BlobUploadedPart[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class BlobUploadPart {
|
||||
@Field()
|
||||
uploadUrl!: string;
|
||||
|
||||
@Field(() => GraphQLJSONObject, { nullable: true })
|
||||
headers?: Record<string, string>;
|
||||
|
||||
@Field(() => Date, { nullable: true })
|
||||
expiresAt?: Date;
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class BlobUploadPartInput {
|
||||
@Field(() => Int)
|
||||
partNumber!: number;
|
||||
|
||||
@Field()
|
||||
etag!: string;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class ListedBlob {
|
||||
@Field()
|
||||
@@ -47,7 +128,8 @@ export class WorkspaceBlobResolver {
|
||||
constructor(
|
||||
private readonly ac: AccessController,
|
||||
private readonly quota: QuotaService,
|
||||
private readonly storage: WorkspaceBlobStorage
|
||||
private readonly storage: WorkspaceBlobStorage,
|
||||
private readonly models: Models
|
||||
) {}
|
||||
|
||||
@ResolveField(() => [ListedBlob], {
|
||||
@@ -110,6 +192,258 @@ export class WorkspaceBlobResolver {
|
||||
return blob.filename;
|
||||
}
|
||||
|
||||
@Mutation(() => BlobUploadInit)
|
||||
async createBlobUpload(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('workspaceId') workspaceId: string,
|
||||
@Args('key') key: string,
|
||||
@Args('size', { type: () => Int }) size: number,
|
||||
@Args('mime') mime: string
|
||||
): Promise<BlobUploadInit> {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspaceId)
|
||||
.assert('Workspace.Blobs.Write');
|
||||
|
||||
let record = await this.models.blob.get(workspaceId, key);
|
||||
mime = mime || 'application/octet-stream';
|
||||
if (record) {
|
||||
if (record.size !== size) {
|
||||
throw new BlobInvalid('Blob size mismatch');
|
||||
}
|
||||
if (record.mime !== mime) {
|
||||
throw new BlobInvalid('Blob mime mismatch');
|
||||
}
|
||||
|
||||
if (record.status === 'completed') {
|
||||
const existingMetadata = await this.storage.head(workspaceId, key);
|
||||
if (!existingMetadata) {
|
||||
// record exists but object is missing, treat as a new upload
|
||||
record = null;
|
||||
} else if (existingMetadata.contentLength !== size) {
|
||||
throw new BlobInvalid('Blob size mismatch');
|
||||
} else if (existingMetadata.contentType !== mime) {
|
||||
throw new BlobInvalid('Blob mime mismatch');
|
||||
} else {
|
||||
return {
|
||||
method: BlobUploadMethod.GRAPHQL,
|
||||
blobKey: key,
|
||||
alreadyUploaded: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const checkExceeded =
|
||||
await this.quota.getWorkspaceQuotaCalculator(workspaceId);
|
||||
const result = checkExceeded(record ? 0 : size);
|
||||
if (result?.blobQuotaExceeded) {
|
||||
throw new BlobQuotaExceeded();
|
||||
} else if (result?.storageQuotaExceeded) {
|
||||
throw new StorageQuotaExceeded();
|
||||
}
|
||||
|
||||
const metadata = { contentType: mime, contentLength: size };
|
||||
let init: BlobUploadInit | null = null;
|
||||
let uploadIdForRecord: string | null = null;
|
||||
|
||||
// try to resume multipart uploads
|
||||
if (record && record.uploadId) {
|
||||
const uploadedParts = await this.storage.listMultipartUploadParts(
|
||||
workspaceId,
|
||||
key,
|
||||
record.uploadId
|
||||
);
|
||||
|
||||
if (uploadedParts) {
|
||||
return {
|
||||
method: BlobUploadMethod.MULTIPART,
|
||||
blobKey: key,
|
||||
uploadId: record.uploadId,
|
||||
partSize: MULTIPART_PART_SIZE,
|
||||
uploadedParts,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (size >= MULTIPART_THRESHOLD) {
|
||||
const multipart = await this.storage.createMultipartUpload(
|
||||
workspaceId,
|
||||
key,
|
||||
metadata
|
||||
);
|
||||
if (multipart) {
|
||||
uploadIdForRecord = multipart.uploadId;
|
||||
init = {
|
||||
method: BlobUploadMethod.MULTIPART,
|
||||
blobKey: key,
|
||||
uploadId: multipart.uploadId,
|
||||
partSize: MULTIPART_PART_SIZE,
|
||||
expiresAt: multipart.expiresAt,
|
||||
uploadedParts: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (!init) {
|
||||
const presigned = await this.storage.presignPut(
|
||||
workspaceId,
|
||||
key,
|
||||
metadata
|
||||
);
|
||||
if (presigned) {
|
||||
init = {
|
||||
method: BlobUploadMethod.PRESIGNED,
|
||||
blobKey: key,
|
||||
uploadUrl: presigned.url,
|
||||
headers: presigned.headers,
|
||||
expiresAt: presigned.expiresAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (!init) {
|
||||
init = {
|
||||
method: BlobUploadMethod.GRAPHQL,
|
||||
blobKey: key,
|
||||
};
|
||||
}
|
||||
|
||||
await this.models.blob.upsert({
|
||||
workspaceId,
|
||||
key,
|
||||
mime,
|
||||
size,
|
||||
status: 'pending',
|
||||
uploadId: uploadIdForRecord,
|
||||
});
|
||||
|
||||
return init;
|
||||
}
|
||||
|
||||
@Mutation(() => String)
|
||||
async completeBlobUpload(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('workspaceId') workspaceId: string,
|
||||
@Args('key') key: string,
|
||||
@Args('uploadId', { nullable: true }) uploadId?: string,
|
||||
@Args({
|
||||
name: 'parts',
|
||||
type: () => [BlobUploadPartInput],
|
||||
nullable: true,
|
||||
})
|
||||
parts?: BlobUploadPartInput[]
|
||||
): Promise<string> {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspaceId)
|
||||
.assert('Workspace.Blobs.Write');
|
||||
|
||||
const record = await this.models.blob.get(workspaceId, key);
|
||||
if (!record) {
|
||||
throw new BlobNotFound({ spaceId: workspaceId, blobId: key });
|
||||
}
|
||||
if (record.status === 'completed') {
|
||||
return key;
|
||||
}
|
||||
|
||||
const hasMultipartInput =
|
||||
uploadId !== undefined || (parts?.length ?? 0) > 0;
|
||||
const hasMultipartRecord = !!record.uploadId;
|
||||
if (hasMultipartRecord) {
|
||||
if (!uploadId || !parts || parts.length === 0) {
|
||||
throw new BlobInvalid(
|
||||
'Multipart upload requires both uploadId and parts'
|
||||
);
|
||||
}
|
||||
if (uploadId !== record.uploadId) {
|
||||
throw new BlobInvalid('Upload id mismatch');
|
||||
}
|
||||
|
||||
const metadata = await this.storage.head(workspaceId, key);
|
||||
if (!metadata) {
|
||||
const completed = await this.storage.completeMultipartUpload(
|
||||
workspaceId,
|
||||
key,
|
||||
uploadId,
|
||||
parts
|
||||
);
|
||||
if (!completed) {
|
||||
throw new BlobInvalid('Multipart upload is not supported');
|
||||
}
|
||||
}
|
||||
} else if (hasMultipartInput) {
|
||||
throw new BlobInvalid('Multipart upload is not initialized');
|
||||
}
|
||||
|
||||
const result = await this.storage.complete(workspaceId, key, {
|
||||
size: record.size,
|
||||
mime: record.mime,
|
||||
});
|
||||
if (!result.ok) {
|
||||
if (result.reason === 'not_found') {
|
||||
throw new BlobNotFound({
|
||||
spaceId: workspaceId,
|
||||
blobId: key,
|
||||
});
|
||||
}
|
||||
if (result.reason === 'size_mismatch') {
|
||||
throw new BlobInvalid('Blob size mismatch');
|
||||
}
|
||||
if (result.reason === 'mime_mismatch') {
|
||||
throw new BlobInvalid('Blob mime mismatch');
|
||||
}
|
||||
throw new BlobInvalid('Blob key mismatch');
|
||||
}
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
@Mutation(() => BlobUploadPart)
|
||||
async getBlobUploadPartUrl(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('workspaceId') workspaceId: string,
|
||||
@Args('key') key: string,
|
||||
@Args('uploadId') uploadId: string,
|
||||
@Args('partNumber', { type: () => Int }) partNumber: number
|
||||
): Promise<BlobUploadPart> {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspaceId)
|
||||
.assert('Workspace.Blobs.Write');
|
||||
|
||||
const part = await this.storage.presignUploadPart(
|
||||
workspaceId,
|
||||
key,
|
||||
uploadId,
|
||||
partNumber
|
||||
);
|
||||
if (!part) {
|
||||
throw new BlobInvalid('Multipart upload is not supported');
|
||||
}
|
||||
|
||||
return {
|
||||
uploadUrl: part.url,
|
||||
headers: part.headers,
|
||||
expiresAt: part.expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
@Mutation(() => Boolean)
|
||||
async abortBlobUpload(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('workspaceId') workspaceId: string,
|
||||
@Args('key') key: string,
|
||||
@Args('uploadId') uploadId: string
|
||||
) {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspaceId)
|
||||
.assert('Workspace.Blobs.Write');
|
||||
|
||||
return this.storage.abortMultipartUpload(workspaceId, key, uploadId);
|
||||
}
|
||||
|
||||
@Mutation(() => Boolean)
|
||||
async deleteBlob(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
|
||||
Reference in New Issue
Block a user