diff --git a/packages/backend/server/src/base/event/def.ts b/packages/backend/server/src/base/event/def.ts index 7f253d9343..71d0e24176 100644 --- a/packages/backend/server/src/base/event/def.ts +++ b/packages/backend/server/src/base/event/def.ts @@ -24,6 +24,10 @@ export interface WorkspaceEvents { workspaceId: Workspace['id']; key: string; }>; + sync: Payload<{ + workspaceId: Workspace['id']; + key: string; + }>; }; } diff --git a/packages/backend/server/src/base/storage/providers/fs.ts b/packages/backend/server/src/base/storage/providers/fs.ts index 41e17f3e66..081e678aca 100644 --- a/packages/backend/server/src/base/storage/providers/fs.ts +++ b/packages/backend/server/src/base/storage/providers/fs.ts @@ -61,6 +61,15 @@ export class FsStorageProvider implements StorageProvider { this.logger.verbose(`Object \`${key}\` put`); } + async head(key: string) { + const metadata = this.readMetadata(key); + if (!metadata) { + this.logger.verbose(`Object \`${key}\` not found`); + return undefined; + } + return metadata; + } + async get(key: string): Promise<{ body?: Readable; metadata?: GetObjectMetadata; diff --git a/packages/backend/server/src/base/storage/providers/provider.ts b/packages/backend/server/src/base/storage/providers/provider.ts index 5b2bad24b2..06d8009773 100644 --- a/packages/backend/server/src/base/storage/providers/provider.ts +++ b/packages/backend/server/src/base/storage/providers/provider.ts @@ -34,6 +34,7 @@ export interface StorageProvider { body: BlobInputType, metadata?: PutObjectMetadata ): Promise; + head(key: string): Promise; get( key: string ): Promise<{ body?: BlobOutputType; metadata?: GetObjectMetadata }>; diff --git a/packages/backend/server/src/core/storage/wrappers/blob.ts b/packages/backend/server/src/core/storage/wrappers/blob.ts index 2ca900af67..85e4d50ad1 100644 --- a/packages/backend/server/src/core/storage/wrappers/blob.ts +++ b/packages/backend/server/src/core/storage/wrappers/blob.ts @@ -32,7 +32,7 @@ export class WorkspaceBlobStorage { const meta: PutObjectMetadata = autoMetadata(blob); await this.provider.put(`${workspaceId}/${key}`, blob, meta); - this.trySyncBlobMeta(workspaceId, key, { + await this.upsert(workspaceId, key, { contentType: meta.contentType ?? 'application/octet-stream', contentLength: blob.length, lastModified: new Date(), @@ -119,53 +119,48 @@ export class WorkspaceBlobStorage { private trySyncBlobsMeta(workspaceId: string, blobs: ListObjectsMetadata[]) { for (const blob of blobs) { - this.trySyncBlobMeta(workspaceId, blob.key); + this.event.emit('workspace.blob.sync', { + workspaceId, + key: blob.key, + }); } } - private trySyncBlobMeta( + private async upsert( workspaceId: string, key: string, - meta?: GetObjectMetadata + meta: GetObjectMetadata ) { - setImmediate(() => { - this.syncBlobMeta(workspaceId, key, meta).catch(() => { - /* never throw */ - }); + await this.db.blob.upsert({ + where: { + workspaceId_key: { + workspaceId, + key, + }, + }, + update: { + mime: meta.contentType, + size: meta.contentLength, + }, + create: { + workspaceId, + key, + mime: meta.contentType, + size: meta.contentLength, + }, }); } - private async syncBlobMeta( - workspaceId: string, - key: string, - meta?: GetObjectMetadata - ) { + @OnEvent('workspace.blob.sync') + async syncBlobMeta({ + workspaceId, + key, + }: EventPayload<'workspace.blob.sync'>) { try { - if (!meta) { - const blob = await this.get(workspaceId, key); - meta = blob.metadata; - blob.body?.destroy(); - } + const meta = await this.provider.head(`${workspaceId}/${key}`); if (meta) { - await this.db.blob.upsert({ - where: { - workspaceId_key: { - workspaceId, - key, - }, - }, - update: { - mime: meta.contentType, - size: meta.contentLength, - }, - create: { - workspaceId, - key, - mime: meta.contentType, - size: meta.contentLength, - }, - }); + await this.upsert(workspaceId, key, meta); } else { await this.db.blob.deleteMany({ where: { diff --git a/packages/backend/server/src/plugins/storage/providers/s3.ts b/packages/backend/server/src/plugins/storage/providers/s3.ts index 2dbb819b36..2b4d693cf2 100644 --- a/packages/backend/server/src/plugins/storage/providers/s3.ts +++ b/packages/backend/server/src/plugins/storage/providers/s3.ts @@ -1,9 +1,10 @@ -/* eslint-disable @typescript-eslint/no-non-null-assertion */ +/* oxlint-disable @typescript-eslint/no-non-null-assertion */ import { Readable } from 'node:stream'; import { DeleteObjectCommand, GetObjectCommand, + HeadObjectCommand, ListObjectsV2Command, NoSuchKey, PutObjectCommand, @@ -75,6 +76,34 @@ export class S3StorageProvider implements StorageProvider { } } + async head(key: string) { + try { + const obj = await this.client.send( + new HeadObjectCommand({ + Bucket: this.bucket, + Key: key, + }) + ); + + return { + contentType: obj.ContentType!, + contentLength: obj.ContentLength!, + lastModified: obj.LastModified!, + checksumCRC32: obj.ChecksumCRC32, + }; + } catch (e) { + // 404 + if (e instanceof NoSuchKey) { + this.logger.verbose(`Object \`${key}\` not found`); + return undefined; + } else { + throw new Error(`Failed to head object \`${key}\``, { + cause: e, + }); + } + } + } + async get(key: string): Promise<{ body?: Readable; metadata?: GetObjectMetadata;