feat(server): add image resize support (#14588)

fix #13842 


#### PR Dependency Tree


* **PR #14588** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Images are now processed natively and converted to WebP for smaller,
optimized files; Copilot and avatar attachments use the processed WebP
output.
* Avatar uploads accept BMP, GIF, JPEG, PNG, WebP (5MB max) and are
downscaled to a standard edge.

* **Error Messages / i18n**
  * Added localized error "Image format not supported: {format}".

* **Tests**
* Added end-to-end and unit tests for conversion, EXIF preservation, and
upload limits.

* **Chores**
  * Added native image-processing dependencies.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-03-07 04:42:12 +08:00
committed by GitHub
parent f34e25e122
commit 86d65b2f64
20 changed files with 743 additions and 29 deletions
@@ -16,6 +16,7 @@ import {
CopilotEmbeddingJob,
MockEmbeddingClient,
} from '../../plugins/copilot/embedding';
import { ChatMessageCache } from '../../plugins/copilot/message';
import { prompts, PromptService } from '../../plugins/copilot/prompt';
import {
CopilotProviderFactory,
@@ -416,6 +417,7 @@ test('should be able to use test provider', async t => {
test('should create message correctly', async t => {
const { app } = t.context;
const messageCache = app.get(ChatMessageCache);
{
const { id } = await createWorkspace(app);
@@ -463,6 +465,19 @@ test('should create message correctly', async t => {
new File([new Uint8Array(pngData)], '1.png', { type: 'image/png' })
);
t.truthy(messageId, 'should be able to create message with blob');
const message = await messageCache.get(messageId);
const attachment = message?.attachments?.[0] as
| { attachment: string; mimeType: string }
| undefined;
const payload = Buffer.from(
attachment?.attachment.split(',').at(1) || '',
'base64'
);
t.is(attachment?.mimeType, 'image/webp');
t.is(payload.subarray(0, 4).toString('ascii'), 'RIFF');
t.is(payload.subarray(8, 12).toString('ascii'), 'WEBP');
}
// with attachments
@@ -4,9 +4,9 @@ import type { TestFn } from 'ava';
import ava from 'ava';
import {
createBmp,
createTestingApp,
getPublicUserById,
smallestGif,
smallestPng,
TestingApp,
updateAvatar,
@@ -40,7 +40,10 @@ test('should be able to upload user avatar', async t => {
const avatarRes = await app.GET(new URL(avatarUrl).pathname);
t.deepEqual(avatarRes.body, avatar);
t.true(avatarRes.headers['content-type'].startsWith('image/webp'));
t.notDeepEqual(avatarRes.body, avatar);
t.is(avatarRes.body.subarray(0, 4).toString('ascii'), 'RIFF');
t.is(avatarRes.body.subarray(8, 12).toString('ascii'), 'WEBP');
});
test('should be able to update user avatar, and invalidate old avatar url', async t => {
@@ -54,9 +57,7 @@ test('should be able to update user avatar, and invalidate old avatar url', asyn
const oldAvatarUrl = res.body.data.uploadAvatar.avatarUrl;
const newAvatar = await fetch(smallestGif)
.then(res => res.arrayBuffer())
.then(b => Buffer.from(b));
const newAvatar = createBmp(32, 32);
res = await updateAvatar(app, newAvatar);
const newAvatarUrl = res.body.data.uploadAvatar.avatarUrl;
@@ -66,7 +67,46 @@ test('should be able to update user avatar, and invalidate old avatar url', asyn
t.is(avatarRes.status, 404);
const newAvatarRes = await app.GET(new URL(newAvatarUrl).pathname);
t.deepEqual(newAvatarRes.body, newAvatar);
t.true(newAvatarRes.headers['content-type'].startsWith('image/webp'));
t.notDeepEqual(newAvatarRes.body, newAvatar);
t.is(newAvatarRes.body.subarray(0, 4).toString('ascii'), 'RIFF');
t.is(newAvatarRes.body.subarray(8, 12).toString('ascii'), 'WEBP');
});
test('should accept avatar uploads up to 5MB after conversion', async t => {
const { app } = t.context;
await app.signup();
const avatar = createBmp(1024, 1024);
t.true(avatar.length > 500 * 1024);
t.true(avatar.length < 5 * 1024 * 1024);
const res = await updateAvatar(app, avatar, {
filename: 'large.bmp',
contentType: 'image/bmp',
});
t.is(res.status, 200);
const avatarUrl = res.body.data.uploadAvatar.avatarUrl;
const avatarRes = await app.GET(new URL(avatarUrl).pathname);
t.true(avatarRes.headers['content-type'].startsWith('image/webp'));
});
test('should reject unsupported vector avatars', async t => {
const { app } = t.context;
await app.signup();
const avatar = Buffer.from(
'<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10"></svg>'
);
const res = await updateAvatar(app, avatar, {
filename: 'avatar.svg',
contentType: 'image/svg+xml',
});
t.is(res.status, 200);
t.is(res.body.errors[0].message, 'Image format not supported: image/svg+xml');
});
test('should be able to get public user by id', async t => {
@@ -7,6 +7,35 @@ export const smallestPng =
'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII';
export const smallestGif = 'data:image/gif;base64,R0lGODlhAQABAAAAACw=';
export function createBmp(width: number, height: number) {
const rowSize = Math.ceil((width * 3) / 4) * 4;
const pixelDataSize = rowSize * height;
const fileSize = 54 + pixelDataSize;
const buffer = Buffer.alloc(fileSize);
buffer.write('BM', 0, 'ascii');
buffer.writeUInt32LE(fileSize, 2);
buffer.writeUInt32LE(54, 10);
buffer.writeUInt32LE(40, 14);
buffer.writeInt32LE(width, 18);
buffer.writeInt32LE(height, 22);
buffer.writeUInt16LE(1, 26);
buffer.writeUInt16LE(24, 28);
buffer.writeUInt32LE(pixelDataSize, 34);
for (let y = 0; y < height; y++) {
const rowOffset = 54 + y * rowSize;
for (let x = 0; x < width; x++) {
const pixelOffset = rowOffset + x * 3;
buffer[pixelOffset] = 0x33;
buffer[pixelOffset + 1] = 0x66;
buffer[pixelOffset + 2] = 0x99;
}
}
return buffer;
}
export async function listBlobs(
app: TestingApp,
workspaceId: string
@@ -121,7 +121,11 @@ export async function deleteAccount(app: TestingApp) {
return res.deleteAccount.success;
}
export async function updateAvatar(app: TestingApp, avatar: Buffer) {
export async function updateAvatar(
app: TestingApp,
avatar: Buffer,
options: { filename?: string; contentType?: string } = {}
) {
return app
.POST('/graphql')
.field(
@@ -138,7 +142,7 @@ export async function updateAvatar(app: TestingApp, avatar: Buffer) {
)
.field('map', JSON.stringify({ '0': ['variables.avatar'] }))
.attach('0', avatar, {
filename: 'test.png',
contentType: 'image/png',
filename: options.filename || 'test.png',
contentType: options.contentType || 'image/png',
});
}
@@ -301,6 +301,11 @@ export const USER_FRIENDLY_ERRORS = {
},
// Input errors
image_format_not_supported: {
type: 'invalid_input',
args: { format: 'string' },
message: ({ format }) => `Image format not supported: ${format}`,
},
query_too_long: {
type: 'invalid_input',
args: { max: 'number' },
@@ -82,6 +82,16 @@ export class EmailServiceNotConfigured extends UserFriendlyError {
}
}
@ObjectType()
class ImageFormatNotSupportedDataType {
@Field() format!: string
}
export class ImageFormatNotSupported extends UserFriendlyError {
constructor(args: ImageFormatNotSupportedDataType, message?: string | ((args: ImageFormatNotSupportedDataType) => string)) {
super('invalid_input', 'image_format_not_supported', message, args);
}
}
@ObjectType()
class QueryTooLongDataType {
@Field() max!: number
}
@@ -1155,6 +1165,7 @@ export enum ErrorNames {
SSRF_BLOCKED_ERROR,
RESPONSE_TOO_LARGE_ERROR,
EMAIL_SERVICE_NOT_CONFIGURED,
IMAGE_FORMAT_NOT_SUPPORTED,
QUERY_TOO_LONG,
VALIDATION_ERROR,
USER_NOT_FOUND,
@@ -1297,5 +1308,5 @@ registerEnumType(ErrorNames, {
export const ErrorDataUnionType = createUnionType({
name: 'ErrorDataUnion',
types: () =>
[GraphqlBadRequestDataType, HttpRequestErrorDataType, SsrfBlockedErrorDataType, ResponseTooLargeErrorDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidOauthResponseDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CalendarProviderRequestErrorDataType, NoCopilotProviderAvailableDataType, CopilotFailedToGenerateEmbeddingDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderNotSupportedDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, MentionUserDocAccessDeniedDataType, InvalidAppConfigDataType, InvalidAppConfigInputDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const,
[GraphqlBadRequestDataType, HttpRequestErrorDataType, SsrfBlockedErrorDataType, ResponseTooLargeErrorDataType, ImageFormatNotSupportedDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidOauthResponseDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CalendarProviderRequestErrorDataType, NoCopilotProviderAvailableDataType, CopilotFailedToGenerateEmbeddingDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderNotSupportedDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, MentionUserDocAccessDeniedDataType, InvalidAppConfigDataType, InvalidAppConfigInputDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const,
});
@@ -17,6 +17,8 @@ import { isNil, omitBy } from 'lodash-es';
import {
CannotDeleteOwnAccount,
type FileUpload,
ImageFormatNotSupported,
OneMB,
readBufferWithLimit,
sniffMime,
Throttle,
@@ -28,6 +30,7 @@ import {
UserFeatureName,
UserSettingsSchema,
} from '../../models';
import { processImage } from '../../native';
import { Public } from '../auth/guard';
import { sessionUser } from '../auth/service';
import { CurrentUser } from '../auth/session';
@@ -115,16 +118,26 @@ export class UserResolver {
throw new UserNotFound();
}
const avatarBuffer = await readBufferWithLimit(avatar.createReadStream());
const contentType = sniffMime(avatarBuffer, avatar.mimetype);
const avatarBuffer = await readBufferWithLimit(
avatar.createReadStream(),
5 * OneMB
);
const contentType = sniffMime(avatarBuffer, avatar.mimetype)?.toLowerCase();
if (!contentType || !contentType.startsWith('image/')) {
throw new Error(`Invalid file type: ${contentType || 'unknown'}`);
throw new ImageFormatNotSupported({ format: contentType || 'unknown' });
}
let processedAvatarBuffer: Buffer;
try {
processedAvatarBuffer = await processImage(avatarBuffer, 512, false);
} catch {
throw new ImageFormatNotSupported({ format: contentType });
}
const avatarUrl = await this.storage.put(
`${user.id}-avatar-${Date.now()}`,
avatarBuffer,
{ contentType }
processedAvatarBuffer,
{ contentType: 'image/webp' }
);
if (user.avatarUrl) {
+1
View File
@@ -40,6 +40,7 @@ export function getTokenEncoder(model?: string | null): Tokenizer | null {
export const getMime = serverNativeModule.getMime;
export const parseDoc = serverNativeModule.parseDoc;
export const htmlSanitize = serverNativeModule.htmlSanitize;
export const processImage = serverNativeModule.processImage;
export const parseYDocFromBinary = serverNativeModule.parseDocFromBinary;
export const parseYDocToMarkdown = serverNativeModule.parseDocToMarkdown;
export const parsePageDocFromBinary = serverNativeModule.parsePageDoc;
@@ -24,6 +24,7 @@ import {
CopilotProviderSideError,
CopilotSessionNotFound,
type FileUpload,
ImageFormatNotSupported,
paginate,
Paginated,
PaginationInput,
@@ -39,6 +40,7 @@ import { DocReader } from '../../core/doc';
import { AccessController, DocAction } from '../../core/permission';
import { UserType } from '../../core/user';
import type { ListSessionOptions, UpdateChatSession } from '../../models';
import { processImage } from '../../native';
import { CopilotCronJobs } from './cron';
import { PromptService } from './prompt/service';
import { CopilotProviderFactory } from './providers/factory';
@@ -48,6 +50,7 @@ import { CopilotStorage } from './storage';
import { type ChatHistory, type ChatMessage, SubmittedMessage } from './types';
export const COPILOT_LOCKER = 'copilot';
const COPILOT_IMAGE_MAX_EDGE = 1536;
// ================== Input Types ==================
@@ -777,19 +780,35 @@ export class CopilotResolver {
for (const blob of blobs) {
const uploaded = await this.storage.handleUpload(user.id, blob);
const detectedMime =
sniffMime(uploaded.buffer, blob.mimetype)?.toLowerCase() ||
blob.mimetype;
let attachmentBuffer = uploaded.buffer;
let attachmentMimeType = detectedMime;
if (detectedMime.startsWith('image/')) {
try {
attachmentBuffer = await processImage(
uploaded.buffer,
COPILOT_IMAGE_MAX_EDGE,
true
);
attachmentMimeType = 'image/webp';
} catch {
throw new ImageFormatNotSupported({ format: detectedMime });
}
}
const filename = createHash('sha256')
.update(uploaded.buffer)
.update(attachmentBuffer)
.digest('base64url');
const attachment = await this.storage.put(
user.id,
workspaceId,
filename,
uploaded.buffer
attachmentBuffer
);
attachments.push({
attachment,
mimeType: sniffMime(uploaded.buffer, blob.mimetype) || blob.mimetype,
});
attachments.push({ attachment, mimeType: attachmentMimeType });
}
}
+6 -1
View File
@@ -877,7 +877,7 @@ type EditorType {
name: String!
}
union ErrorDataUnion = AlreadyInSpaceDataType | BlobNotFoundDataType | CalendarProviderRequestErrorDataType | CopilotContextFileNotSupportedDataType | CopilotDocNotFoundDataType | CopilotFailedToAddWorkspaceFileEmbeddingDataType | CopilotFailedToGenerateEmbeddingDataType | CopilotFailedToMatchContextDataType | CopilotFailedToMatchGlobalContextDataType | CopilotFailedToModifyContextDataType | CopilotInvalidContextDataType | CopilotMessageNotFoundDataType | CopilotPromptNotFoundDataType | CopilotProviderNotSupportedDataType | CopilotProviderSideErrorDataType | DocActionDeniedDataType | DocHistoryNotFoundDataType | DocNotFoundDataType | DocUpdateBlockedDataType | ExpectToGrantDocUserRolesDataType | ExpectToRevokeDocUserRolesDataType | ExpectToUpdateDocUserRoleDataType | GraphqlBadRequestDataType | HttpRequestErrorDataType | InvalidAppConfigDataType | InvalidAppConfigInputDataType | InvalidEmailDataType | InvalidHistoryTimestampDataType | InvalidIndexerInputDataType | InvalidLicenseToActivateDataType | InvalidLicenseUpdateParamsDataType | InvalidOauthCallbackCodeDataType | InvalidOauthResponseDataType | InvalidPasswordLengthDataType | InvalidRuntimeConfigTypeDataType | InvalidSearchProviderRequestDataType | MemberNotFoundInSpaceDataType | MentionUserDocAccessDeniedDataType | MissingOauthQueryParameterDataType | NoCopilotProviderAvailableDataType | NoMoreSeatDataType | NotInSpaceDataType | QueryTooLongDataType | ResponseTooLargeErrorDataType | RuntimeConfigNotFoundDataType | SameSubscriptionRecurringDataType | SpaceAccessDeniedDataType | SpaceNotFoundDataType | SpaceOwnerNotFoundDataType | SpaceShouldHaveOnlyOneOwnerDataType | SsrfBlockedErrorDataType | SubscriptionAlreadyExistsDataType | SubscriptionNotExistsDataType | SubscriptionPlanNotFoundDataType | UnknownOauthProviderDataType | UnsupportedClientVersionDataType | UnsupportedSubscriptionPlanDataType | ValidationErrorDataType | VersionRejectedDataType | WorkspacePermissionNotFoundDataType | WrongSignInCredentialsDataType
union ErrorDataUnion = AlreadyInSpaceDataType | BlobNotFoundDataType | CalendarProviderRequestErrorDataType | CopilotContextFileNotSupportedDataType | CopilotDocNotFoundDataType | CopilotFailedToAddWorkspaceFileEmbeddingDataType | CopilotFailedToGenerateEmbeddingDataType | CopilotFailedToMatchContextDataType | CopilotFailedToMatchGlobalContextDataType | CopilotFailedToModifyContextDataType | CopilotInvalidContextDataType | CopilotMessageNotFoundDataType | CopilotPromptNotFoundDataType | CopilotProviderNotSupportedDataType | CopilotProviderSideErrorDataType | DocActionDeniedDataType | DocHistoryNotFoundDataType | DocNotFoundDataType | DocUpdateBlockedDataType | ExpectToGrantDocUserRolesDataType | ExpectToRevokeDocUserRolesDataType | ExpectToUpdateDocUserRoleDataType | GraphqlBadRequestDataType | HttpRequestErrorDataType | ImageFormatNotSupportedDataType | InvalidAppConfigDataType | InvalidAppConfigInputDataType | InvalidEmailDataType | InvalidHistoryTimestampDataType | InvalidIndexerInputDataType | InvalidLicenseToActivateDataType | InvalidLicenseUpdateParamsDataType | InvalidOauthCallbackCodeDataType | InvalidOauthResponseDataType | InvalidPasswordLengthDataType | InvalidRuntimeConfigTypeDataType | InvalidSearchProviderRequestDataType | MemberNotFoundInSpaceDataType | MentionUserDocAccessDeniedDataType | MissingOauthQueryParameterDataType | NoCopilotProviderAvailableDataType | NoMoreSeatDataType | NotInSpaceDataType | QueryTooLongDataType | ResponseTooLargeErrorDataType | RuntimeConfigNotFoundDataType | SameSubscriptionRecurringDataType | SpaceAccessDeniedDataType | SpaceNotFoundDataType | SpaceOwnerNotFoundDataType | SpaceShouldHaveOnlyOneOwnerDataType | SsrfBlockedErrorDataType | SubscriptionAlreadyExistsDataType | SubscriptionNotExistsDataType | SubscriptionPlanNotFoundDataType | UnknownOauthProviderDataType | UnsupportedClientVersionDataType | UnsupportedSubscriptionPlanDataType | ValidationErrorDataType | VersionRejectedDataType | WorkspacePermissionNotFoundDataType | WrongSignInCredentialsDataType
enum ErrorNames {
ACCESS_DENIED
@@ -947,6 +947,7 @@ enum ErrorNames {
FAILED_TO_UPSERT_SNAPSHOT
GRAPHQL_BAD_REQUEST
HTTP_REQUEST_ERROR
IMAGE_FORMAT_NOT_SUPPORTED
INTERNAL_SERVER_ERROR
INVALID_APP_CONFIG
INVALID_APP_CONFIG_INPUT
@@ -1095,6 +1096,10 @@ type HttpRequestErrorDataType {
message: String!
}
type ImageFormatNotSupportedDataType {
format: String!
}
input ImportUsersInput {
users: [CreateUserInput!]!
}