Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-20 11:36:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): native safe fetch (#14931)

Browse Source
This commit is contained in:
DarkSky
2026-05-09 02:40:25 +08:00
committed by GitHub
parent 32a94d68dc
commit bcbde16c04
30 changed files with 1370 additions and 526 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/server/src/plugins/calendar/providers/caldav.ts
+1 -7
View File
@@ -35,8 +35,6 @@ const XML_PARSER = new XMLParser({
const DEFAULT_REQUEST_TIMEOUT_MS = 10_000;
const DEFAULT_MAX_REDIRECTS = 5;
const CALDAV_ALLOWED_PROTOCOLS = new Set(['https:']);
const CALDAV_ALLOWED_PROTOCOLS_INSECURE = new Set(['http:', 'https:']);
type CalDAVCredentials = {
username: string;
@@ -607,11 +605,7 @@ class CalDAVRequestPolicy {
}
try {
await assertSsrFSafeUrl(url, {
allowedProtocols: this.allowInsecureHttp
? CALDAV_ALLOWED_PROTOCOLS_INSECURE
: CALDAV_ALLOWED_PROTOCOLS,
});
await assertSsrFSafeUrl(url);
} catch (error) {
if (error instanceof SsrfBlockedError) {
const reason = String(error.data?.reason ?? '');
packages/backend/server/src/plugins/copilot/providers/utils.ts
+4 -44
View File
@@ -2,7 +2,8 @@ import { Logger } from '@nestjs/common';
import { GoogleAuth, GoogleAuthOptions } from 'google-auth-library';
import z from 'zod';
import { OneMinute, safeFetch } from '../../../base';
import { OneMinute } from '../../../base';
import { inferRemoteMimeType } from '../../../native';
import { PromptAttachment, StreamObject } from './types';
export type VertexProviderConfig = {
@@ -33,30 +34,7 @@ type CopilotTextStreamPart =
}
| { type: 'error'; error: unknown };
const ATTACH_HEAD_PARAMS = { timeoutMs: OneMinute / 12, maxRedirects: 3 };
const FORMAT_INFER_MAP: Record<string, string> = {
pdf: 'application/pdf',
mp3: 'audio/mpeg',
opus: 'audio/opus',
ogg: 'audio/ogg',
aac: 'audio/aac',
m4a: 'audio/aac',
flac: 'audio/flac',
ogv: 'video/ogg',
wav: 'audio/wav',
png: 'image/png',
jpeg: 'image/jpeg',
jpg: 'image/jpeg',
webp: 'image/webp',
txt: 'text/plain',
md: 'text/plain',
mov: 'video/mov',
mpeg: 'video/mpeg',
mp4: 'video/mp4',
avi: 'video/avi',
wmv: 'video/wmv',
flv: 'video/flv',
};
const ATTACH_HEAD_TIMEOUT_MS = OneMinute / 12;
function toBase64Data(data: string, encoding: 'base64' | 'utf8' = 'base64') {
return encoding === 'base64'
@@ -97,25 +75,7 @@ export async function inferMimeType(url: string) {
if (url.startsWith('data:')) {
return url.split(';')[0].split(':')[1];
}
const pathname = new URL(url).pathname;
const extension = pathname.split('.').pop();
if (extension) {
const ext = FORMAT_INFER_MAP[extension];
if (ext) {
return ext;
}
}
try {
const mimeType = await safeFetch(
url,
{ method: 'HEAD' },
ATTACH_HEAD_PARAMS
).then(res => res.headers.get('content-type'));
if (mimeType) return mimeType;
} catch {
// ignore and fallback to default
}
return 'application/octet-stream';
return inferRemoteMimeType({ url, timeoutMs: ATTACH_HEAD_TIMEOUT_MS });
}
type CitationIndexedEvent = {
packages/backend/server/src/plugins/copilot/runtime/hosts/attachment-materializer.ts
+21 -29
View File
@@ -1,10 +1,7 @@
import { Injectable } from '@nestjs/common';
import {
Config,
readResponseBufferWithLimit,
safeFetch,
} from '../../../../base';
import { Config } from '../../../../base';
import { fetchRemoteAttachment } from '../../../../native';
type FetchRemoteAttachmentOptions = {
signal?: AbortSignal;
@@ -37,9 +34,12 @@ export class AttachmentMaterializer {
constructor(private readonly config: Config) {}
private buildFetchOptions(url: URL, trustedHostSuffixes: string[]) {
const baseOptions = { timeoutMs: 15_000, maxRedirects: 3 } as const;
const baseOptions = {
timeoutMs: 15_000,
allowPrivateTargetOrigin: false,
};
if (!env.prod) {
return { ...baseOptions, allowPrivateOrigins: new Set([url.origin]) };
return { ...baseOptions, allowPrivateTargetOrigin: true };
}
const trustedOrigins = new Set<string>();
@@ -80,7 +80,7 @@ export class AttachmentMaterializer {
suffix => hostname === suffix || hostname.endsWith(`.${suffix}`)
);
if (trustedOrigins.has(url.origin) || trustedByHost) {
return { ...baseOptions, allowPrivateOrigins: new Set([url.origin]) };
return { ...baseOptions, allowPrivateTargetOrigin: true };
}
return baseOptions;
@@ -91,30 +91,22 @@ export class AttachmentMaterializer {
options: FetchRemoteAttachmentOptions
) {
const parsed = resolveAttachmentFetchUrl(url);
const response = await safeFetch(
parsed,
{ method: 'GET', signal: options.signal },
this.buildFetchOptions(parsed, options.trustedHostSuffixes ?? [])
);
if (!response.ok) {
throw new Error(
`Failed to fetch attachment: ${response.status} ${response.statusText}`
);
}
const buffer = await readResponseBufferWithLimit(
response,
options.maxBytes
);
const headerMimeType = normalizeMimeType(
response.headers.get('content-type') || ''
);
options.signal?.throwIfAborted();
const response = await fetchRemoteAttachment({
url: parsed.toString(),
...this.buildFetchOptions(parsed, options.trustedHostSuffixes ?? []),
maxBytes: options.maxBytes,
});
options.signal?.throwIfAborted();
return {
data: buffer.toString('base64'),
data: response.body.toString('base64'),
mimeType: options.detectMimeType
? options.detectMimeType(buffer, headerMimeType)
: headerMimeType,
? options.detectMimeType(
response.body,
normalizeMimeType(response.mimeType)
)
: normalizeMimeType(response.mimeType),
};
}
}
packages/backend/server/src/plugins/copilot/storage.ts
+10 -7
View File
@@ -7,7 +7,6 @@ import {
BlobQuotaExceeded,
CallMetric,
Config,
fetchBuffer,
type FileUpload,
OneMB,
OnEvent,
@@ -17,6 +16,7 @@ import {
URLHelper,
} from '../../base';
import { QuotaService } from '../../core/quota';
import { fetchRemoteAttachment } from '../../native';
const REMOTE_BLOB_MAX_BYTES = 20 * OneMB;
@@ -93,12 +93,15 @@ export class CopilotStorage {
@CallMetric('ai', 'blob_proxy_remote_url')
async handleRemoteLink(userId: string, workspaceId: string, link: string) {
const { buffer, type } = await fetchBuffer(
link,
REMOTE_BLOB_MAX_BYTES,
'image/'
);
const { body, mimeType } = await fetchRemoteAttachment({
url: link,
maxBytes: REMOTE_BLOB_MAX_BYTES,
expectedContentTypePrefix: 'image/',
maxImageHeight: 4096,
maxImageWidth: 4096,
});
const buffer = Buffer.from(body);
const filename = createHash('sha256').update(buffer).digest('base64url');
return this.put(userId, workspaceId, filename, buffer, type);
return this.put(userId, workspaceId, filename, buffer, mimeType);
}
}
packages/backend/server/src/plugins/worker/controller.ts
+36 -29
View File
@@ -17,15 +17,16 @@ import {
applyAttachHeaders,
BadRequest,
Cache,
readResponseBufferWithLimit,
ResponseTooLargeError,
safeFetch,
SsrfBlockedError,
type SSRFBlockReason,
Throttle,
URLHelper,
UseNamedGuard,
} from '../../base';
import { Public } from '../../core/auth';
import { inspectImageForProxy } from '../../native';
import { WorkerService } from './service';
import type { LinkPreviewRequest, LinkPreviewResponse } from './types';
import {
@@ -64,6 +65,7 @@ function toBadRequestReason(reason: SSRFBlockReason) {
@Public()
@UseNamedGuard('selfhost')
@Throttle('default', { limit: 60, ttl: 60_000 })
@Controller('/api/worker')
export class WorkerController {
private readonly logger = new Logger(WorkerController.name);
@@ -133,7 +135,8 @@ export class WorkerController {
...(origin ? { Vary: 'Origin' } : {}),
'Access-Control-Allow-Methods': 'GET',
});
applyAttachHeaders(resp, { buffer });
const image = this.inspectProxyImage(buffer, imageURL);
applyAttachHeaders(resp, { buffer, contentType: image.mimeType });
const contentType = resp.getHeader('Content-Type') as string | undefined;
if (contentType?.startsWith('image/')) {
return resp.status(200).send(buffer);
@@ -147,7 +150,11 @@ export class WorkerController {
response = await safeFetch(
targetURL.toString(),
{ method: 'GET', headers: cloneHeader(req.headers) },
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
{
timeoutMs: FETCH_TIMEOUT_MS,
maxRedirects: MAX_REDIRECTS,
maxBytes: IMAGE_PROXY_MAX_BYTES,
}
);
} catch (error) {
if (error instanceof SsrfBlockedError) {
@@ -155,7 +162,6 @@ export class WorkerController {
this.logger.warn('Blocked image proxy target', {
url: imageURL,
reason,
context: (error as any).context,
});
throw new BadRequest(toBadRequestReason(reason ?? 'invalid_url'));
}
@@ -175,23 +181,8 @@ export class WorkerController {
throw new BadRequest('Failed to fetch image');
}
if (response.ok) {
let buffer: Buffer;
try {
buffer = await readResponseBufferWithLimit(
response,
IMAGE_PROXY_MAX_BYTES
);
} catch (error) {
if (error instanceof ResponseTooLargeError) {
this.logger.warn('Image proxy response too large', {
url: imageURL,
limitBytes: error.data?.limitBytes,
receivedBytes: error.data?.receivedBytes,
});
throw new BadRequest('Response too large');
}
throw error;
}
const buffer = Buffer.from(await response.arrayBuffer());
const image = this.inspectProxyImage(buffer, imageURL);
await this.cache.set(cachedUrl, buffer.toString('base64'), {
ttl: CACHE_TTL,
});
@@ -204,7 +195,7 @@ export class WorkerController {
if (contentDisposition) {
resp.setHeader('Content-Disposition', contentDisposition);
}
applyAttachHeaders(resp, { buffer });
applyAttachHeaders(resp, { buffer, contentType: image.mimeType });
const contentType = resp.getHeader('Content-Type') as string | undefined;
if (contentType?.startsWith('image/')) {
return resp.status(200).send(buffer);
@@ -227,6 +218,18 @@ export class WorkerController {
}
}
private inspectProxyImage(buffer: Buffer, url: string) {
try {
return inspectImageForProxy(buffer);
} catch (error) {
this.logger.warn('Image proxy rejected invalid image', {
url,
error,
});
throw new BadRequest('Invalid image');
}
}
@Options('/link-preview')
linkPreviewOption(
@Req() request: ExpressRequest,
@@ -291,7 +294,11 @@ export class WorkerController {
const response = await safeFetch(
targetURL.toString(),
{ method, headers: cloneHeader(request.headers) },
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
{
timeoutMs: FETCH_TIMEOUT_MS,
maxRedirects: MAX_REDIRECTS,
maxBytes: LINK_PREVIEW_MAX_BYTES,
}
);
this.logger.debug('Fetched URL', {
origin,
@@ -318,10 +325,7 @@ export class WorkerController {
};
if (response.body) {
const body = await readResponseBufferWithLimit(
response,
LINK_PREVIEW_MAX_BYTES
);
const body = Buffer.from(await response.arrayBuffer());
const limitedResponse = new Response(body, response);
const resp = await decodeWithCharset(limitedResponse, res);
@@ -402,7 +406,11 @@ export class WorkerController {
const faviconResponse = await safeFetch(
faviconUrl.toString(),
{ method: 'HEAD' },
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
{
timeoutMs: FETCH_TIMEOUT_MS,
maxRedirects: MAX_REDIRECTS,
maxBytes: 0,
}
);
if (faviconResponse.ok) {
appendUrl(faviconUrl.toString(), res.favicons);
@@ -433,7 +441,6 @@ export class WorkerController {
origin,
url: requestBody?.url,
reason,
context: (error as any).context,
});
throw new BadRequest(toBadRequestReason(reason ?? 'invalid_url'));
}