feat(server): native safe fetch (#14931)

This commit is contained in:
DarkSky
2026-05-09 02:40:25 +08:00
committed by GitHub
parent 32a94d68dc
commit bcbde16c04
30 changed files with 1370 additions and 526 deletions
+4 -4
View File
@@ -135,17 +135,17 @@
},
"throttlers.default": {
"type": "object",
"description": "The config for the default throttler.\n@default {\"ttl\":60,\"limit\":120}",
"description": "The config for the default throttler.\n@default {\"ttl\":60000,\"limit\":120}",
"default": {
"ttl": 60,
"ttl": 60000,
"limit": 120
}
},
"throttlers.strict": {
"type": "object",
"description": "The config for the strict throttler.\n@default {\"ttl\":60,\"limit\":20}",
"description": "The config for the strict throttler.\n@default {\"ttl\":60000,\"limit\":20}",
"default": {
"ttl": 60,
"ttl": 60000,
"limit": 20
}
}
Generated
+78 -2
View File
@@ -209,12 +209,14 @@ dependencies = [
"napi-derive",
"rand 0.9.4",
"rayon",
"reqwest",
"schemars",
"serde",
"serde_json",
"sha3",
"tiktoken-rs",
"tokio",
"url",
"v_htmlescape",
"y-octo",
]
@@ -3719,6 +3721,12 @@ dependencies = [
"hashbrown 0.16.1",
]
[[package]]
name = "lru-slab"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "mac"
version = "0.1.1"
@@ -4978,6 +4986,62 @@ dependencies = [
"serde",
]
[[package]]
name = "quinn"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
dependencies = [
"bytes",
"cfg_aliases",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash 2.1.1",
"rustls",
"socket2",
"thiserror 2.0.18",
"tokio",
"tracing",
"web-time",
]
[[package]]
name = "quinn-proto"
version = "0.11.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
dependencies = [
"aws-lc-rs",
"bytes",
"getrandom 0.3.4",
"lru-slab",
"rand 0.9.4",
"ring",
"rustc-hash 2.1.1",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 2.0.18",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2",
"tracing",
"windows-sys 0.59.0",
]
[[package]]
name = "quote"
version = "1.0.45"
@@ -5232,9 +5296,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
name = "reqwest"
version = "0.13.2"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801"
checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
dependencies = [
"base64",
"bytes",
@@ -5252,6 +5316,7 @@ dependencies = [
"log",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls",
"rustls-pki-types",
"rustls-platform-verifier",
@@ -5459,6 +5524,7 @@ version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
dependencies = [
"web-time",
"zeroize",
]
@@ -8130,6 +8196,16 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-root-certs"
version = "1.0.7"
@@ -0,0 +1,92 @@
import { describe, expect, test } from 'vitest';
import { bilibiliConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/bilibili.js';
import { excalidrawConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/excalidraw.js';
import { genericConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/generic.js';
import { googleDocsConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/google-docs.js';
import { googleDriveConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/google-drive.js';
import { miroConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/miro.js';
import { spotifyConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/spotify.js';
describe('embed iframe provider config', () => {
test('validates final iframe URLs from oEmbed providers', () => {
expect(
spotifyConfig.validateIframeUrl?.(
'https://open.spotify.com/embed/track/0TK2YIli7K1leLovkQiNik'
)
).toBe(true);
expect(
spotifyConfig.validateIframeUrl?.(
'https://example.com/embed/track/0TK2YIli7K1leLovkQiNik'
)
).toBe(false);
});
test('validates provider-specific iframe URL shapes', () => {
expect(
googleDriveConfig.validateIframeUrl?.(
'https://drive.google.com/file/d/file-id/preview?usp=embed_googleplus'
)
).toBe(true);
expect(
googleDriveConfig.validateIframeUrl?.(
'https://drive.google.com/drive/folders/folder-id?usp=sharing'
)
).toBe(false);
expect(
bilibiliConfig.validateIframeUrl?.(
'https://player.bilibili.com/player.html?bvid=BV1xx411c7mD&autoplay=0'
)
).toBe(true);
expect(
bilibiliConfig.match(
'https://player.bilibili.com/player.html?aid=123&autoplay=0'
)
).toBe(true);
expect(
bilibiliConfig.buildOEmbedUrl(
'https://player.bilibili.com/video/BV1xx411c7mD'
)
).toBe(
'https://player.bilibili.com/player.html?bvid=BV1xx411c7mD&autoplay=0'
);
expect(
bilibiliConfig.validateIframeUrl?.(
'https://www.bilibili.com/video/BV1xx411c7mD'
)
).toBe(false);
expect(
googleDocsConfig.validateIframeUrl?.(
'https://docs.google.com/document/d/doc-id/edit?usp=sharing'
)
).toBe(true);
expect(
miroConfig.validateIframeUrl?.(
'https://miro.com/app/live-embed/board-id/'
)
).toBe(true);
expect(
excalidrawConfig.validateIframeUrl?.('https://excalidraw.com/#room-id')
).toBe(true);
});
test('generic iframe validation excludes affine and non-https URLs', () => {
expect(genericConfig.validateIframeUrl?.('https://example.com/embed')).toBe(
true
);
expect(genericConfig.validateIframeUrl?.('http://example.com/embed')).toBe(
false
);
expect(
genericConfig.validateIframeUrl?.('https://app.affine.pro/embed')
).toBe(false);
expect(genericConfig.validateIframeUrl?.('https://127.0.0.1/embed')).toBe(
false
);
expect(genericConfig.validateIframeUrl?.('https://localhost/embed')).toBe(
false
);
});
});
@@ -35,7 +35,7 @@ const extractBvid = (url: string) => {
const buildBiliPlayerEmbedUrl = (url: string) => {
// If the user pasted the embed URL directly, keep it
if (validateEmbedIframeUrl(url, biliPlayerValidationOptions)) {
if (isValidBiliPlayerUrl(url)) {
return url;
}
const avid = extractAvid(url);
@@ -57,13 +57,31 @@ const buildBiliPlayerEmbedUrl = (url: string) => {
return undefined;
};
const bilibiliConfig = {
function isValidBiliPlayerUrl(url: string) {
try {
if (!validateEmbedIframeUrl(url, biliPlayerValidationOptions)) {
return false;
}
const parsedUrl = new URL(url);
return (
parsedUrl.pathname === '/player.html' &&
(!!parsedUrl.searchParams.get('aid') ||
!!parsedUrl.searchParams.get('bvid'))
);
} catch {
return false;
}
}
export const bilibiliConfig = {
name: 'bilibili',
match: (url: string) =>
validateEmbedIframeUrl(url, bilibiliValidationOptions) &&
(!!extractAvid(url) || !!extractBvid(url)),
isValidBiliPlayerUrl(url) ||
(validateEmbedIframeUrl(url, bilibiliValidationOptions) &&
(!!extractAvid(url) || !!extractBvid(url))),
buildOEmbedUrl: buildBiliPlayerEmbedUrl,
useOEmbedUrlDirectly: true,
validateIframeUrl: (iframeUrl: string) => isValidBiliPlayerUrl(iframeUrl),
options: {
widthInSurface: BILIBILI_DEFAULT_WIDTH_IN_SURFACE,
heightInSurface: BILIBILI_DEFAULT_HEIGHT_IN_SURFACE,
@@ -15,7 +15,7 @@ const excalidrawUrlValidationOptions: EmbedIframeUrlValidationOptions = {
hostnames: ['excalidraw.com'],
};
const excalidrawConfig = {
export const excalidrawConfig = {
name: 'excalidraw',
match: (url: string) =>
validateEmbedIframeUrl(url, excalidrawUrlValidationOptions),
@@ -27,6 +27,8 @@ const excalidrawConfig = {
return url;
},
useOEmbedUrlDirectly: true,
validateIframeUrl: (iframeUrl: string) =>
validateEmbedIframeUrl(iframeUrl, excalidrawUrlValidationOptions),
options: {
widthInSurface: EXCALIDRAW_DEFAULT_WIDTH_IN_SURFACE,
heightInSurface: EXCALIDRAW_DEFAULT_HEIGHT_IN_SURFACE,
@@ -1,5 +1,10 @@
import { EmbedIframeConfigExtension } from '@blocksuite/affine-shared/services';
import {
type EmbedIframeUrlValidationOptions,
validateEmbedIframeUrl,
} from '../../utils';
const GENERIC_DEFAULT_WIDTH_IN_SURFACE = 800;
const GENERIC_DEFAULT_HEIGHT_IN_SURFACE = 600;
const GENERIC_DEFAULT_WIDTH_PERCENT = 100;
@@ -17,6 +22,11 @@ const AFFINE_DOMAINS = [
'apple.getaffineapp.com', // Cloud domain for Apple app
];
const genericUrlValidationOptions: EmbedIframeUrlValidationOptions = {
protocols: ['https:'],
hostnames: [],
};
/**
* Validates if a URL is suitable for generic iframe embedding
* Allows HTTPS URLs but excludes AFFiNE domains
@@ -27,8 +37,12 @@ function isValidGenericEmbedUrl(url: string): boolean {
try {
const parsedUrl = new URL(url);
// Only allow HTTPS for security
if (parsedUrl.protocol !== 'https:') {
if (
!validateEmbedIframeUrl(url, {
...genericUrlValidationOptions,
hostnames: [parsedUrl.hostname],
})
) {
return false;
}
@@ -49,7 +63,7 @@ function isValidGenericEmbedUrl(url: string): boolean {
}
}
const genericConfig = {
export const genericConfig = {
name: 'generic',
match: (url: string) => isValidGenericEmbedUrl(url),
buildOEmbedUrl: (url: string) => {
@@ -59,6 +73,7 @@ const genericConfig = {
return url;
},
useOEmbedUrlDirectly: true,
validateIframeUrl: (iframeUrl: string) => isValidGenericEmbedUrl(iframeUrl),
options: {
widthInSurface: GENERIC_DEFAULT_WIDTH_IN_SURFACE,
heightInSurface: GENERIC_DEFAULT_HEIGHT_IN_SURFACE,
@@ -57,7 +57,7 @@ function isValidGoogleDocsUrl(url: string, strictMode = true): boolean {
}
}
const googleDocsConfig = {
export const googleDocsConfig = {
name: 'google-docs',
match: (url: string) => isValidGoogleDocsUrl(url),
buildOEmbedUrl: (url: string) => {
@@ -67,6 +67,7 @@ const googleDocsConfig = {
return url;
},
useOEmbedUrlDirectly: true,
validateIframeUrl: (iframeUrl: string) => isValidGoogleDocsUrl(iframeUrl),
options: {
widthInSurface: GOOGLE_DOCS_DEFAULT_WIDTH_IN_SURFACE,
heightInSurface: GOOGLE_DOCS_DEFAULT_HEIGHT_IN_SURFACE,
@@ -113,6 +113,29 @@ function isValidGoogleDriveUrl(url: string, strictMode = true): boolean {
}
}
function isValidGoogleDriveIframeUrl(url: string): boolean {
try {
if (!validateEmbedIframeUrl(url, googleDriveUrlValidationOptions)) {
return false;
}
const parsedUrl = new URL(url);
const pathSegments = parsedUrl.pathname.split('/').filter(Boolean);
if (isValidGoogleDriveFileUrl(parsedUrl)) {
return pathSegments[3] === 'preview';
}
return (
parsedUrl.pathname === '/embeddedfolderview' &&
!!parsedUrl.searchParams.get('id')
);
} catch (e) {
console.warn('Invalid Google Drive iframe URL:', e);
return false;
}
}
/**
* Build embed URL for Google Drive files
* @param fileId File ID
@@ -171,7 +194,7 @@ function buildGoogleDriveEmbedUrl(url: string): string | undefined {
}
}
const googleDriveConfig = {
export const googleDriveConfig = {
name: 'google-drive',
match: (url: string) => isValidGoogleDriveUrl(url),
buildOEmbedUrl: (url: string) => {
@@ -183,6 +206,8 @@ const googleDriveConfig = {
return buildGoogleDriveEmbedUrl(url);
},
useOEmbedUrlDirectly: true,
validateIframeUrl: (iframeUrl: string) =>
isValidGoogleDriveIframeUrl(iframeUrl),
options: {
widthInSurface: GOOGLE_DRIVE_DEFAULT_WIDTH_IN_SURFACE,
heightInSurface: GOOGLE_DRIVE_DEFAULT_HEIGHT_IN_SURFACE,
@@ -18,7 +18,7 @@ const miroUrlValidationOptions: EmbedIframeUrlValidationOptions = {
hostnames: ['miro.com'],
};
const miroConfig = {
export const miroConfig = {
name: 'miro',
match: (url: string) => validateEmbedIframeUrl(url, miroUrlValidationOptions),
buildOEmbedUrl: (url: string) => {
@@ -31,6 +31,12 @@ const miroConfig = {
return oEmbedUrl;
},
useOEmbedUrlDirectly: false,
validateIframeUrl: (iframeUrl: string) => {
if (!validateEmbedIframeUrl(iframeUrl, miroUrlValidationOptions)) {
return false;
}
return new URL(iframeUrl).pathname.startsWith('/app/live-embed/');
},
options: {
widthInSurface: MIRO_DEFAULT_WIDTH_IN_SURFACE,
heightInSurface: MIRO_DEFAULT_HEIGHT_IN_SURFACE,
@@ -18,7 +18,12 @@ const spotifyUrlValidationOptions: EmbedIframeUrlValidationOptions = {
hostnames: ['open.spotify.com', 'spotify.link'],
};
const spotifyConfig = {
const spotifyIframeUrlValidationOptions: EmbedIframeUrlValidationOptions = {
protocols: ['https:'],
hostnames: ['open.spotify.com'],
};
export const spotifyConfig = {
name: 'spotify',
match: (url: string) =>
validateEmbedIframeUrl(url, spotifyUrlValidationOptions),
@@ -32,6 +37,13 @@ const spotifyConfig = {
return oEmbedUrl;
},
useOEmbedUrlDirectly: false,
validateIframeUrl: (iframeUrl: string) => {
if (!validateEmbedIframeUrl(iframeUrl, spotifyIframeUrlValidationOptions)) {
return false;
}
const parsedUrl = new URL(iframeUrl);
return parsedUrl.pathname.split('/').find(Boolean) === 'embed';
},
options: {
widthInSurface: SPOTIFY_DEFAULT_WIDTH_IN_SURFACE,
heightInSurface: SPOTIFY_DEFAULT_HEIGHT_IN_SURFACE,
@@ -141,7 +141,7 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
});
return;
}
window.open(link, '_blank');
window.open(link, '_blank', 'noopener,noreferrer');
};
refreshData = async () => {
@@ -183,6 +183,12 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
// update model
const iframeUrl = this._getIframeUrl(embedData) ?? currentIframeUrl;
if (!this._validateIframeUrl(url, iframeUrl)) {
throw new BlockSuiteError(
ErrorCode.ValueNotExists,
'Invalid embed iframe url'
);
}
this.store.updateBlock(this.model, {
iframeUrl,
title: embedData?.title || previewData?.title,
@@ -291,6 +297,19 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
}
};
private readonly _validateIframeUrl = (url: string, iframeUrl?: string) => {
if (!iframeUrl) {
return false;
}
const config = this.embedIframeService?.getConfig(url);
if (!config) {
return false;
}
return config.validateIframeUrl
? config.validateIframeUrl(iframeUrl, url)
: config.match(iframeUrl);
};
private readonly _handleDoubleClick = () => {
this.open();
};
@@ -329,6 +348,16 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
private readonly _renderIframe = () => {
const { iframeUrl } = this.model.props;
if (!iframeUrl || !this._isIframeUrlAllowed(iframeUrl)) {
return html`<embed-iframe-error-card
.error=${new Error('Invalid iframe URL')}
.model=${this.model}
.onRetry=${this._handleRetry}
.std=${this.std}
.inSurface=${this.inSurface}
.options=${this._statusCardOptions}
></embed-iframe-error-card>`;
}
const {
widthPercent,
heightInNote,
@@ -368,6 +397,10 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
: nothing}`;
};
private readonly _isIframeUrlAllowed = (iframeUrl: string) => {
return this._validateIframeUrl(this.model.props.url, iframeUrl);
};
private readonly _getSourceHost = () => {
const url = this.model.props.url ?? this.model.props.iframeUrl;
if (!url) return null;
@@ -437,7 +470,12 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
} else {
// update iframe options, to ensure the iframe is rendered with the correct options
this._updateIframeOptions(this.model.props.url);
this.status$.value = 'success';
this.status$.value = this._validateIframeUrl(
this.model.props.url,
this.model.props.iframeUrl
)
? 'success'
: 'error';
}
// refresh data when original url changes
@@ -9,6 +9,25 @@ export interface EmbedIframeUrlValidationOptions {
hostnames: string[]; // Allowed hostnames, e.g. ['docs.google.com']
}
function isLocalOrIpHostname(hostname: string): boolean {
const lower = hostname.toLowerCase();
if (
lower === 'localhost' ||
lower.endsWith('.localhost') ||
lower === '0.0.0.0' ||
lower === '::' ||
lower === '::1'
) {
return true;
}
if (/^\d{1,3}(\.\d{1,3}){3}$/.test(lower)) {
return true;
}
return lower.startsWith('[') && lower.endsWith(']');
}
/**
* Validate the url is allowed to embed in the iframe
* @param url URL to validate
@@ -23,6 +42,15 @@ export function validateEmbedIframeUrl(
const parsedUrl = new URL(url);
const { protocols, hostnames } = options;
if (
parsedUrl.username ||
parsedUrl.password ||
parsedUrl.port ||
isLocalOrIpHostname(parsedUrl.hostname)
) {
return false;
}
return (
protocols.includes(parsedUrl.protocol) &&
hostnames.includes(parsedUrl.hostname)
@@ -66,6 +66,10 @@ export type EmbedIframeConfig = {
* The function to build the oEmbed URL for fetching embed data
*/
buildOEmbedUrl: (url: string) => string | undefined;
/**
* Validate the final iframe src before rendering.
*/
validateIframeUrl?: (iframeUrl: string, originalUrl?: string) => boolean;
/**
* Use oEmbed URL directly as iframe src without fetching oEmbed data
*/
+5
View File
@@ -31,11 +31,16 @@ mp4parse = { workspace = true }
napi = { workspace = true, features = ["async", "serde-json"] }
napi-derive = { workspace = true }
rand = { workspace = true }
reqwest = { version = "0.13.3", default-features = false, features = [
"blocking",
"rustls",
] }
schemars = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sha3 = { workspace = true }
tiktoken-rs = { workspace = true }
url = { workspace = true }
v_htmlescape = { workspace = true }
y-octo = { workspace = true, features = ["large_refs"] }
+67
View File
@@ -66,6 +66,12 @@ export const AFFINE_PRO_LICENSE_AES_KEY: string | undefined | null
export const AFFINE_PRO_PUBLIC_KEY: string | undefined | null
export declare function assertSafeUrl(request: AssertSafeUrlRequest): void
export interface AssertSafeUrlRequest {
url: string
}
export declare function buildPublicRootDoc(rootDocBin: Buffer, docMetas: Array<PublicDocMetaInput>): Buffer
export interface BuiltInPromptRenderContract {
@@ -163,12 +169,30 @@ export interface Chunk {
*/
export declare function createDocWithMarkdown(title: string, markdown: string, docId: string): Buffer
export declare function fetchRemoteAttachment(request: RemoteAttachmentFetchRequest): Promise<RemoteAttachmentFetchResponse>
export declare function fromModelName(modelName: string): Tokenizer | null
export declare function getMime(input: Uint8Array): string
export declare function htmlSanitize(input: string): string
export interface ImageInspection {
mimeType: string
width: number
height: number
}
export interface ImageInspectionOptions {
maxWidth?: number
maxHeight?: number
maxPixels?: number
}
export declare function inferRemoteMimeType(request: RemoteMimeTypeRequest): Promise<string>
export declare function inspectImageForProxy(input: Buffer, options?: ImageInspectionOptions | undefined | null): ImageInspection
export declare function llmBuildCanonicalRequest(request: CanonicalChatRequestContract): LlmRequestContract
export declare function llmBuildCanonicalStructuredRequest(request: CanonicalStructuredRequestContract): LlmStructuredRequestContract
@@ -572,6 +596,28 @@ export interface PublicDocMetaInput {
export declare function readAllDocIdsFromRootDoc(docBin: Buffer, includeTrash?: boolean | undefined | null): Array<string>
export interface RemoteAttachmentFetchRequest {
url: string
timeoutMs?: number
maxBytes: number
allowPrivateTargetOrigin?: boolean
expectedContentTypePrefix?: string
maxImageWidth?: number
maxImageHeight?: number
maxImagePixels?: number
}
export interface RemoteAttachmentFetchResponse {
finalUrl: string
mimeType: string
body: Buffer
}
export interface RemoteMimeTypeRequest {
url: string
timeoutMs?: number
}
export interface RequestedModelMatchRequest {
providerIds: Array<string>
optionalModels: Array<string>
@@ -591,6 +637,27 @@ export interface RerankCandidate {
export declare function runNativeActionRecipePreparedStream(input: ActionRuntimeInput, callback: ((err: Error | null, arg: string) => void)): LlmStreamHandle
export declare function safeFetch(request: SafeFetchRequest): Promise<SafeFetchResponse>
export type SafeFetchMethod = 'get'|
'head';
export interface SafeFetchRequest {
url: string
method?: SafeFetchMethod
headers?: Record<string, string>
timeoutMs?: number
maxRedirects?: number
maxBytes?: number
}
export interface SafeFetchResponse {
status: number
finalUrl: string
headers: Record<string, string>
body: Buffer
}
export interface ToolContract {
name: string
description?: string
+1
View File
@@ -9,6 +9,7 @@ pub mod hashcash;
pub mod html_sanitize;
pub mod image;
pub mod llm;
pub mod safe_fetch;
pub mod tiktoken;
use affine_common::napi_utils::map_napi_err;
+668
View File
@@ -0,0 +1,668 @@
use std::{
collections::HashMap,
io::{Cursor, Read},
net::{IpAddr, SocketAddr, ToSocketAddrs},
time::Duration,
};
use ::image::{ImageFormat, ImageReader};
use anyhow::{Context, Result as AnyResult, bail};
use napi::{
Env, Error, Result, Status, Task,
bindgen_prelude::{AsyncTask, Buffer},
};
use napi_derive::napi;
use reqwest::{
Method,
blocking::{Client, Response},
header::{HeaderMap, HeaderName, HeaderValue, LOCATION},
};
use url::Url;
const DEFAULT_TIMEOUT_MS: u32 = 10_000;
const DEFAULT_MAX_REDIRECTS: u32 = 3;
const DEFAULT_MAX_BYTES: u32 = 10 * 1024 * 1024;
const MAX_IMAGE_DIMENSION: u32 = 16_384;
const MAX_IMAGE_PIXELS: u64 = 40_000_000;
#[napi(string_enum = "snake_case")]
#[derive(Clone, Copy, Debug)]
pub enum SafeFetchMethod {
Get,
Head,
}
#[napi(object)]
#[derive(Clone, Debug)]
pub struct SafeFetchRequest {
pub url: String,
pub method: Option<SafeFetchMethod>,
pub headers: Option<HashMap<String, String>>,
pub timeout_ms: Option<u32>,
pub max_redirects: Option<u32>,
pub max_bytes: Option<u32>,
}
#[napi(object)]
#[derive(Clone, Debug)]
pub struct AssertSafeUrlRequest {
pub url: String,
}
#[napi(object)]
pub struct SafeFetchResponse {
pub status: u16,
pub final_url: String,
pub headers: HashMap<String, String>,
pub body: Buffer,
}
#[napi(object)]
#[derive(Clone, Debug)]
pub struct ImageInspectionOptions {
pub max_width: Option<u32>,
pub max_height: Option<u32>,
pub max_pixels: Option<u32>,
}
#[napi(object)]
pub struct ImageInspection {
pub mime_type: String,
pub width: u32,
pub height: u32,
}
#[napi(object)]
#[derive(Clone, Debug)]
pub struct RemoteAttachmentFetchRequest {
pub url: String,
pub timeout_ms: Option<u32>,
pub max_bytes: u32,
pub allow_private_target_origin: Option<bool>,
pub expected_content_type_prefix: Option<String>,
pub max_image_width: Option<u32>,
pub max_image_height: Option<u32>,
pub max_image_pixels: Option<u32>,
}
#[napi(object)]
pub struct RemoteAttachmentFetchResponse {
pub final_url: String,
pub mime_type: String,
pub body: Buffer,
}
#[napi(object)]
#[derive(Clone, Debug)]
pub struct RemoteMimeTypeRequest {
pub url: String,
pub timeout_ms: Option<u32>,
}
pub struct AsyncSafeFetchTask {
request: SafeFetchRequest,
}
pub struct AsyncRemoteAttachmentFetchTask {
request: RemoteAttachmentFetchRequest,
}
pub struct AsyncRemoteMimeTypeTask {
request: RemoteMimeTypeRequest,
}
pub struct SafeFetchOutput {
status: u16,
final_url: String,
headers: HashMap<String, String>,
body: Vec<u8>,
}
struct SafeFetchParams {
url: String,
method: Option<SafeFetchMethod>,
headers: Option<HashMap<String, String>>,
timeout_ms: Option<u32>,
max_redirects: Option<u32>,
max_bytes: Option<u32>,
allow_private_origins: Option<Vec<String>>,
}
pub struct RemoteAttachmentFetchOutput {
final_url: String,
mime_type: String,
body: Vec<u8>,
}
#[napi]
impl Task for AsyncSafeFetchTask {
type Output = SafeFetchOutput;
type JsValue = SafeFetchResponse;
fn compute(&mut self) -> Result<Self::Output> {
safe_fetch_inner(&self.request).map_err(|error| Error::new(Status::InvalidArg, error.to_string()))
}
fn resolve(&mut self, _: Env, output: Self::Output) -> Result<Self::JsValue> {
Ok(SafeFetchResponse {
status: output.status,
final_url: output.final_url,
headers: output.headers,
body: output.body.into(),
})
}
}
#[napi]
pub fn safe_fetch(request: SafeFetchRequest) -> AsyncTask<AsyncSafeFetchTask> {
AsyncTask::new(AsyncSafeFetchTask { request })
}
#[napi]
pub fn assert_safe_url(request: AssertSafeUrlRequest) -> Result<()> {
assert_safe_url_inner(&request).map_err(|error| Error::new(Status::InvalidArg, error.to_string()))
}
#[napi]
pub fn inspect_image_for_proxy(input: Buffer, options: Option<ImageInspectionOptions>) -> Result<ImageInspection> {
inspect_image_for_proxy_inner(
&input,
options.unwrap_or(ImageInspectionOptions {
max_width: None,
max_height: None,
max_pixels: None,
}),
)
.map_err(|error| Error::new(Status::InvalidArg, error.to_string()))
}
#[napi]
impl Task for AsyncRemoteAttachmentFetchTask {
type Output = RemoteAttachmentFetchOutput;
type JsValue = RemoteAttachmentFetchResponse;
fn compute(&mut self) -> Result<Self::Output> {
fetch_remote_attachment_inner(&self.request).map_err(|error| Error::new(Status::InvalidArg, error.to_string()))
}
fn resolve(&mut self, _: Env, output: Self::Output) -> Result<Self::JsValue> {
Ok(RemoteAttachmentFetchResponse {
final_url: output.final_url,
mime_type: output.mime_type,
body: output.body.into(),
})
}
}
#[napi]
pub fn fetch_remote_attachment(request: RemoteAttachmentFetchRequest) -> AsyncTask<AsyncRemoteAttachmentFetchTask> {
AsyncTask::new(AsyncRemoteAttachmentFetchTask { request })
}
#[napi]
impl Task for AsyncRemoteMimeTypeTask {
type Output = String;
type JsValue = String;
fn compute(&mut self) -> Result<Self::Output> {
Ok(infer_remote_mime_type_inner(&self.request))
}
fn resolve(&mut self, _: Env, output: Self::Output) -> Result<Self::JsValue> {
Ok(output)
}
}
#[napi]
pub fn infer_remote_mime_type(request: RemoteMimeTypeRequest) -> AsyncTask<AsyncRemoteMimeTypeTask> {
AsyncTask::new(AsyncRemoteMimeTypeTask { request })
}
fn safe_fetch_inner(request: &SafeFetchRequest) -> AnyResult<SafeFetchOutput> {
safe_fetch_params_inner(&SafeFetchParams {
url: request.url.clone(),
method: request.method,
headers: request.headers.clone(),
timeout_ms: request.timeout_ms,
max_redirects: request.max_redirects,
max_bytes: request.max_bytes,
allow_private_origins: None,
})
}
fn safe_fetch_params_inner(request: &SafeFetchParams) -> AnyResult<SafeFetchOutput> {
let timeout = Duration::from_millis(u64::from(request.timeout_ms.unwrap_or(DEFAULT_TIMEOUT_MS)));
let max_redirects = request.max_redirects.unwrap_or(DEFAULT_MAX_REDIRECTS);
let max_bytes = usize::try_from(request.max_bytes.unwrap_or(DEFAULT_MAX_BYTES)).context("invalid maxBytes")?;
let method = request.method.unwrap_or(SafeFetchMethod::Get);
let mut current = parse_safe_url(&request.url)?;
let headers = build_headers(request.headers.as_ref())?;
for redirect_count in 0..=max_redirects {
let addrs = resolve_safe_socket_addrs(&current, request.allow_private_origins.as_deref())?;
let client = build_pinned_client(&current, &addrs, timeout)?;
let response = send_request(&client, method, current.clone(), headers.clone())?;
if response.status().is_redirection() {
if redirect_count >= max_redirects {
bail!("too_many_redirects");
}
let Some(location) = response.headers().get(LOCATION) else {
return response_to_output(response, current, max_bytes, method);
};
let location = location.to_str().context("invalid redirect location")?;
current = parse_safe_url(current.join(location).context("invalid redirect location")?.as_str())?;
continue;
}
return response_to_output(response, current, max_bytes, method);
}
bail!("too_many_redirects")
}
fn fetch_remote_attachment_inner(request: &RemoteAttachmentFetchRequest) -> AnyResult<RemoteAttachmentFetchOutput> {
let allow_private_origins =
private_target_origin_allowlist(&request.url, request.allow_private_target_origin.unwrap_or(false))?;
let response = safe_fetch_params_inner(&SafeFetchParams {
url: request.url.clone(),
method: Some(SafeFetchMethod::Get),
headers: None,
timeout_ms: request.timeout_ms,
max_redirects: Some(DEFAULT_MAX_REDIRECTS),
max_bytes: Some(request.max_bytes),
allow_private_origins,
})?;
if !(200..300).contains(&response.status) {
bail!("fetch_failed_status: {}", response.status);
}
let mime_type = normalize_mime_type(response.headers.get("content-type"));
if let Some(expected) = request.expected_content_type_prefix.as_deref() {
if !mime_type.starts_with(expected) {
bail!("content_type_mismatch");
}
if expected.starts_with("image/") {
inspect_image_for_proxy_inner(
&response.body,
ImageInspectionOptions {
max_width: request.max_image_width,
max_height: request.max_image_height,
max_pixels: request.max_image_pixels,
},
)?;
}
}
Ok(RemoteAttachmentFetchOutput {
final_url: response.final_url,
mime_type,
body: response.body,
})
}
fn infer_remote_mime_type_inner(request: &RemoteMimeTypeRequest) -> String {
let Ok(url) = Url::parse(&request.url) else {
return "application/octet-stream".to_string();
};
if let Some(mime_type) = infer_mime_type_from_extension(&url) {
return mime_type.to_string();
}
let Ok(response) = safe_fetch_params_inner(&SafeFetchParams {
url: request.url.clone(),
method: Some(SafeFetchMethod::Head),
headers: None,
timeout_ms: request.timeout_ms,
max_redirects: Some(DEFAULT_MAX_REDIRECTS),
max_bytes: Some(0),
allow_private_origins: None,
}) else {
return "application/octet-stream".to_string();
};
normalize_mime_type(response.headers.get("content-type"))
}
fn private_target_origin_allowlist(raw_url: &str, allow_private_target_origin: bool) -> AnyResult<Option<Vec<String>>> {
if !allow_private_target_origin {
return Ok(None);
}
Ok(Some(vec![parse_safe_url(raw_url)?.origin().ascii_serialization()]))
}
fn normalize_mime_type(value: Option<&String>) -> String {
value
.and_then(|value| value.split(';').next())
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("application/octet-stream")
.to_string()
}
fn infer_mime_type_from_extension(url: &Url) -> Option<&'static str> {
let extension = url.path_segments()?.next_back()?.rsplit_once('.')?.1;
match extension.to_ascii_lowercase().as_str() {
"pdf" => Some("application/pdf"),
"mp3" => Some("audio/mpeg"),
"opus" => Some("audio/opus"),
"ogg" => Some("audio/ogg"),
"aac" => Some("audio/aac"),
"m4a" => Some("audio/aac"),
"flac" => Some("audio/flac"),
"ogv" => Some("video/ogg"),
"wav" => Some("audio/wav"),
"png" => Some("image/png"),
"jpeg" | "jpg" => Some("image/jpeg"),
"webp" => Some("image/webp"),
"txt" | "md" => Some("text/plain"),
"mov" => Some("video/mov"),
"mpeg" => Some("video/mpeg"),
"mp4" => Some("video/mp4"),
"avi" => Some("video/avi"),
"wmv" => Some("video/wmv"),
"flv" => Some("video/flv"),
_ => None,
}
}
fn assert_safe_url_inner(request: &AssertSafeUrlRequest) -> AnyResult<()> {
let url = parse_safe_url(&request.url)?;
resolve_safe_socket_addrs(&url, None)?;
Ok(())
}
fn parse_safe_url(raw: &str) -> AnyResult<Url> {
let url = Url::parse(raw).context("invalid_url")?;
match url.scheme() {
"http" | "https" => {}
_ => bail!("disallowed_protocol"),
}
if !url.username().is_empty() || url.password().is_some() {
bail!("url_has_credentials");
}
if url.host_str().is_none() {
bail!("blocked_hostname");
}
Ok(url)
}
fn resolve_safe_socket_addrs(url: &Url, allow_private_origins: Option<&[String]>) -> AnyResult<Vec<SocketAddr>> {
let host = url.host_str().context("blocked_hostname")?;
let port = url.port_or_known_default().context("blocked_hostname")?;
let origin = url.origin().ascii_serialization();
let allow_private = allow_private_origins
.map(|origins| origins.iter().any(|allowed| allowed == &origin))
.unwrap_or(false);
let addrs: Vec<SocketAddr> = (host, port)
.to_socket_addrs()
.context("unresolvable_hostname")?
.collect();
if addrs.is_empty() {
bail!("unresolvable_hostname");
}
for addr in &addrs {
if is_blocked_ip(addr.ip()) && !allow_private {
bail!("blocked_ip");
}
}
Ok(addrs)
}
fn build_pinned_client(url: &Url, addrs: &[SocketAddr], timeout: Duration) -> AnyResult<Client> {
let host = url.host_str().context("blocked_hostname")?;
Client::builder()
.timeout(timeout)
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.resolve_to_addrs(host, addrs)
.build()
.context("failed to build http client")
}
fn build_headers(headers: Option<&HashMap<String, String>>) -> AnyResult<HeaderMap> {
let mut out = HeaderMap::new();
let Some(headers) = headers else {
return Ok(out);
};
for (name, value) in headers {
let lower = name.to_ascii_lowercase();
if !(lower.starts_with("sec-") || lower.starts_with("accept") || lower == "user-agent") {
continue;
}
out.insert(
HeaderName::from_bytes(name.as_bytes()).context("invalid header name")?,
HeaderValue::from_str(value).context("invalid header value")?,
);
}
Ok(out)
}
fn send_request(client: &Client, method: SafeFetchMethod, url: Url, headers: HeaderMap) -> AnyResult<Response> {
let method = match method {
SafeFetchMethod::Get => Method::GET,
SafeFetchMethod::Head => Method::HEAD,
};
client
.request(method, url)
.headers(headers)
.send()
.context("failed to fetch url")
}
fn response_to_output(
mut response: Response,
url: Url,
max_bytes: usize,
method: SafeFetchMethod,
) -> AnyResult<SafeFetchOutput> {
let status = response.status().as_u16();
let headers = response_headers(response.headers());
if matches!(method, SafeFetchMethod::Head) {
return Ok(SafeFetchOutput {
status,
final_url: url.to_string(),
headers,
body: Vec::new(),
});
}
let mut body = Vec::new();
if let Some(len) = response.content_length()
&& len > max_bytes as u64
{
bail!("response_too_large");
}
response
.by_ref()
.take(u64::try_from(max_bytes).unwrap_or(u64::MAX) + 1)
.read_to_end(&mut body)
.context("failed to read response")?;
if body.len() > max_bytes {
bail!("response_too_large");
}
Ok(SafeFetchOutput {
status,
final_url: url.to_string(),
headers,
body,
})
}
fn response_headers(headers: &HeaderMap) -> HashMap<String, String> {
headers
.iter()
.filter_map(|(name, value)| {
value
.to_str()
.ok()
.map(|value| (name.as_str().to_string(), value.to_string()))
})
.collect()
}
fn is_blocked_ip(ip: IpAddr) -> bool {
match ip {
IpAddr::V4(ip) => {
ip.is_private()
|| ip.is_loopback()
|| ip.is_link_local()
|| ip.is_broadcast()
|| ip.is_multicast()
|| ip.is_documentation()
|| ip.octets()[0] == 0
|| ip.octets()[0] >= 224
|| ip.octets()[0] == 100 && (64..=127).contains(&ip.octets()[1])
|| ip.octets()[0] == 169 && ip.octets()[1] == 254
|| ip.octets()[0] == 198 && (18..=19).contains(&ip.octets()[1])
|| ip.octets()[0] == 192 && ip.octets()[1] == 0 && ip.octets()[2] == 0
}
IpAddr::V6(ip) => {
if let Some(v4) = ip.to_ipv4_mapped() {
return is_blocked_ip(IpAddr::V4(v4));
}
if let Some(v4) = extract_6to4_ipv4(ip).or_else(|| extract_teredo_client_ipv4(ip)) {
return is_blocked_ip(IpAddr::V4(v4));
}
(ip.segments()[0] & 0xe000 != 0x2000)
|| ip.is_loopback()
|| ip.is_unspecified()
|| ip.is_multicast()
|| (ip.segments()[0] & 0xfe00 == 0xfc00)
|| (ip.segments()[0] & 0xffc0 == 0xfe80)
|| (ip.segments()[0] == 0x2001 && ip.segments()[1] == 0x0db8)
}
}
}
fn extract_6to4_ipv4(ip: std::net::Ipv6Addr) -> Option<std::net::Ipv4Addr> {
let segments = ip.segments();
if segments[0] != 0x2002 {
return None;
}
Some(std::net::Ipv4Addr::new(
(segments[1] >> 8) as u8,
segments[1] as u8,
(segments[2] >> 8) as u8,
segments[2] as u8,
))
}
fn extract_teredo_client_ipv4(ip: std::net::Ipv6Addr) -> Option<std::net::Ipv4Addr> {
let segments = ip.segments();
if segments[0] != 0x2001 || segments[1] != 0 {
return None;
}
Some(std::net::Ipv4Addr::new(
(!(segments[6] >> 8)) as u8,
(!segments[6]) as u8,
(!(segments[7] >> 8)) as u8,
(!segments[7]) as u8,
))
}
fn inspect_image_for_proxy_inner(input: &[u8], options: ImageInspectionOptions) -> AnyResult<ImageInspection> {
let inspection = parse_image_header(input).context("failed to decode image")?;
validate_image_dimensions(&inspection, options)?;
Ok(inspection)
}
fn validate_image_dimensions(image: &ImageInspection, options: ImageInspectionOptions) -> AnyResult<()> {
let max_width = options.max_width.unwrap_or(MAX_IMAGE_DIMENSION);
let max_height = options.max_height.unwrap_or(MAX_IMAGE_DIMENSION);
let max_pixels = u64::from(options.max_pixels.unwrap_or(MAX_IMAGE_PIXELS as u32));
if image.width == 0 || image.height == 0 {
bail!("failed to decode image");
}
if image.width > max_width || image.height > max_height {
bail!("image dimensions exceed limit");
}
if u64::from(image.width) * u64::from(image.height) > max_pixels {
bail!("image pixel count exceeds limit");
}
Ok(())
}
fn parse_image_header(input: &[u8]) -> AnyResult<ImageInspection> {
let format = ::image::guess_format(input).context("unsupported image format")?;
let mime_type = image_mime_type(format).context("unsupported image format")?;
let (width, height) = ImageReader::with_format(Cursor::new(input), format)
.into_dimensions()
.context("failed to decode image")?;
Ok(ImageInspection {
mime_type: mime_type.to_string(),
width,
height,
})
}
fn image_mime_type(format: ImageFormat) -> Option<&'static str> {
match format {
ImageFormat::Png => Some("image/png"),
ImageFormat::Jpeg => Some("image/jpeg"),
ImageFormat::Gif => Some("image/gif"),
ImageFormat::WebP => Some("image/webp"),
ImageFormat::Bmp => Some("image/bmp"),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn blocks_private_ips() {
assert!(is_blocked_ip("127.0.0.1".parse().unwrap()));
assert!(is_blocked_ip("10.0.0.1".parse().unwrap()));
assert!(is_blocked_ip("169.254.169.254".parse().unwrap()));
assert!(is_blocked_ip("::1".parse().unwrap()));
assert!(is_blocked_ip("::ffff:127.0.0.1".parse().unwrap()));
assert!(is_blocked_ip("2002:7f00:0001::1".parse().unwrap()));
assert!(is_blocked_ip("2002:c0a8:0001::1".parse().unwrap()));
assert!(is_blocked_ip(
"2001:0000:4136:e378:8000:63bf:807f:fffe".parse().unwrap()
));
assert!(!is_blocked_ip("8.8.8.8".parse().unwrap()));
assert!(!is_blocked_ip("2002:0808:0808::1".parse().unwrap()));
}
#[test]
fn inspects_png_dimensions_without_decode() {
let png = base64_simd::STANDARD
.decode_to_vec(b"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jfJ8AAAAASUVORK5CYII=")
.unwrap();
let inspected = inspect_image_for_proxy_inner(
&png,
ImageInspectionOptions {
max_width: None,
max_height: None,
max_pixels: None,
},
)
.unwrap();
assert_eq!(inspected.mime_type, "image/png");
assert_eq!(inspected.width, 1);
assert_eq!(inspected.height, 1);
}
#[test]
fn rejects_oversized_dimensions() {
let png = [
b"\x89PNG\r\n\x1a\n".as_slice(),
&[0, 0, 0, 13],
b"IHDR".as_slice(),
&100_000u32.to_be_bytes(),
&100_000u32.to_be_bytes(),
&[8, 6, 0, 0, 0],
]
.concat();
assert!(
inspect_image_for_proxy_inner(
&png,
ImageInspectionOptions {
max_width: None,
max_height: None,
max_pixels: None,
}
)
.is_err()
);
}
}
@@ -1,27 +0,0 @@
import test from 'ava';
import { readResponseBufferWithLimit } from '../../base';
test('readResponseBufferWithLimit rejects timed out web streams without crashing', async t => {
const response = new Response(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array([1, 2, 3]));
queueMicrotask(() => {
controller.error(
new DOMException(
'The operation was aborted due to timeout',
'TimeoutError'
)
);
});
},
})
);
const error = await t.throwsAsync(
readResponseBufferWithLimit(response, 1024)
);
t.is(error?.name, 'TimeoutError');
});
@@ -18,6 +18,7 @@ import {
JobQueue,
OneMB,
} from '../../base';
import { ThrottlerStorage } from '../../base/throttler';
import { SocketIoAdapter } from '../../base/websocket';
import { AuthGuard, AuthService } from '../../core/auth';
import { Mailer } from '../../core/mail';
@@ -163,6 +164,10 @@ export class TestingApp extends NestApplication {
return await this.create(MockUser, overrides);
}
resetRateLimit() {
this.get(ThrottlerStorage, { strict: false }).storage.clear();
}
async signup(overrides?: Partial<MockUserInput>) {
const user = await this.create(MockUser, overrides);
await this.login(user);
@@ -170,6 +175,7 @@ export class TestingApp extends NestApplication {
}
async login(user: MockedUser) {
this.resetRateLimit();
return await this.POST('/api/auth/sign-in').send({
email: user.email,
password: user.password,
@@ -195,6 +201,7 @@ export class TestingApp extends NestApplication {
}
async logout(userId?: string) {
this.resetRateLimit();
const res = await this.POST(
'/api/auth/sign-out' + (userId ? `?user_id=${userId}` : '')
).expect(200);
@@ -92,7 +92,7 @@ test.before(async t => {
throttle: {
throttlers: {
default: {
ttl: 60,
ttl: 60_000,
limit: 120,
},
},
@@ -109,6 +109,7 @@ test.before(async t => {
test.beforeEach(async t => {
const { app } = t.context;
t.context.storage.storage.clear();
await app.initTestingDB();
});
@@ -1,49 +1,79 @@
import { LookupAddress } from 'node:dns';
import serverNativeModule from '@affine/server-native';
import type { ExecutionContext, TestFn } from 'ava';
import ava from 'ava';
import Sinon from 'sinon';
import type { Response } from 'supertest';
import {
__resetDnsLookupForTests,
__setDnsLookupForTests,
type DnsLookup,
} from '../base/utils/ssrf';
import { createTestingApp, TestingApp } from './utils';
import type { TestingApp } from './utils';
type TestContext = {
app: TestingApp;
};
const test = ava as TestFn<TestContext>;
const LookupAddressStub = (async (_hostname, options) => {
const result = [{ address: '76.76.21.21', family: 4 }] as LookupAddress[];
const isOptions = options && typeof options === 'object';
if (isOptions && 'all' in options && options.all) {
return result;
let safeFetchStub: Sinon.SinonStub | undefined;
let safeFetchHandler:
| ((request: { url: string; method?: 'get' | 'head' }) => {
status?: number;
finalUrl?: string;
headers?: Record<string, string>;
body?: Buffer | string;
})
| undefined;
const stubSafeFetch = (
handler: (request: { url: string; method?: 'get' | 'head' }) => {
status?: number;
finalUrl?: string;
headers?: Record<string, string>;
body?: Buffer | string;
}
return result[0];
}) as DnsLookup;
) => {
safeFetchHandler = handler;
return {
restore() {
safeFetchHandler = undefined;
},
};
};
test.before(async t => {
// @ts-expect-error test
env.DEPLOYMENT_TYPE = 'selfhosted';
// Avoid relying on real DNS during tests. SSRF protection uses dns.lookup().
__setDnsLookupForTests(LookupAddressStub);
safeFetchStub = Sinon.stub(serverNativeModule, 'safeFetch').callsFake(
async request => {
if (!safeFetchHandler) {
throw new Error('Unexpected safeFetch call');
}
const nativeRequest = request as {
url: string;
method?: 'get' | 'head';
};
const response = safeFetchHandler(nativeRequest);
return {
status: response.status ?? 200,
finalUrl: response.finalUrl ?? nativeRequest.url,
headers: response.headers ?? {},
body: Buffer.isBuffer(response.body)
? response.body
: Buffer.from(response.body ?? ''),
};
}
);
const { createTestingApp } = await import('./utils');
const app = await createTestingApp();
t.context.app = app;
});
test.afterEach.always(() => {
Sinon.restore();
safeFetchHandler = undefined;
});
test.after.always(async t => {
__resetDnsLookupForTests();
safeFetchStub?.restore();
await t.context.app.close();
});
@@ -128,15 +158,13 @@ test('should proxy image', async t => {
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jfJ8AAAAASUVORK5CYII=',
'base64'
);
const fakeResponse = new Response(fakeBuffer, {
status: 200,
const fetchSpy = stubSafeFetch(() => ({
body: fakeBuffer,
headers: {
'content-type': 'image/png',
'content-disposition': 'inline',
},
});
const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeResponse);
}));
try {
await assertAndSnapshot(
`/api/worker/image-proxy?url=${imageUrl}`,
@@ -146,6 +174,42 @@ test('should proxy image', async t => {
fetchSpy.restore();
}
}
{
const invalidImageUrl = `http://example.com/not-image-${Date.now()}.png`;
const invalidFetchSpy = stubSafeFetch(() => ({
body: 'not an image',
headers: { 'content-type': 'image/png' },
}));
try {
await t.context.app
.GET(`/api/worker/image-proxy?url=${invalidImageUrl}`)
.set('Origin', 'http://localhost:3010')
.send()
.expect(400);
} finally {
invalidFetchSpy.restore();
}
const validImageUrl = `http://example.com/valid-image-${Date.now()}.png`;
const fakeBuffer = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jfJ8AAAAASUVORK5CYII=',
'base64'
);
const validFetchSpy = stubSafeFetch(() => ({
body: fakeBuffer,
headers: { 'content-type': 'image/png' },
}));
try {
await t.context.app
.GET(`/api/worker/image-proxy?url=${validImageUrl}`)
.set('Origin', 'http://localhost:3010')
.send()
.expect(200);
} finally {
validFetchSpy.restore();
}
}
});
test('should preview link', async t => {
@@ -190,7 +254,8 @@ test('should preview link', async t => {
);
{
const fakeHTML = new Response(`
const pageUrl = `http://external.com/page-${Date.now()}`;
const fakeHTML = `
<html>
<head>
<meta property="og:title" content="Test Title" />
@@ -201,13 +266,18 @@ test('should preview link', async t => {
<title>Fallback Title</title>
</body>
</html>
`);
`;
Object.defineProperty(fakeHTML, 'url', {
value: 'http://example.com/page',
const fetchSpy = stubSafeFetch(request => {
if (request.url.includes('/favicon.ico')) {
return { status: 204, finalUrl: request.url };
}
return {
body: fakeHTML,
finalUrl: 'http://example.com/page',
headers: { 'content-type': 'text/html;charset=UTF-8' },
};
});
const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeHTML);
try {
await assertAndSnapshot(
'/api/worker/link-preview',
@@ -215,7 +285,7 @@ test('should preview link', async t => {
{
status: 200,
method: 'POST',
body: { url: 'http://external.com/page' },
body: { url: pageUrl },
}
);
} finally {
@@ -244,6 +314,7 @@ test('should preview link', async t => {
];
for (const { content, charset } of encoded) {
const pageUrl = `http://example.com/${charset}-${Date.now()}`;
const before = Buffer.from(`<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=${charset}" />
@@ -253,13 +324,18 @@ test('should preview link', async t => {
</head>
</html>
`);
const fakeHTML = new Response(Buffer.concat([before, encoded, after]));
const fakeHTML = Buffer.concat([before, encoded, after]);
Object.defineProperty(fakeHTML, 'url', {
value: `http://example.com/${charset}`,
const fetchSpy = stubSafeFetch(request => {
if (request.url.includes('/favicon.ico')) {
return { status: 204, finalUrl: request.url };
}
return {
body: fakeHTML,
finalUrl: `http://example.com/${charset}`,
headers: { 'content-type': `text/html;charset=${charset}` },
};
});
const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeHTML);
try {
await assertAndSnapshot(
'/api/worker/link-preview',
@@ -267,7 +343,7 @@ test('should preview link', async t => {
{
status: 200,
method: 'POST',
body: { url: `http://example.com/${charset}` },
body: { url: pageUrl },
}
);
} finally {
@@ -24,14 +24,14 @@ defineModuleConfig('throttle', {
'throttlers.default': {
desc: 'The config for the default throttler.',
default: {
ttl: 60,
ttl: 60_000,
limit: 120,
},
},
'throttlers.strict': {
desc: 'The config for the strict throttler.',
default: {
ttl: 60,
ttl: 60_000,
limit: 20,
},
},
@@ -13,7 +13,7 @@ export const THROTTLER_PROTECTED = 'affine_throttler:protected';
* it will never be rate limited.
*
* - default: 120 calls within 60 seconds
* - strict: 10 calls within 60 seconds
* - strict: 20 calls within 60 seconds
* - authenticated: no rate limit for authenticated users, apply [default] throttler for unauthenticated users
*
* @example
@@ -23,7 +23,7 @@ export const THROTTLER_PROTECTED = 'affine_throttler:protected';
*
* // the config call be override by the second parameter,
* // and the call count will be calculated separately
* \@Throttle('default', { limit: 10, ttl: 10 })
* \@Throttle('default', { limit: 10, ttl: 10_000 })
*
*/
export function Throttle(
+73 -321
View File
@@ -1,23 +1,9 @@
import * as dns from 'node:dns/promises';
import { BlockList, isIP } from 'node:net';
import {
assertSafeUrl as assertSafeUrlFromNative,
safeFetch as safeFetchFromNative,
type SafeFetchRequest,
} from '../../native';
import { ResponseTooLargeError, SsrfBlockedError } from '../error/errors.gen';
import { OneMinute } from './unit';
const DEFAULT_ALLOWED_PROTOCOLS = new Set(['http:', 'https:']);
const BLOCKED_IPS = new BlockList();
const ALLOWED_IPV6 = new BlockList();
export type DnsLookup = typeof dns.lookup;
let dnsLookup: DnsLookup = dns.lookup;
export function __setDnsLookupForTests(lookup: DnsLookup) {
dnsLookup = lookup;
}
export function __resetDnsLookupForTests() {
dnsLookup = dns.lookup;
}
export type SSRFBlockReason =
| 'invalid_url'
@@ -28,187 +14,55 @@ export type SSRFBlockReason =
| 'blocked_ip'
| 'too_many_redirects';
type SsrfErrorContext = { url?: string; hostname?: string; address?: string };
const SSRF_REASONS = new Set<string>([
'invalid_url',
'disallowed_protocol',
'url_has_credentials',
'blocked_hostname',
'unresolvable_hostname',
'blocked_ip',
'too_many_redirects',
]);
function createSsrfBlockedError(
reason: SSRFBlockReason,
context?: SsrfErrorContext
) {
const err = new SsrfBlockedError({ reason });
// For logging/debugging only (not part of UserFriendlyError JSON).
(err as any).context = context;
return err;
function createSsrfBlockedError(reason: SSRFBlockReason) {
return new SsrfBlockedError({ reason });
}
export interface SSRFProtectionOptions {
allowedProtocols?: ReadonlySet<string>;
/**
* Allow fetching private/reserved IPs when URL.origin is allowlisted.
* Defaults to an empty allowlist (i.e. private IPs are blocked).
*/
allowPrivateOrigins?: ReadonlySet<string>;
}
function stripZoneId(address: string) {
const idx = address.indexOf('%');
return idx === -1 ? address : address.slice(0, idx);
}
// IPv4: RFC1918 + loopback + link-local + CGNAT + special/reserved
for (const [network, prefix] of [
['0.0.0.0', 8],
['10.0.0.0', 8],
['127.0.0.0', 8],
['169.254.0.0', 16],
['172.16.0.0', 12],
['192.168.0.0', 16],
['100.64.0.0', 10], // CGNAT
['192.0.0.0', 24],
['192.0.2.0', 24], // TEST-NET-1
['198.51.100.0', 24], // TEST-NET-2
['203.0.113.0', 24], // TEST-NET-3
['198.18.0.0', 15], // benchmark
['192.88.99.0', 24], // 6to4 relay
['224.0.0.0', 4], // multicast
['240.0.0.0', 4], // reserved (includes broadcast)
] as const) {
BLOCKED_IPS.addSubnet(network, prefix, 'ipv4');
}
// IPv6: block loopback/unspecified/link-local/ULA/multicast/doc; allow only global unicast.
BLOCKED_IPS.addAddress('::', 'ipv6');
BLOCKED_IPS.addAddress('::1', 'ipv6');
BLOCKED_IPS.addSubnet('ff00::', 8, 'ipv6'); // multicast
BLOCKED_IPS.addSubnet('fc00::', 7, 'ipv6'); // unique local
BLOCKED_IPS.addSubnet('fe80::', 10, 'ipv6'); // link-local
BLOCKED_IPS.addSubnet('2001:db8::', 32, 'ipv6'); // documentation
ALLOWED_IPV6.addSubnet('2000::', 3, 'ipv6'); // global unicast
function extractEmbeddedIPv4FromIPv6(address: string): string | null {
if (!address.includes('.')) {
return null;
function mapNativeFetchError(error: unknown, limitBytes?: number) {
const message = error instanceof Error ? error.message : String(error);
const reason = [...SSRF_REASONS].find(reason => message.includes(reason));
if (reason) {
return createSsrfBlockedError(reason as SSRFBlockReason);
}
const idx = address.lastIndexOf(':');
if (idx === -1) {
return null;
if (message.includes('response_too_large')) {
return new ResponseTooLargeError({
limitBytes: limitBytes ?? 0,
receivedBytes: limitBytes ? limitBytes + 1 : 0,
});
}
const tail = address.slice(idx + 1);
return isIP(tail) === 4 ? tail : null;
return error;
}
function isBlockedIpAddress(address: string): boolean {
const ip = stripZoneId(address);
const family = isIP(ip);
if (family === 4) {
return BLOCKED_IPS.check(ip, 'ipv4');
}
if (family === 6) {
const embeddedV4 = extractEmbeddedIPv4FromIPv6(ip);
if (embeddedV4) {
return isBlockedIpAddress(embeddedV4);
}
if (!ALLOWED_IPV6.check(ip, 'ipv6')) {
return true;
}
return BLOCKED_IPS.check(ip, 'ipv6');
}
return true;
export interface SafeFetchOptions {
timeoutMs?: number;
maxRedirects?: number;
maxBytes?: number;
}
async function resolveHostAddresses(hostname: string): Promise<string[]> {
// Normalize common localhost aliases without DNS.
const lowered = hostname.toLowerCase();
if (lowered === 'localhost' || lowered.endsWith('.localhost')) {
return ['127.0.0.1', '::1'];
}
const results = await dnsLookup(hostname, {
all: true,
verbatim: true,
});
return results.map(r => r.address);
}
export async function assertSsrFSafeUrl(
rawUrl: string | URL,
options: SSRFProtectionOptions = {}
): Promise<URL> {
const allowedProtocols =
options.allowedProtocols ?? DEFAULT_ALLOWED_PROTOCOLS;
export async function assertSsrFSafeUrl(rawUrl: string | URL): Promise<URL> {
let url: URL;
try {
url = rawUrl instanceof URL ? rawUrl : new URL(rawUrl);
} catch {
throw createSsrfBlockedError('invalid_url', {
url: typeof rawUrl === 'string' ? rawUrl : undefined,
});
throw createSsrfBlockedError('invalid_url');
}
if (!allowedProtocols.has(url.protocol)) {
throw createSsrfBlockedError('disallowed_protocol', {
url: url.toString(),
});
}
if (url.username || url.password) {
throw createSsrfBlockedError('url_has_credentials', {
url: url.toString(),
});
}
const hostname = url.hostname;
if (!hostname) {
throw createSsrfBlockedError('blocked_hostname', { url: url.toString() });
}
const allowPrivate =
options.allowPrivateOrigins && options.allowPrivateOrigins.has(url.origin);
// IP literal
if (isIP(hostname)) {
if (isBlockedIpAddress(hostname) && !allowPrivate) {
throw createSsrfBlockedError('blocked_ip', {
url: url.toString(),
address: hostname,
});
}
return url;
}
let addresses: string[];
try {
addresses = await resolveHostAddresses(hostname);
} catch {
throw createSsrfBlockedError('unresolvable_hostname', {
url: url.toString(),
hostname,
});
assertSafeUrlFromNative({ url: url.toString() });
return url;
} catch (error) {
throw mapNativeFetchError(error);
}
if (addresses.length === 0) {
throw createSsrfBlockedError('unresolvable_hostname', {
url: url.toString(),
hostname,
});
}
for (const address of addresses) {
if (isBlockedIpAddress(address) && !allowPrivate) {
throw createSsrfBlockedError('blocked_ip', {
url: url.toString(),
hostname,
address,
});
}
}
return url;
}
export interface SafeFetchOptions extends SSRFProtectionOptions {
timeoutMs?: number;
maxRedirects?: number;
}
export async function safeFetch(
@@ -216,153 +70,51 @@ export async function safeFetch(
init: RequestInit = {},
options: SafeFetchOptions = {}
): Promise<Response> {
const timeoutMs = options.timeoutMs ?? 10_000;
const maxRedirects = options.maxRedirects ?? 3;
const timeoutSignal = AbortSignal.timeout(timeoutMs);
const signal = init.signal
? AbortSignal.any([init.signal, timeoutSignal])
: timeoutSignal;
let current = await assertSsrFSafeUrl(rawUrl, options);
let redirects = 0;
// Always handle redirects manually (SSRF-safe on each hop).
let requestInit: RequestInit = {
...init,
redirect: 'manual',
signal,
};
while (true) {
const response = await fetch(current, requestInit);
if (response.status >= 300 && response.status < 400) {
const location = response.headers.get('location');
if (!location) {
return response;
}
// Drain/cancel body before following redirect to avoid leaking resources.
try {
await response.body?.cancel();
} catch {
// ignore
}
if (redirects >= maxRedirects) {
throw createSsrfBlockedError('too_many_redirects', {
url: current.toString(),
});
}
const next = new URL(location, current);
current = await assertSsrFSafeUrl(next, options);
redirects += 1;
// 303 forces GET semantics
if (
response.status === 303 &&
requestInit.method &&
requestInit.method !== 'GET'
) {
requestInit = { ...requestInit, method: 'GET', body: undefined };
}
continue;
}
return response;
const url = rawUrl.toString();
const method = String(init.method ?? 'GET').toUpperCase();
if (method !== 'GET' && method !== 'HEAD') {
throw new Error(`Unsupported safeFetch method: ${method}`);
}
}
export async function readResponseBufferWithLimit(
response: Response,
limitBytes: number
): Promise<Buffer> {
const rawLen = response.headers.get('content-length');
if (rawLen) {
const len = Number.parseInt(rawLen, 10);
if (Number.isFinite(len) && len > limitBytes) {
try {
await response.body?.cancel();
} catch {
// ignore
}
throw new ResponseTooLargeError({ limitBytes, receivedBytes: len });
}
}
if (!response.body) {
return Buffer.alloc(0);
}
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let total = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
const chunk = value ?? new Uint8Array();
total += chunk.byteLength;
if (total > limitBytes) {
try {
await reader.cancel();
} catch {
// ignore
}
throw new ResponseTooLargeError({ limitBytes, receivedBytes: total });
}
chunks.push(chunk);
}
const response = await safeFetchFromNative({
url,
method: (method === 'HEAD' ? 'head' : 'get') as NonNullable<
SafeFetchRequest['method']
>,
headers: normalizeHeaders(init.headers),
timeoutMs: options.timeoutMs,
maxRedirects: options.maxRedirects,
maxBytes: options.maxBytes,
});
const body =
method === 'HEAD' || [204, 205, 304].includes(response.status)
? null
: response.body;
const webResponse = new Response(body, {
status: response.status,
headers: response.headers,
});
Object.defineProperty(webResponse, 'url', {
value: response.finalUrl,
});
return webResponse;
} catch (error) {
try {
await reader.cancel(error);
} catch {
// ignore
}
throw error;
} finally {
try {
reader.releaseLock();
} catch {
// ignore
}
throw mapNativeFetchError(error, options.maxBytes);
}
return Buffer.concat(
chunks.map(chunk => Buffer.from(chunk)),
total
);
}
type FetchBufferResult = { buffer: Buffer; type: string };
const ATTACH_GET_PARAMS = { timeoutMs: OneMinute / 6, maxRedirects: 3 };
export async function fetchBuffer(
url: string,
limit: number,
contentType?: string
): Promise<FetchBufferResult> {
const resp = url.startsWith('data:')
? await fetch(url)
: await safeFetch(url, { method: 'GET' }, ATTACH_GET_PARAMS);
if (!resp.ok) {
throw new Error(
`Failed to fetch attachment: ${resp.status} ${resp.statusText}`
);
function normalizeHeaders(headers: RequestInit['headers'] | undefined) {
if (!headers) return undefined;
if (headers instanceof Headers) {
return Object.fromEntries(headers.entries());
}
const type = resp.headers.get('content-type') || 'application/octet-stream';
if (contentType && !type.startsWith(contentType)) {
throw new Error(
`Attachment content-type mismatch: expected ${contentType} but got ${type}`
);
if (Array.isArray(headers)) {
return Object.fromEntries(headers);
}
const buffer = await readResponseBufferWithLimit(resp, limit);
return { buffer, type: type };
return Object.fromEntries(
Object.entries(headers).map(([key, value]) => [key, String(value)])
);
}
export function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
+21
View File
@@ -1,6 +1,7 @@
import serverNativeModule, {
type ActionEvent as NativeActionEventContract,
type ActionRuntimeInput as NativeActionRuntimeInputContract,
type AssertSafeUrlRequest,
type BuiltInPromptRenderContract,
type BuiltInPromptSessionContract,
type BuiltInPromptSpec,
@@ -8,6 +9,8 @@ import serverNativeModule, {
type CanonicalStructuredRequestContract,
type CapabilityAttachmentContract,
type CapabilityModelCapability,
type ImageInspection,
type ImageInspectionOptions,
type LlmCoreMessage,
type LlmEmbeddingRequestContract,
type LlmImageRequestContract,
@@ -27,16 +30,29 @@ import serverNativeModule, {
type PromptStructuredResponseContract,
type PromptTokenCountContract,
type PromptTokenCountResult,
type RemoteAttachmentFetchRequest,
type RemoteAttachmentFetchResponse,
type RemoteMimeTypeRequest,
type RequestedModelMatchResponse,
type SafeFetchRequest,
type SafeFetchResponse,
type Tokenizer,
} from '@affine/server-native';
export type {
AssertSafeUrlRequest,
CapabilityAttachmentContract,
CapabilityModelCapability,
ImageInspection,
ImageInspectionOptions,
ModelConditionsContract,
PromptMessageContract,
PromptStructuredResponseContract,
RemoteAttachmentFetchRequest,
RemoteAttachmentFetchResponse,
RemoteMimeTypeRequest,
SafeFetchRequest,
SafeFetchResponse,
};
export type ActionEventType =
@@ -118,6 +134,11 @@ export function getTokenEncoder(model?: string | null): Tokenizer | null {
}
export const getMime = serverNativeModule.getMime;
export const inspectImageForProxy = serverNativeModule.inspectImageForProxy;
export const fetchRemoteAttachment = serverNativeModule.fetchRemoteAttachment;
export const inferRemoteMimeType = serverNativeModule.inferRemoteMimeType;
export const assertSafeUrl = serverNativeModule.assertSafeUrl;
export const safeFetch = serverNativeModule.safeFetch;
export const parseDoc = serverNativeModule.parseDoc;
export const htmlSanitize = serverNativeModule.htmlSanitize;
export const processImage = serverNativeModule.processImage;
@@ -35,8 +35,6 @@ const XML_PARSER = new XMLParser({
const DEFAULT_REQUEST_TIMEOUT_MS = 10_000;
const DEFAULT_MAX_REDIRECTS = 5;
const CALDAV_ALLOWED_PROTOCOLS = new Set(['https:']);
const CALDAV_ALLOWED_PROTOCOLS_INSECURE = new Set(['http:', 'https:']);
type CalDAVCredentials = {
username: string;
@@ -607,11 +605,7 @@ class CalDAVRequestPolicy {
}
try {
await assertSsrFSafeUrl(url, {
allowedProtocols: this.allowInsecureHttp
? CALDAV_ALLOWED_PROTOCOLS_INSECURE
: CALDAV_ALLOWED_PROTOCOLS,
});
await assertSsrFSafeUrl(url);
} catch (error) {
if (error instanceof SsrfBlockedError) {
const reason = String(error.data?.reason ?? '');
@@ -2,7 +2,8 @@ import { Logger } from '@nestjs/common';
import { GoogleAuth, GoogleAuthOptions } from 'google-auth-library';
import z from 'zod';
import { OneMinute, safeFetch } from '../../../base';
import { OneMinute } from '../../../base';
import { inferRemoteMimeType } from '../../../native';
import { PromptAttachment, StreamObject } from './types';
export type VertexProviderConfig = {
@@ -33,30 +34,7 @@ type CopilotTextStreamPart =
}
| { type: 'error'; error: unknown };
const ATTACH_HEAD_PARAMS = { timeoutMs: OneMinute / 12, maxRedirects: 3 };
const FORMAT_INFER_MAP: Record<string, string> = {
pdf: 'application/pdf',
mp3: 'audio/mpeg',
opus: 'audio/opus',
ogg: 'audio/ogg',
aac: 'audio/aac',
m4a: 'audio/aac',
flac: 'audio/flac',
ogv: 'video/ogg',
wav: 'audio/wav',
png: 'image/png',
jpeg: 'image/jpeg',
jpg: 'image/jpeg',
webp: 'image/webp',
txt: 'text/plain',
md: 'text/plain',
mov: 'video/mov',
mpeg: 'video/mpeg',
mp4: 'video/mp4',
avi: 'video/avi',
wmv: 'video/wmv',
flv: 'video/flv',
};
const ATTACH_HEAD_TIMEOUT_MS = OneMinute / 12;
function toBase64Data(data: string, encoding: 'base64' | 'utf8' = 'base64') {
return encoding === 'base64'
@@ -97,25 +75,7 @@ export async function inferMimeType(url: string) {
if (url.startsWith('data:')) {
return url.split(';')[0].split(':')[1];
}
const pathname = new URL(url).pathname;
const extension = pathname.split('.').pop();
if (extension) {
const ext = FORMAT_INFER_MAP[extension];
if (ext) {
return ext;
}
}
try {
const mimeType = await safeFetch(
url,
{ method: 'HEAD' },
ATTACH_HEAD_PARAMS
).then(res => res.headers.get('content-type'));
if (mimeType) return mimeType;
} catch {
// ignore and fallback to default
}
return 'application/octet-stream';
return inferRemoteMimeType({ url, timeoutMs: ATTACH_HEAD_TIMEOUT_MS });
}
type CitationIndexedEvent = {
@@ -1,10 +1,7 @@
import { Injectable } from '@nestjs/common';
import {
Config,
readResponseBufferWithLimit,
safeFetch,
} from '../../../../base';
import { Config } from '../../../../base';
import { fetchRemoteAttachment } from '../../../../native';
type FetchRemoteAttachmentOptions = {
signal?: AbortSignal;
@@ -37,9 +34,12 @@ export class AttachmentMaterializer {
constructor(private readonly config: Config) {}
private buildFetchOptions(url: URL, trustedHostSuffixes: string[]) {
const baseOptions = { timeoutMs: 15_000, maxRedirects: 3 } as const;
const baseOptions = {
timeoutMs: 15_000,
allowPrivateTargetOrigin: false,
};
if (!env.prod) {
return { ...baseOptions, allowPrivateOrigins: new Set([url.origin]) };
return { ...baseOptions, allowPrivateTargetOrigin: true };
}
const trustedOrigins = new Set<string>();
@@ -80,7 +80,7 @@ export class AttachmentMaterializer {
suffix => hostname === suffix || hostname.endsWith(`.${suffix}`)
);
if (trustedOrigins.has(url.origin) || trustedByHost) {
return { ...baseOptions, allowPrivateOrigins: new Set([url.origin]) };
return { ...baseOptions, allowPrivateTargetOrigin: true };
}
return baseOptions;
@@ -91,30 +91,22 @@ export class AttachmentMaterializer {
options: FetchRemoteAttachmentOptions
) {
const parsed = resolveAttachmentFetchUrl(url);
const response = await safeFetch(
parsed,
{ method: 'GET', signal: options.signal },
this.buildFetchOptions(parsed, options.trustedHostSuffixes ?? [])
);
if (!response.ok) {
throw new Error(
`Failed to fetch attachment: ${response.status} ${response.statusText}`
);
}
const buffer = await readResponseBufferWithLimit(
response,
options.maxBytes
);
const headerMimeType = normalizeMimeType(
response.headers.get('content-type') || ''
);
options.signal?.throwIfAborted();
const response = await fetchRemoteAttachment({
url: parsed.toString(),
...this.buildFetchOptions(parsed, options.trustedHostSuffixes ?? []),
maxBytes: options.maxBytes,
});
options.signal?.throwIfAborted();
return {
data: buffer.toString('base64'),
data: response.body.toString('base64'),
mimeType: options.detectMimeType
? options.detectMimeType(buffer, headerMimeType)
: headerMimeType,
? options.detectMimeType(
response.body,
normalizeMimeType(response.mimeType)
)
: normalizeMimeType(response.mimeType),
};
}
}
@@ -7,7 +7,6 @@ import {
BlobQuotaExceeded,
CallMetric,
Config,
fetchBuffer,
type FileUpload,
OneMB,
OnEvent,
@@ -17,6 +16,7 @@ import {
URLHelper,
} from '../../base';
import { QuotaService } from '../../core/quota';
import { fetchRemoteAttachment } from '../../native';
const REMOTE_BLOB_MAX_BYTES = 20 * OneMB;
@@ -93,12 +93,15 @@ export class CopilotStorage {
@CallMetric('ai', 'blob_proxy_remote_url')
async handleRemoteLink(userId: string, workspaceId: string, link: string) {
const { buffer, type } = await fetchBuffer(
link,
REMOTE_BLOB_MAX_BYTES,
'image/'
);
const { body, mimeType } = await fetchRemoteAttachment({
url: link,
maxBytes: REMOTE_BLOB_MAX_BYTES,
expectedContentTypePrefix: 'image/',
maxImageHeight: 4096,
maxImageWidth: 4096,
});
const buffer = Buffer.from(body);
const filename = createHash('sha256').update(buffer).digest('base64url');
return this.put(userId, workspaceId, filename, buffer, type);
return this.put(userId, workspaceId, filename, buffer, mimeType);
}
}
@@ -17,15 +17,16 @@ import {
applyAttachHeaders,
BadRequest,
Cache,
readResponseBufferWithLimit,
ResponseTooLargeError,
safeFetch,
SsrfBlockedError,
type SSRFBlockReason,
Throttle,
URLHelper,
UseNamedGuard,
} from '../../base';
import { Public } from '../../core/auth';
import { inspectImageForProxy } from '../../native';
import { WorkerService } from './service';
import type { LinkPreviewRequest, LinkPreviewResponse } from './types';
import {
@@ -64,6 +65,7 @@ function toBadRequestReason(reason: SSRFBlockReason) {
@Public()
@UseNamedGuard('selfhost')
@Throttle('default', { limit: 60, ttl: 60_000 })
@Controller('/api/worker')
export class WorkerController {
private readonly logger = new Logger(WorkerController.name);
@@ -133,7 +135,8 @@ export class WorkerController {
...(origin ? { Vary: 'Origin' } : {}),
'Access-Control-Allow-Methods': 'GET',
});
applyAttachHeaders(resp, { buffer });
const image = this.inspectProxyImage(buffer, imageURL);
applyAttachHeaders(resp, { buffer, contentType: image.mimeType });
const contentType = resp.getHeader('Content-Type') as string | undefined;
if (contentType?.startsWith('image/')) {
return resp.status(200).send(buffer);
@@ -147,7 +150,11 @@ export class WorkerController {
response = await safeFetch(
targetURL.toString(),
{ method: 'GET', headers: cloneHeader(req.headers) },
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
{
timeoutMs: FETCH_TIMEOUT_MS,
maxRedirects: MAX_REDIRECTS,
maxBytes: IMAGE_PROXY_MAX_BYTES,
}
);
} catch (error) {
if (error instanceof SsrfBlockedError) {
@@ -155,7 +162,6 @@ export class WorkerController {
this.logger.warn('Blocked image proxy target', {
url: imageURL,
reason,
context: (error as any).context,
});
throw new BadRequest(toBadRequestReason(reason ?? 'invalid_url'));
}
@@ -175,23 +181,8 @@ export class WorkerController {
throw new BadRequest('Failed to fetch image');
}
if (response.ok) {
let buffer: Buffer;
try {
buffer = await readResponseBufferWithLimit(
response,
IMAGE_PROXY_MAX_BYTES
);
} catch (error) {
if (error instanceof ResponseTooLargeError) {
this.logger.warn('Image proxy response too large', {
url: imageURL,
limitBytes: error.data?.limitBytes,
receivedBytes: error.data?.receivedBytes,
});
throw new BadRequest('Response too large');
}
throw error;
}
const buffer = Buffer.from(await response.arrayBuffer());
const image = this.inspectProxyImage(buffer, imageURL);
await this.cache.set(cachedUrl, buffer.toString('base64'), {
ttl: CACHE_TTL,
});
@@ -204,7 +195,7 @@ export class WorkerController {
if (contentDisposition) {
resp.setHeader('Content-Disposition', contentDisposition);
}
applyAttachHeaders(resp, { buffer });
applyAttachHeaders(resp, { buffer, contentType: image.mimeType });
const contentType = resp.getHeader('Content-Type') as string | undefined;
if (contentType?.startsWith('image/')) {
return resp.status(200).send(buffer);
@@ -227,6 +218,18 @@ export class WorkerController {
}
}
private inspectProxyImage(buffer: Buffer, url: string) {
try {
return inspectImageForProxy(buffer);
} catch (error) {
this.logger.warn('Image proxy rejected invalid image', {
url,
error,
});
throw new BadRequest('Invalid image');
}
}
@Options('/link-preview')
linkPreviewOption(
@Req() request: ExpressRequest,
@@ -291,7 +294,11 @@ export class WorkerController {
const response = await safeFetch(
targetURL.toString(),
{ method, headers: cloneHeader(request.headers) },
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
{
timeoutMs: FETCH_TIMEOUT_MS,
maxRedirects: MAX_REDIRECTS,
maxBytes: LINK_PREVIEW_MAX_BYTES,
}
);
this.logger.debug('Fetched URL', {
origin,
@@ -318,10 +325,7 @@ export class WorkerController {
};
if (response.body) {
const body = await readResponseBufferWithLimit(
response,
LINK_PREVIEW_MAX_BYTES
);
const body = Buffer.from(await response.arrayBuffer());
const limitedResponse = new Response(body, response);
const resp = await decodeWithCharset(limitedResponse, res);
@@ -402,7 +406,11 @@ export class WorkerController {
const faviconResponse = await safeFetch(
faviconUrl.toString(),
{ method: 'HEAD' },
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
{
timeoutMs: FETCH_TIMEOUT_MS,
maxRedirects: MAX_REDIRECTS,
maxBytes: 0,
}
);
if (faviconResponse.ok) {
appendUrl(faviconUrl.toString(), res.favicons);
@@ -433,7 +441,6 @@ export class WorkerController {
origin,
url: requestBody?.url,
reason,
context: (error as any).context,
});
throw new BadRequest(toBadRequestReason(reason ?? 'invalid_url'));
}