mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-14 08:36:22 +08:00
feat(server): native safe fetch (#14931)
This commit is contained in:
@@ -135,17 +135,17 @@
|
||||
},
|
||||
"throttlers.default": {
|
||||
"type": "object",
|
||||
"description": "The config for the default throttler.\n@default {\"ttl\":60,\"limit\":120}",
|
||||
"description": "The config for the default throttler.\n@default {\"ttl\":60000,\"limit\":120}",
|
||||
"default": {
|
||||
"ttl": 60,
|
||||
"ttl": 60000,
|
||||
"limit": 120
|
||||
}
|
||||
},
|
||||
"throttlers.strict": {
|
||||
"type": "object",
|
||||
"description": "The config for the strict throttler.\n@default {\"ttl\":60,\"limit\":20}",
|
||||
"description": "The config for the strict throttler.\n@default {\"ttl\":60000,\"limit\":20}",
|
||||
"default": {
|
||||
"ttl": 60,
|
||||
"ttl": 60000,
|
||||
"limit": 20
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+78
-2
@@ -209,12 +209,14 @@ dependencies = [
|
||||
"napi-derive",
|
||||
"rand 0.9.4",
|
||||
"rayon",
|
||||
"reqwest",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha3",
|
||||
"tiktoken-rs",
|
||||
"tokio",
|
||||
"url",
|
||||
"v_htmlescape",
|
||||
"y-octo",
|
||||
]
|
||||
@@ -3719,6 +3721,12 @@ dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru-slab"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||
|
||||
[[package]]
|
||||
name = "mac"
|
||||
version = "0.1.1"
|
||||
@@ -4978,6 +4986,62 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"cfg_aliases",
|
||||
"pin-project-lite",
|
||||
"quinn-proto",
|
||||
"quinn-udp",
|
||||
"rustc-hash 2.1.1",
|
||||
"rustls",
|
||||
"socket2",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn-proto"
|
||||
version = "0.11.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"bytes",
|
||||
"getrandom 0.3.4",
|
||||
"lru-slab",
|
||||
"rand 0.9.4",
|
||||
"ring",
|
||||
"rustc-hash 2.1.1",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"slab",
|
||||
"thiserror 2.0.18",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn-udp"
|
||||
version = "0.5.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
|
||||
dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
@@ -5232,9 +5296,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.13.2"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801"
|
||||
checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes",
|
||||
@@ -5252,6 +5316,7 @@ dependencies = [
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"quinn",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"rustls-platform-verifier",
|
||||
@@ -5459,6 +5524,7 @@ version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
|
||||
dependencies = [
|
||||
"web-time",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -8130,6 +8196,16 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-time"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.7"
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
|
||||
import { bilibiliConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/bilibili.js';
|
||||
import { excalidrawConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/excalidraw.js';
|
||||
import { genericConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/generic.js';
|
||||
import { googleDocsConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/google-docs.js';
|
||||
import { googleDriveConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/google-drive.js';
|
||||
import { miroConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/miro.js';
|
||||
import { spotifyConfig } from '../../../blocks/embed/src/embed-iframe-block/configs/providers/spotify.js';
|
||||
|
||||
describe('embed iframe provider config', () => {
|
||||
test('validates final iframe URLs from oEmbed providers', () => {
|
||||
expect(
|
||||
spotifyConfig.validateIframeUrl?.(
|
||||
'https://open.spotify.com/embed/track/0TK2YIli7K1leLovkQiNik'
|
||||
)
|
||||
).toBe(true);
|
||||
expect(
|
||||
spotifyConfig.validateIframeUrl?.(
|
||||
'https://example.com/embed/track/0TK2YIli7K1leLovkQiNik'
|
||||
)
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test('validates provider-specific iframe URL shapes', () => {
|
||||
expect(
|
||||
googleDriveConfig.validateIframeUrl?.(
|
||||
'https://drive.google.com/file/d/file-id/preview?usp=embed_googleplus'
|
||||
)
|
||||
).toBe(true);
|
||||
expect(
|
||||
googleDriveConfig.validateIframeUrl?.(
|
||||
'https://drive.google.com/drive/folders/folder-id?usp=sharing'
|
||||
)
|
||||
).toBe(false);
|
||||
|
||||
expect(
|
||||
bilibiliConfig.validateIframeUrl?.(
|
||||
'https://player.bilibili.com/player.html?bvid=BV1xx411c7mD&autoplay=0'
|
||||
)
|
||||
).toBe(true);
|
||||
expect(
|
||||
bilibiliConfig.match(
|
||||
'https://player.bilibili.com/player.html?aid=123&autoplay=0'
|
||||
)
|
||||
).toBe(true);
|
||||
expect(
|
||||
bilibiliConfig.buildOEmbedUrl(
|
||||
'https://player.bilibili.com/video/BV1xx411c7mD'
|
||||
)
|
||||
).toBe(
|
||||
'https://player.bilibili.com/player.html?bvid=BV1xx411c7mD&autoplay=0'
|
||||
);
|
||||
expect(
|
||||
bilibiliConfig.validateIframeUrl?.(
|
||||
'https://www.bilibili.com/video/BV1xx411c7mD'
|
||||
)
|
||||
).toBe(false);
|
||||
|
||||
expect(
|
||||
googleDocsConfig.validateIframeUrl?.(
|
||||
'https://docs.google.com/document/d/doc-id/edit?usp=sharing'
|
||||
)
|
||||
).toBe(true);
|
||||
expect(
|
||||
miroConfig.validateIframeUrl?.(
|
||||
'https://miro.com/app/live-embed/board-id/'
|
||||
)
|
||||
).toBe(true);
|
||||
expect(
|
||||
excalidrawConfig.validateIframeUrl?.('https://excalidraw.com/#room-id')
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test('generic iframe validation excludes affine and non-https URLs', () => {
|
||||
expect(genericConfig.validateIframeUrl?.('https://example.com/embed')).toBe(
|
||||
true
|
||||
);
|
||||
expect(genericConfig.validateIframeUrl?.('http://example.com/embed')).toBe(
|
||||
false
|
||||
);
|
||||
expect(
|
||||
genericConfig.validateIframeUrl?.('https://app.affine.pro/embed')
|
||||
).toBe(false);
|
||||
expect(genericConfig.validateIframeUrl?.('https://127.0.0.1/embed')).toBe(
|
||||
false
|
||||
);
|
||||
expect(genericConfig.validateIframeUrl?.('https://localhost/embed')).toBe(
|
||||
false
|
||||
);
|
||||
});
|
||||
});
|
||||
+22
-4
@@ -35,7 +35,7 @@ const extractBvid = (url: string) => {
|
||||
|
||||
const buildBiliPlayerEmbedUrl = (url: string) => {
|
||||
// If the user pasted the embed URL directly, keep it
|
||||
if (validateEmbedIframeUrl(url, biliPlayerValidationOptions)) {
|
||||
if (isValidBiliPlayerUrl(url)) {
|
||||
return url;
|
||||
}
|
||||
const avid = extractAvid(url);
|
||||
@@ -57,13 +57,31 @@ const buildBiliPlayerEmbedUrl = (url: string) => {
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const bilibiliConfig = {
|
||||
function isValidBiliPlayerUrl(url: string) {
|
||||
try {
|
||||
if (!validateEmbedIframeUrl(url, biliPlayerValidationOptions)) {
|
||||
return false;
|
||||
}
|
||||
const parsedUrl = new URL(url);
|
||||
return (
|
||||
parsedUrl.pathname === '/player.html' &&
|
||||
(!!parsedUrl.searchParams.get('aid') ||
|
||||
!!parsedUrl.searchParams.get('bvid'))
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export const bilibiliConfig = {
|
||||
name: 'bilibili',
|
||||
match: (url: string) =>
|
||||
validateEmbedIframeUrl(url, bilibiliValidationOptions) &&
|
||||
(!!extractAvid(url) || !!extractBvid(url)),
|
||||
isValidBiliPlayerUrl(url) ||
|
||||
(validateEmbedIframeUrl(url, bilibiliValidationOptions) &&
|
||||
(!!extractAvid(url) || !!extractBvid(url))),
|
||||
buildOEmbedUrl: buildBiliPlayerEmbedUrl,
|
||||
useOEmbedUrlDirectly: true,
|
||||
validateIframeUrl: (iframeUrl: string) => isValidBiliPlayerUrl(iframeUrl),
|
||||
options: {
|
||||
widthInSurface: BILIBILI_DEFAULT_WIDTH_IN_SURFACE,
|
||||
heightInSurface: BILIBILI_DEFAULT_HEIGHT_IN_SURFACE,
|
||||
|
||||
+3
-1
@@ -15,7 +15,7 @@ const excalidrawUrlValidationOptions: EmbedIframeUrlValidationOptions = {
|
||||
hostnames: ['excalidraw.com'],
|
||||
};
|
||||
|
||||
const excalidrawConfig = {
|
||||
export const excalidrawConfig = {
|
||||
name: 'excalidraw',
|
||||
match: (url: string) =>
|
||||
validateEmbedIframeUrl(url, excalidrawUrlValidationOptions),
|
||||
@@ -27,6 +27,8 @@ const excalidrawConfig = {
|
||||
return url;
|
||||
},
|
||||
useOEmbedUrlDirectly: true,
|
||||
validateIframeUrl: (iframeUrl: string) =>
|
||||
validateEmbedIframeUrl(iframeUrl, excalidrawUrlValidationOptions),
|
||||
options: {
|
||||
widthInSurface: EXCALIDRAW_DEFAULT_WIDTH_IN_SURFACE,
|
||||
heightInSurface: EXCALIDRAW_DEFAULT_HEIGHT_IN_SURFACE,
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { EmbedIframeConfigExtension } from '@blocksuite/affine-shared/services';
|
||||
|
||||
import {
|
||||
type EmbedIframeUrlValidationOptions,
|
||||
validateEmbedIframeUrl,
|
||||
} from '../../utils';
|
||||
|
||||
const GENERIC_DEFAULT_WIDTH_IN_SURFACE = 800;
|
||||
const GENERIC_DEFAULT_HEIGHT_IN_SURFACE = 600;
|
||||
const GENERIC_DEFAULT_WIDTH_PERCENT = 100;
|
||||
@@ -17,6 +22,11 @@ const AFFINE_DOMAINS = [
|
||||
'apple.getaffineapp.com', // Cloud domain for Apple app
|
||||
];
|
||||
|
||||
const genericUrlValidationOptions: EmbedIframeUrlValidationOptions = {
|
||||
protocols: ['https:'],
|
||||
hostnames: [],
|
||||
};
|
||||
|
||||
/**
|
||||
* Validates if a URL is suitable for generic iframe embedding
|
||||
* Allows HTTPS URLs but excludes AFFiNE domains
|
||||
@@ -27,8 +37,12 @@ function isValidGenericEmbedUrl(url: string): boolean {
|
||||
try {
|
||||
const parsedUrl = new URL(url);
|
||||
|
||||
// Only allow HTTPS for security
|
||||
if (parsedUrl.protocol !== 'https:') {
|
||||
if (
|
||||
!validateEmbedIframeUrl(url, {
|
||||
...genericUrlValidationOptions,
|
||||
hostnames: [parsedUrl.hostname],
|
||||
})
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -49,7 +63,7 @@ function isValidGenericEmbedUrl(url: string): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
const genericConfig = {
|
||||
export const genericConfig = {
|
||||
name: 'generic',
|
||||
match: (url: string) => isValidGenericEmbedUrl(url),
|
||||
buildOEmbedUrl: (url: string) => {
|
||||
@@ -59,6 +73,7 @@ const genericConfig = {
|
||||
return url;
|
||||
},
|
||||
useOEmbedUrlDirectly: true,
|
||||
validateIframeUrl: (iframeUrl: string) => isValidGenericEmbedUrl(iframeUrl),
|
||||
options: {
|
||||
widthInSurface: GENERIC_DEFAULT_WIDTH_IN_SURFACE,
|
||||
heightInSurface: GENERIC_DEFAULT_HEIGHT_IN_SURFACE,
|
||||
|
||||
+2
-1
@@ -57,7 +57,7 @@ function isValidGoogleDocsUrl(url: string, strictMode = true): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
const googleDocsConfig = {
|
||||
export const googleDocsConfig = {
|
||||
name: 'google-docs',
|
||||
match: (url: string) => isValidGoogleDocsUrl(url),
|
||||
buildOEmbedUrl: (url: string) => {
|
||||
@@ -67,6 +67,7 @@ const googleDocsConfig = {
|
||||
return url;
|
||||
},
|
||||
useOEmbedUrlDirectly: true,
|
||||
validateIframeUrl: (iframeUrl: string) => isValidGoogleDocsUrl(iframeUrl),
|
||||
options: {
|
||||
widthInSurface: GOOGLE_DOCS_DEFAULT_WIDTH_IN_SURFACE,
|
||||
heightInSurface: GOOGLE_DOCS_DEFAULT_HEIGHT_IN_SURFACE,
|
||||
|
||||
+26
-1
@@ -113,6 +113,29 @@ function isValidGoogleDriveUrl(url: string, strictMode = true): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
function isValidGoogleDriveIframeUrl(url: string): boolean {
|
||||
try {
|
||||
if (!validateEmbedIframeUrl(url, googleDriveUrlValidationOptions)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const parsedUrl = new URL(url);
|
||||
const pathSegments = parsedUrl.pathname.split('/').filter(Boolean);
|
||||
|
||||
if (isValidGoogleDriveFileUrl(parsedUrl)) {
|
||||
return pathSegments[3] === 'preview';
|
||||
}
|
||||
|
||||
return (
|
||||
parsedUrl.pathname === '/embeddedfolderview' &&
|
||||
!!parsedUrl.searchParams.get('id')
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn('Invalid Google Drive iframe URL:', e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build embed URL for Google Drive files
|
||||
* @param fileId File ID
|
||||
@@ -171,7 +194,7 @@ function buildGoogleDriveEmbedUrl(url: string): string | undefined {
|
||||
}
|
||||
}
|
||||
|
||||
const googleDriveConfig = {
|
||||
export const googleDriveConfig = {
|
||||
name: 'google-drive',
|
||||
match: (url: string) => isValidGoogleDriveUrl(url),
|
||||
buildOEmbedUrl: (url: string) => {
|
||||
@@ -183,6 +206,8 @@ const googleDriveConfig = {
|
||||
return buildGoogleDriveEmbedUrl(url);
|
||||
},
|
||||
useOEmbedUrlDirectly: true,
|
||||
validateIframeUrl: (iframeUrl: string) =>
|
||||
isValidGoogleDriveIframeUrl(iframeUrl),
|
||||
options: {
|
||||
widthInSurface: GOOGLE_DRIVE_DEFAULT_WIDTH_IN_SURFACE,
|
||||
heightInSurface: GOOGLE_DRIVE_DEFAULT_HEIGHT_IN_SURFACE,
|
||||
|
||||
@@ -18,7 +18,7 @@ const miroUrlValidationOptions: EmbedIframeUrlValidationOptions = {
|
||||
hostnames: ['miro.com'],
|
||||
};
|
||||
|
||||
const miroConfig = {
|
||||
export const miroConfig = {
|
||||
name: 'miro',
|
||||
match: (url: string) => validateEmbedIframeUrl(url, miroUrlValidationOptions),
|
||||
buildOEmbedUrl: (url: string) => {
|
||||
@@ -31,6 +31,12 @@ const miroConfig = {
|
||||
return oEmbedUrl;
|
||||
},
|
||||
useOEmbedUrlDirectly: false,
|
||||
validateIframeUrl: (iframeUrl: string) => {
|
||||
if (!validateEmbedIframeUrl(iframeUrl, miroUrlValidationOptions)) {
|
||||
return false;
|
||||
}
|
||||
return new URL(iframeUrl).pathname.startsWith('/app/live-embed/');
|
||||
},
|
||||
options: {
|
||||
widthInSurface: MIRO_DEFAULT_WIDTH_IN_SURFACE,
|
||||
heightInSurface: MIRO_DEFAULT_HEIGHT_IN_SURFACE,
|
||||
|
||||
@@ -18,7 +18,12 @@ const spotifyUrlValidationOptions: EmbedIframeUrlValidationOptions = {
|
||||
hostnames: ['open.spotify.com', 'spotify.link'],
|
||||
};
|
||||
|
||||
const spotifyConfig = {
|
||||
const spotifyIframeUrlValidationOptions: EmbedIframeUrlValidationOptions = {
|
||||
protocols: ['https:'],
|
||||
hostnames: ['open.spotify.com'],
|
||||
};
|
||||
|
||||
export const spotifyConfig = {
|
||||
name: 'spotify',
|
||||
match: (url: string) =>
|
||||
validateEmbedIframeUrl(url, spotifyUrlValidationOptions),
|
||||
@@ -32,6 +37,13 @@ const spotifyConfig = {
|
||||
return oEmbedUrl;
|
||||
},
|
||||
useOEmbedUrlDirectly: false,
|
||||
validateIframeUrl: (iframeUrl: string) => {
|
||||
if (!validateEmbedIframeUrl(iframeUrl, spotifyIframeUrlValidationOptions)) {
|
||||
return false;
|
||||
}
|
||||
const parsedUrl = new URL(iframeUrl);
|
||||
return parsedUrl.pathname.split('/').find(Boolean) === 'embed';
|
||||
},
|
||||
options: {
|
||||
widthInSurface: SPOTIFY_DEFAULT_WIDTH_IN_SURFACE,
|
||||
heightInSurface: SPOTIFY_DEFAULT_HEIGHT_IN_SURFACE,
|
||||
|
||||
@@ -141,7 +141,7 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
|
||||
});
|
||||
return;
|
||||
}
|
||||
window.open(link, '_blank');
|
||||
window.open(link, '_blank', 'noopener,noreferrer');
|
||||
};
|
||||
|
||||
refreshData = async () => {
|
||||
@@ -183,6 +183,12 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
|
||||
|
||||
// update model
|
||||
const iframeUrl = this._getIframeUrl(embedData) ?? currentIframeUrl;
|
||||
if (!this._validateIframeUrl(url, iframeUrl)) {
|
||||
throw new BlockSuiteError(
|
||||
ErrorCode.ValueNotExists,
|
||||
'Invalid embed iframe url'
|
||||
);
|
||||
}
|
||||
this.store.updateBlock(this.model, {
|
||||
iframeUrl,
|
||||
title: embedData?.title || previewData?.title,
|
||||
@@ -291,6 +297,19 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
|
||||
}
|
||||
};
|
||||
|
||||
private readonly _validateIframeUrl = (url: string, iframeUrl?: string) => {
|
||||
if (!iframeUrl) {
|
||||
return false;
|
||||
}
|
||||
const config = this.embedIframeService?.getConfig(url);
|
||||
if (!config) {
|
||||
return false;
|
||||
}
|
||||
return config.validateIframeUrl
|
||||
? config.validateIframeUrl(iframeUrl, url)
|
||||
: config.match(iframeUrl);
|
||||
};
|
||||
|
||||
private readonly _handleDoubleClick = () => {
|
||||
this.open();
|
||||
};
|
||||
@@ -329,6 +348,16 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
|
||||
|
||||
private readonly _renderIframe = () => {
|
||||
const { iframeUrl } = this.model.props;
|
||||
if (!iframeUrl || !this._isIframeUrlAllowed(iframeUrl)) {
|
||||
return html`<embed-iframe-error-card
|
||||
.error=${new Error('Invalid iframe URL')}
|
||||
.model=${this.model}
|
||||
.onRetry=${this._handleRetry}
|
||||
.std=${this.std}
|
||||
.inSurface=${this.inSurface}
|
||||
.options=${this._statusCardOptions}
|
||||
></embed-iframe-error-card>`;
|
||||
}
|
||||
const {
|
||||
widthPercent,
|
||||
heightInNote,
|
||||
@@ -368,6 +397,10 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
|
||||
: nothing}`;
|
||||
};
|
||||
|
||||
private readonly _isIframeUrlAllowed = (iframeUrl: string) => {
|
||||
return this._validateIframeUrl(this.model.props.url, iframeUrl);
|
||||
};
|
||||
|
||||
private readonly _getSourceHost = () => {
|
||||
const url = this.model.props.url ?? this.model.props.iframeUrl;
|
||||
if (!url) return null;
|
||||
@@ -437,7 +470,12 @@ export class EmbedIframeBlockComponent extends CaptionedBlockComponent<EmbedIfra
|
||||
} else {
|
||||
// update iframe options, to ensure the iframe is rendered with the correct options
|
||||
this._updateIframeOptions(this.model.props.url);
|
||||
this.status$.value = 'success';
|
||||
this.status$.value = this._validateIframeUrl(
|
||||
this.model.props.url,
|
||||
this.model.props.iframeUrl
|
||||
)
|
||||
? 'success'
|
||||
: 'error';
|
||||
}
|
||||
|
||||
// refresh data when original url changes
|
||||
|
||||
@@ -9,6 +9,25 @@ export interface EmbedIframeUrlValidationOptions {
|
||||
hostnames: string[]; // Allowed hostnames, e.g. ['docs.google.com']
|
||||
}
|
||||
|
||||
function isLocalOrIpHostname(hostname: string): boolean {
|
||||
const lower = hostname.toLowerCase();
|
||||
if (
|
||||
lower === 'localhost' ||
|
||||
lower.endsWith('.localhost') ||
|
||||
lower === '0.0.0.0' ||
|
||||
lower === '::' ||
|
||||
lower === '::1'
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (/^\d{1,3}(\.\d{1,3}){3}$/.test(lower)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return lower.startsWith('[') && lower.endsWith(']');
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the url is allowed to embed in the iframe
|
||||
* @param url URL to validate
|
||||
@@ -23,6 +42,15 @@ export function validateEmbedIframeUrl(
|
||||
const parsedUrl = new URL(url);
|
||||
|
||||
const { protocols, hostnames } = options;
|
||||
if (
|
||||
parsedUrl.username ||
|
||||
parsedUrl.password ||
|
||||
parsedUrl.port ||
|
||||
isLocalOrIpHostname(parsedUrl.hostname)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
protocols.includes(parsedUrl.protocol) &&
|
||||
hostnames.includes(parsedUrl.hostname)
|
||||
|
||||
@@ -66,6 +66,10 @@ export type EmbedIframeConfig = {
|
||||
* The function to build the oEmbed URL for fetching embed data
|
||||
*/
|
||||
buildOEmbedUrl: (url: string) => string | undefined;
|
||||
/**
|
||||
* Validate the final iframe src before rendering.
|
||||
*/
|
||||
validateIframeUrl?: (iframeUrl: string, originalUrl?: string) => boolean;
|
||||
/**
|
||||
* Use oEmbed URL directly as iframe src without fetching oEmbed data
|
||||
*/
|
||||
|
||||
@@ -31,11 +31,16 @@ mp4parse = { workspace = true }
|
||||
napi = { workspace = true, features = ["async", "serde-json"] }
|
||||
napi-derive = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
reqwest = { version = "0.13.3", default-features = false, features = [
|
||||
"blocking",
|
||||
"rustls",
|
||||
] }
|
||||
schemars = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
sha3 = { workspace = true }
|
||||
tiktoken-rs = { workspace = true }
|
||||
url = { workspace = true }
|
||||
v_htmlescape = { workspace = true }
|
||||
y-octo = { workspace = true, features = ["large_refs"] }
|
||||
|
||||
|
||||
Vendored
+67
@@ -66,6 +66,12 @@ export const AFFINE_PRO_LICENSE_AES_KEY: string | undefined | null
|
||||
|
||||
export const AFFINE_PRO_PUBLIC_KEY: string | undefined | null
|
||||
|
||||
export declare function assertSafeUrl(request: AssertSafeUrlRequest): void
|
||||
|
||||
export interface AssertSafeUrlRequest {
|
||||
url: string
|
||||
}
|
||||
|
||||
export declare function buildPublicRootDoc(rootDocBin: Buffer, docMetas: Array<PublicDocMetaInput>): Buffer
|
||||
|
||||
export interface BuiltInPromptRenderContract {
|
||||
@@ -163,12 +169,30 @@ export interface Chunk {
|
||||
*/
|
||||
export declare function createDocWithMarkdown(title: string, markdown: string, docId: string): Buffer
|
||||
|
||||
export declare function fetchRemoteAttachment(request: RemoteAttachmentFetchRequest): Promise<RemoteAttachmentFetchResponse>
|
||||
|
||||
export declare function fromModelName(modelName: string): Tokenizer | null
|
||||
|
||||
export declare function getMime(input: Uint8Array): string
|
||||
|
||||
export declare function htmlSanitize(input: string): string
|
||||
|
||||
export interface ImageInspection {
|
||||
mimeType: string
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
export interface ImageInspectionOptions {
|
||||
maxWidth?: number
|
||||
maxHeight?: number
|
||||
maxPixels?: number
|
||||
}
|
||||
|
||||
export declare function inferRemoteMimeType(request: RemoteMimeTypeRequest): Promise<string>
|
||||
|
||||
export declare function inspectImageForProxy(input: Buffer, options?: ImageInspectionOptions | undefined | null): ImageInspection
|
||||
|
||||
export declare function llmBuildCanonicalRequest(request: CanonicalChatRequestContract): LlmRequestContract
|
||||
|
||||
export declare function llmBuildCanonicalStructuredRequest(request: CanonicalStructuredRequestContract): LlmStructuredRequestContract
|
||||
@@ -572,6 +596,28 @@ export interface PublicDocMetaInput {
|
||||
|
||||
export declare function readAllDocIdsFromRootDoc(docBin: Buffer, includeTrash?: boolean | undefined | null): Array<string>
|
||||
|
||||
export interface RemoteAttachmentFetchRequest {
|
||||
url: string
|
||||
timeoutMs?: number
|
||||
maxBytes: number
|
||||
allowPrivateTargetOrigin?: boolean
|
||||
expectedContentTypePrefix?: string
|
||||
maxImageWidth?: number
|
||||
maxImageHeight?: number
|
||||
maxImagePixels?: number
|
||||
}
|
||||
|
||||
export interface RemoteAttachmentFetchResponse {
|
||||
finalUrl: string
|
||||
mimeType: string
|
||||
body: Buffer
|
||||
}
|
||||
|
||||
export interface RemoteMimeTypeRequest {
|
||||
url: string
|
||||
timeoutMs?: number
|
||||
}
|
||||
|
||||
export interface RequestedModelMatchRequest {
|
||||
providerIds: Array<string>
|
||||
optionalModels: Array<string>
|
||||
@@ -591,6 +637,27 @@ export interface RerankCandidate {
|
||||
|
||||
export declare function runNativeActionRecipePreparedStream(input: ActionRuntimeInput, callback: ((err: Error | null, arg: string) => void)): LlmStreamHandle
|
||||
|
||||
export declare function safeFetch(request: SafeFetchRequest): Promise<SafeFetchResponse>
|
||||
|
||||
export type SafeFetchMethod = 'get'|
|
||||
'head';
|
||||
|
||||
export interface SafeFetchRequest {
|
||||
url: string
|
||||
method?: SafeFetchMethod
|
||||
headers?: Record<string, string>
|
||||
timeoutMs?: number
|
||||
maxRedirects?: number
|
||||
maxBytes?: number
|
||||
}
|
||||
|
||||
export interface SafeFetchResponse {
|
||||
status: number
|
||||
finalUrl: string
|
||||
headers: Record<string, string>
|
||||
body: Buffer
|
||||
}
|
||||
|
||||
export interface ToolContract {
|
||||
name: string
|
||||
description?: string
|
||||
|
||||
@@ -9,6 +9,7 @@ pub mod hashcash;
|
||||
pub mod html_sanitize;
|
||||
pub mod image;
|
||||
pub mod llm;
|
||||
pub mod safe_fetch;
|
||||
pub mod tiktoken;
|
||||
|
||||
use affine_common::napi_utils::map_napi_err;
|
||||
|
||||
@@ -0,0 +1,668 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
io::{Cursor, Read},
|
||||
net::{IpAddr, SocketAddr, ToSocketAddrs},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use ::image::{ImageFormat, ImageReader};
|
||||
use anyhow::{Context, Result as AnyResult, bail};
|
||||
use napi::{
|
||||
Env, Error, Result, Status, Task,
|
||||
bindgen_prelude::{AsyncTask, Buffer},
|
||||
};
|
||||
use napi_derive::napi;
|
||||
use reqwest::{
|
||||
Method,
|
||||
blocking::{Client, Response},
|
||||
header::{HeaderMap, HeaderName, HeaderValue, LOCATION},
|
||||
};
|
||||
use url::Url;
|
||||
|
||||
const DEFAULT_TIMEOUT_MS: u32 = 10_000;
|
||||
const DEFAULT_MAX_REDIRECTS: u32 = 3;
|
||||
const DEFAULT_MAX_BYTES: u32 = 10 * 1024 * 1024;
|
||||
const MAX_IMAGE_DIMENSION: u32 = 16_384;
|
||||
const MAX_IMAGE_PIXELS: u64 = 40_000_000;
|
||||
|
||||
#[napi(string_enum = "snake_case")]
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum SafeFetchMethod {
|
||||
Get,
|
||||
Head,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SafeFetchRequest {
|
||||
pub url: String,
|
||||
pub method: Option<SafeFetchMethod>,
|
||||
pub headers: Option<HashMap<String, String>>,
|
||||
pub timeout_ms: Option<u32>,
|
||||
pub max_redirects: Option<u32>,
|
||||
pub max_bytes: Option<u32>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AssertSafeUrlRequest {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct SafeFetchResponse {
|
||||
pub status: u16,
|
||||
pub final_url: String,
|
||||
pub headers: HashMap<String, String>,
|
||||
pub body: Buffer,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ImageInspectionOptions {
|
||||
pub max_width: Option<u32>,
|
||||
pub max_height: Option<u32>,
|
||||
pub max_pixels: Option<u32>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct ImageInspection {
|
||||
pub mime_type: String,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RemoteAttachmentFetchRequest {
|
||||
pub url: String,
|
||||
pub timeout_ms: Option<u32>,
|
||||
pub max_bytes: u32,
|
||||
pub allow_private_target_origin: Option<bool>,
|
||||
pub expected_content_type_prefix: Option<String>,
|
||||
pub max_image_width: Option<u32>,
|
||||
pub max_image_height: Option<u32>,
|
||||
pub max_image_pixels: Option<u32>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct RemoteAttachmentFetchResponse {
|
||||
pub final_url: String,
|
||||
pub mime_type: String,
|
||||
pub body: Buffer,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RemoteMimeTypeRequest {
|
||||
pub url: String,
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
pub struct AsyncSafeFetchTask {
|
||||
request: SafeFetchRequest,
|
||||
}
|
||||
|
||||
pub struct AsyncRemoteAttachmentFetchTask {
|
||||
request: RemoteAttachmentFetchRequest,
|
||||
}
|
||||
|
||||
pub struct AsyncRemoteMimeTypeTask {
|
||||
request: RemoteMimeTypeRequest,
|
||||
}
|
||||
|
||||
pub struct SafeFetchOutput {
|
||||
status: u16,
|
||||
final_url: String,
|
||||
headers: HashMap<String, String>,
|
||||
body: Vec<u8>,
|
||||
}
|
||||
|
||||
struct SafeFetchParams {
|
||||
url: String,
|
||||
method: Option<SafeFetchMethod>,
|
||||
headers: Option<HashMap<String, String>>,
|
||||
timeout_ms: Option<u32>,
|
||||
max_redirects: Option<u32>,
|
||||
max_bytes: Option<u32>,
|
||||
allow_private_origins: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
pub struct RemoteAttachmentFetchOutput {
|
||||
final_url: String,
|
||||
mime_type: String,
|
||||
body: Vec<u8>,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl Task for AsyncSafeFetchTask {
|
||||
type Output = SafeFetchOutput;
|
||||
type JsValue = SafeFetchResponse;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
safe_fetch_inner(&self.request).map_err(|error| Error::new(Status::InvalidArg, error.to_string()))
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _: Env, output: Self::Output) -> Result<Self::JsValue> {
|
||||
Ok(SafeFetchResponse {
|
||||
status: output.status,
|
||||
final_url: output.final_url,
|
||||
headers: output.headers,
|
||||
body: output.body.into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn safe_fetch(request: SafeFetchRequest) -> AsyncTask<AsyncSafeFetchTask> {
|
||||
AsyncTask::new(AsyncSafeFetchTask { request })
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn assert_safe_url(request: AssertSafeUrlRequest) -> Result<()> {
|
||||
assert_safe_url_inner(&request).map_err(|error| Error::new(Status::InvalidArg, error.to_string()))
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn inspect_image_for_proxy(input: Buffer, options: Option<ImageInspectionOptions>) -> Result<ImageInspection> {
|
||||
inspect_image_for_proxy_inner(
|
||||
&input,
|
||||
options.unwrap_or(ImageInspectionOptions {
|
||||
max_width: None,
|
||||
max_height: None,
|
||||
max_pixels: None,
|
||||
}),
|
||||
)
|
||||
.map_err(|error| Error::new(Status::InvalidArg, error.to_string()))
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl Task for AsyncRemoteAttachmentFetchTask {
|
||||
type Output = RemoteAttachmentFetchOutput;
|
||||
type JsValue = RemoteAttachmentFetchResponse;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
fetch_remote_attachment_inner(&self.request).map_err(|error| Error::new(Status::InvalidArg, error.to_string()))
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _: Env, output: Self::Output) -> Result<Self::JsValue> {
|
||||
Ok(RemoteAttachmentFetchResponse {
|
||||
final_url: output.final_url,
|
||||
mime_type: output.mime_type,
|
||||
body: output.body.into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn fetch_remote_attachment(request: RemoteAttachmentFetchRequest) -> AsyncTask<AsyncRemoteAttachmentFetchTask> {
|
||||
AsyncTask::new(AsyncRemoteAttachmentFetchTask { request })
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl Task for AsyncRemoteMimeTypeTask {
|
||||
type Output = String;
|
||||
type JsValue = String;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
Ok(infer_remote_mime_type_inner(&self.request))
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _: Env, output: Self::Output) -> Result<Self::JsValue> {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn infer_remote_mime_type(request: RemoteMimeTypeRequest) -> AsyncTask<AsyncRemoteMimeTypeTask> {
|
||||
AsyncTask::new(AsyncRemoteMimeTypeTask { request })
|
||||
}
|
||||
|
||||
fn safe_fetch_inner(request: &SafeFetchRequest) -> AnyResult<SafeFetchOutput> {
|
||||
safe_fetch_params_inner(&SafeFetchParams {
|
||||
url: request.url.clone(),
|
||||
method: request.method,
|
||||
headers: request.headers.clone(),
|
||||
timeout_ms: request.timeout_ms,
|
||||
max_redirects: request.max_redirects,
|
||||
max_bytes: request.max_bytes,
|
||||
allow_private_origins: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn safe_fetch_params_inner(request: &SafeFetchParams) -> AnyResult<SafeFetchOutput> {
|
||||
let timeout = Duration::from_millis(u64::from(request.timeout_ms.unwrap_or(DEFAULT_TIMEOUT_MS)));
|
||||
let max_redirects = request.max_redirects.unwrap_or(DEFAULT_MAX_REDIRECTS);
|
||||
let max_bytes = usize::try_from(request.max_bytes.unwrap_or(DEFAULT_MAX_BYTES)).context("invalid maxBytes")?;
|
||||
let method = request.method.unwrap_or(SafeFetchMethod::Get);
|
||||
let mut current = parse_safe_url(&request.url)?;
|
||||
let headers = build_headers(request.headers.as_ref())?;
|
||||
|
||||
for redirect_count in 0..=max_redirects {
|
||||
let addrs = resolve_safe_socket_addrs(¤t, request.allow_private_origins.as_deref())?;
|
||||
let client = build_pinned_client(¤t, &addrs, timeout)?;
|
||||
let response = send_request(&client, method, current.clone(), headers.clone())?;
|
||||
|
||||
if response.status().is_redirection() {
|
||||
if redirect_count >= max_redirects {
|
||||
bail!("too_many_redirects");
|
||||
}
|
||||
let Some(location) = response.headers().get(LOCATION) else {
|
||||
return response_to_output(response, current, max_bytes, method);
|
||||
};
|
||||
let location = location.to_str().context("invalid redirect location")?;
|
||||
current = parse_safe_url(current.join(location).context("invalid redirect location")?.as_str())?;
|
||||
continue;
|
||||
}
|
||||
|
||||
return response_to_output(response, current, max_bytes, method);
|
||||
}
|
||||
|
||||
bail!("too_many_redirects")
|
||||
}
|
||||
|
||||
fn fetch_remote_attachment_inner(request: &RemoteAttachmentFetchRequest) -> AnyResult<RemoteAttachmentFetchOutput> {
|
||||
let allow_private_origins =
|
||||
private_target_origin_allowlist(&request.url, request.allow_private_target_origin.unwrap_or(false))?;
|
||||
let response = safe_fetch_params_inner(&SafeFetchParams {
|
||||
url: request.url.clone(),
|
||||
method: Some(SafeFetchMethod::Get),
|
||||
headers: None,
|
||||
timeout_ms: request.timeout_ms,
|
||||
max_redirects: Some(DEFAULT_MAX_REDIRECTS),
|
||||
max_bytes: Some(request.max_bytes),
|
||||
allow_private_origins,
|
||||
})?;
|
||||
if !(200..300).contains(&response.status) {
|
||||
bail!("fetch_failed_status: {}", response.status);
|
||||
}
|
||||
let mime_type = normalize_mime_type(response.headers.get("content-type"));
|
||||
if let Some(expected) = request.expected_content_type_prefix.as_deref() {
|
||||
if !mime_type.starts_with(expected) {
|
||||
bail!("content_type_mismatch");
|
||||
}
|
||||
if expected.starts_with("image/") {
|
||||
inspect_image_for_proxy_inner(
|
||||
&response.body,
|
||||
ImageInspectionOptions {
|
||||
max_width: request.max_image_width,
|
||||
max_height: request.max_image_height,
|
||||
max_pixels: request.max_image_pixels,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(RemoteAttachmentFetchOutput {
|
||||
final_url: response.final_url,
|
||||
mime_type,
|
||||
body: response.body,
|
||||
})
|
||||
}
|
||||
|
||||
fn infer_remote_mime_type_inner(request: &RemoteMimeTypeRequest) -> String {
|
||||
let Ok(url) = Url::parse(&request.url) else {
|
||||
return "application/octet-stream".to_string();
|
||||
};
|
||||
if let Some(mime_type) = infer_mime_type_from_extension(&url) {
|
||||
return mime_type.to_string();
|
||||
}
|
||||
let Ok(response) = safe_fetch_params_inner(&SafeFetchParams {
|
||||
url: request.url.clone(),
|
||||
method: Some(SafeFetchMethod::Head),
|
||||
headers: None,
|
||||
timeout_ms: request.timeout_ms,
|
||||
max_redirects: Some(DEFAULT_MAX_REDIRECTS),
|
||||
max_bytes: Some(0),
|
||||
allow_private_origins: None,
|
||||
}) else {
|
||||
return "application/octet-stream".to_string();
|
||||
};
|
||||
normalize_mime_type(response.headers.get("content-type"))
|
||||
}
|
||||
|
||||
fn private_target_origin_allowlist(raw_url: &str, allow_private_target_origin: bool) -> AnyResult<Option<Vec<String>>> {
|
||||
if !allow_private_target_origin {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(vec![parse_safe_url(raw_url)?.origin().ascii_serialization()]))
|
||||
}
|
||||
|
||||
fn normalize_mime_type(value: Option<&String>) -> String {
|
||||
value
|
||||
.and_then(|value| value.split(';').next())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or("application/octet-stream")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn infer_mime_type_from_extension(url: &Url) -> Option<&'static str> {
|
||||
let extension = url.path_segments()?.next_back()?.rsplit_once('.')?.1;
|
||||
match extension.to_ascii_lowercase().as_str() {
|
||||
"pdf" => Some("application/pdf"),
|
||||
"mp3" => Some("audio/mpeg"),
|
||||
"opus" => Some("audio/opus"),
|
||||
"ogg" => Some("audio/ogg"),
|
||||
"aac" => Some("audio/aac"),
|
||||
"m4a" => Some("audio/aac"),
|
||||
"flac" => Some("audio/flac"),
|
||||
"ogv" => Some("video/ogg"),
|
||||
"wav" => Some("audio/wav"),
|
||||
"png" => Some("image/png"),
|
||||
"jpeg" | "jpg" => Some("image/jpeg"),
|
||||
"webp" => Some("image/webp"),
|
||||
"txt" | "md" => Some("text/plain"),
|
||||
"mov" => Some("video/mov"),
|
||||
"mpeg" => Some("video/mpeg"),
|
||||
"mp4" => Some("video/mp4"),
|
||||
"avi" => Some("video/avi"),
|
||||
"wmv" => Some("video/wmv"),
|
||||
"flv" => Some("video/flv"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_safe_url_inner(request: &AssertSafeUrlRequest) -> AnyResult<()> {
|
||||
let url = parse_safe_url(&request.url)?;
|
||||
resolve_safe_socket_addrs(&url, None)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_safe_url(raw: &str) -> AnyResult<Url> {
|
||||
let url = Url::parse(raw).context("invalid_url")?;
|
||||
match url.scheme() {
|
||||
"http" | "https" => {}
|
||||
_ => bail!("disallowed_protocol"),
|
||||
}
|
||||
if !url.username().is_empty() || url.password().is_some() {
|
||||
bail!("url_has_credentials");
|
||||
}
|
||||
if url.host_str().is_none() {
|
||||
bail!("blocked_hostname");
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn resolve_safe_socket_addrs(url: &Url, allow_private_origins: Option<&[String]>) -> AnyResult<Vec<SocketAddr>> {
|
||||
let host = url.host_str().context("blocked_hostname")?;
|
||||
let port = url.port_or_known_default().context("blocked_hostname")?;
|
||||
let origin = url.origin().ascii_serialization();
|
||||
let allow_private = allow_private_origins
|
||||
.map(|origins| origins.iter().any(|allowed| allowed == &origin))
|
||||
.unwrap_or(false);
|
||||
let addrs: Vec<SocketAddr> = (host, port)
|
||||
.to_socket_addrs()
|
||||
.context("unresolvable_hostname")?
|
||||
.collect();
|
||||
if addrs.is_empty() {
|
||||
bail!("unresolvable_hostname");
|
||||
}
|
||||
for addr in &addrs {
|
||||
if is_blocked_ip(addr.ip()) && !allow_private {
|
||||
bail!("blocked_ip");
|
||||
}
|
||||
}
|
||||
Ok(addrs)
|
||||
}
|
||||
|
||||
fn build_pinned_client(url: &Url, addrs: &[SocketAddr], timeout: Duration) -> AnyResult<Client> {
|
||||
let host = url.host_str().context("blocked_hostname")?;
|
||||
Client::builder()
|
||||
.timeout(timeout)
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.resolve_to_addrs(host, addrs)
|
||||
.build()
|
||||
.context("failed to build http client")
|
||||
}
|
||||
|
||||
fn build_headers(headers: Option<&HashMap<String, String>>) -> AnyResult<HeaderMap> {
|
||||
let mut out = HeaderMap::new();
|
||||
let Some(headers) = headers else {
|
||||
return Ok(out);
|
||||
};
|
||||
for (name, value) in headers {
|
||||
let lower = name.to_ascii_lowercase();
|
||||
if !(lower.starts_with("sec-") || lower.starts_with("accept") || lower == "user-agent") {
|
||||
continue;
|
||||
}
|
||||
out.insert(
|
||||
HeaderName::from_bytes(name.as_bytes()).context("invalid header name")?,
|
||||
HeaderValue::from_str(value).context("invalid header value")?,
|
||||
);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn send_request(client: &Client, method: SafeFetchMethod, url: Url, headers: HeaderMap) -> AnyResult<Response> {
|
||||
let method = match method {
|
||||
SafeFetchMethod::Get => Method::GET,
|
||||
SafeFetchMethod::Head => Method::HEAD,
|
||||
};
|
||||
client
|
||||
.request(method, url)
|
||||
.headers(headers)
|
||||
.send()
|
||||
.context("failed to fetch url")
|
||||
}
|
||||
|
||||
fn response_to_output(
|
||||
mut response: Response,
|
||||
url: Url,
|
||||
max_bytes: usize,
|
||||
method: SafeFetchMethod,
|
||||
) -> AnyResult<SafeFetchOutput> {
|
||||
let status = response.status().as_u16();
|
||||
let headers = response_headers(response.headers());
|
||||
if matches!(method, SafeFetchMethod::Head) {
|
||||
return Ok(SafeFetchOutput {
|
||||
status,
|
||||
final_url: url.to_string(),
|
||||
headers,
|
||||
body: Vec::new(),
|
||||
});
|
||||
}
|
||||
let mut body = Vec::new();
|
||||
if let Some(len) = response.content_length()
|
||||
&& len > max_bytes as u64
|
||||
{
|
||||
bail!("response_too_large");
|
||||
}
|
||||
response
|
||||
.by_ref()
|
||||
.take(u64::try_from(max_bytes).unwrap_or(u64::MAX) + 1)
|
||||
.read_to_end(&mut body)
|
||||
.context("failed to read response")?;
|
||||
if body.len() > max_bytes {
|
||||
bail!("response_too_large");
|
||||
}
|
||||
Ok(SafeFetchOutput {
|
||||
status,
|
||||
final_url: url.to_string(),
|
||||
headers,
|
||||
body,
|
||||
})
|
||||
}
|
||||
|
||||
fn response_headers(headers: &HeaderMap) -> HashMap<String, String> {
|
||||
headers
|
||||
.iter()
|
||||
.filter_map(|(name, value)| {
|
||||
value
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(|value| (name.as_str().to_string(), value.to_string()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn is_blocked_ip(ip: IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(ip) => {
|
||||
ip.is_private()
|
||||
|| ip.is_loopback()
|
||||
|| ip.is_link_local()
|
||||
|| ip.is_broadcast()
|
||||
|| ip.is_multicast()
|
||||
|| ip.is_documentation()
|
||||
|| ip.octets()[0] == 0
|
||||
|| ip.octets()[0] >= 224
|
||||
|| ip.octets()[0] == 100 && (64..=127).contains(&ip.octets()[1])
|
||||
|| ip.octets()[0] == 169 && ip.octets()[1] == 254
|
||||
|| ip.octets()[0] == 198 && (18..=19).contains(&ip.octets()[1])
|
||||
|| ip.octets()[0] == 192 && ip.octets()[1] == 0 && ip.octets()[2] == 0
|
||||
}
|
||||
IpAddr::V6(ip) => {
|
||||
if let Some(v4) = ip.to_ipv4_mapped() {
|
||||
return is_blocked_ip(IpAddr::V4(v4));
|
||||
}
|
||||
if let Some(v4) = extract_6to4_ipv4(ip).or_else(|| extract_teredo_client_ipv4(ip)) {
|
||||
return is_blocked_ip(IpAddr::V4(v4));
|
||||
}
|
||||
(ip.segments()[0] & 0xe000 != 0x2000)
|
||||
|| ip.is_loopback()
|
||||
|| ip.is_unspecified()
|
||||
|| ip.is_multicast()
|
||||
|| (ip.segments()[0] & 0xfe00 == 0xfc00)
|
||||
|| (ip.segments()[0] & 0xffc0 == 0xfe80)
|
||||
|| (ip.segments()[0] == 0x2001 && ip.segments()[1] == 0x0db8)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_6to4_ipv4(ip: std::net::Ipv6Addr) -> Option<std::net::Ipv4Addr> {
|
||||
let segments = ip.segments();
|
||||
if segments[0] != 0x2002 {
|
||||
return None;
|
||||
}
|
||||
Some(std::net::Ipv4Addr::new(
|
||||
(segments[1] >> 8) as u8,
|
||||
segments[1] as u8,
|
||||
(segments[2] >> 8) as u8,
|
||||
segments[2] as u8,
|
||||
))
|
||||
}
|
||||
|
||||
fn extract_teredo_client_ipv4(ip: std::net::Ipv6Addr) -> Option<std::net::Ipv4Addr> {
|
||||
let segments = ip.segments();
|
||||
if segments[0] != 0x2001 || segments[1] != 0 {
|
||||
return None;
|
||||
}
|
||||
Some(std::net::Ipv4Addr::new(
|
||||
(!(segments[6] >> 8)) as u8,
|
||||
(!segments[6]) as u8,
|
||||
(!(segments[7] >> 8)) as u8,
|
||||
(!segments[7]) as u8,
|
||||
))
|
||||
}
|
||||
|
||||
fn inspect_image_for_proxy_inner(input: &[u8], options: ImageInspectionOptions) -> AnyResult<ImageInspection> {
|
||||
let inspection = parse_image_header(input).context("failed to decode image")?;
|
||||
validate_image_dimensions(&inspection, options)?;
|
||||
Ok(inspection)
|
||||
}
|
||||
|
||||
fn validate_image_dimensions(image: &ImageInspection, options: ImageInspectionOptions) -> AnyResult<()> {
|
||||
let max_width = options.max_width.unwrap_or(MAX_IMAGE_DIMENSION);
|
||||
let max_height = options.max_height.unwrap_or(MAX_IMAGE_DIMENSION);
|
||||
let max_pixels = u64::from(options.max_pixels.unwrap_or(MAX_IMAGE_PIXELS as u32));
|
||||
if image.width == 0 || image.height == 0 {
|
||||
bail!("failed to decode image");
|
||||
}
|
||||
if image.width > max_width || image.height > max_height {
|
||||
bail!("image dimensions exceed limit");
|
||||
}
|
||||
if u64::from(image.width) * u64::from(image.height) > max_pixels {
|
||||
bail!("image pixel count exceeds limit");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_image_header(input: &[u8]) -> AnyResult<ImageInspection> {
|
||||
let format = ::image::guess_format(input).context("unsupported image format")?;
|
||||
let mime_type = image_mime_type(format).context("unsupported image format")?;
|
||||
let (width, height) = ImageReader::with_format(Cursor::new(input), format)
|
||||
.into_dimensions()
|
||||
.context("failed to decode image")?;
|
||||
Ok(ImageInspection {
|
||||
mime_type: mime_type.to_string(),
|
||||
width,
|
||||
height,
|
||||
})
|
||||
}
|
||||
|
||||
fn image_mime_type(format: ImageFormat) -> Option<&'static str> {
|
||||
match format {
|
||||
ImageFormat::Png => Some("image/png"),
|
||||
ImageFormat::Jpeg => Some("image/jpeg"),
|
||||
ImageFormat::Gif => Some("image/gif"),
|
||||
ImageFormat::WebP => Some("image/webp"),
|
||||
ImageFormat::Bmp => Some("image/bmp"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn blocks_private_ips() {
|
||||
assert!(is_blocked_ip("127.0.0.1".parse().unwrap()));
|
||||
assert!(is_blocked_ip("10.0.0.1".parse().unwrap()));
|
||||
assert!(is_blocked_ip("169.254.169.254".parse().unwrap()));
|
||||
assert!(is_blocked_ip("::1".parse().unwrap()));
|
||||
assert!(is_blocked_ip("::ffff:127.0.0.1".parse().unwrap()));
|
||||
assert!(is_blocked_ip("2002:7f00:0001::1".parse().unwrap()));
|
||||
assert!(is_blocked_ip("2002:c0a8:0001::1".parse().unwrap()));
|
||||
assert!(is_blocked_ip(
|
||||
"2001:0000:4136:e378:8000:63bf:807f:fffe".parse().unwrap()
|
||||
));
|
||||
assert!(!is_blocked_ip("8.8.8.8".parse().unwrap()));
|
||||
assert!(!is_blocked_ip("2002:0808:0808::1".parse().unwrap()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inspects_png_dimensions_without_decode() {
|
||||
let png = base64_simd::STANDARD
|
||||
.decode_to_vec(b"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jfJ8AAAAASUVORK5CYII=")
|
||||
.unwrap();
|
||||
let inspected = inspect_image_for_proxy_inner(
|
||||
&png,
|
||||
ImageInspectionOptions {
|
||||
max_width: None,
|
||||
max_height: None,
|
||||
max_pixels: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(inspected.mime_type, "image/png");
|
||||
assert_eq!(inspected.width, 1);
|
||||
assert_eq!(inspected.height, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_oversized_dimensions() {
|
||||
let png = [
|
||||
b"\x89PNG\r\n\x1a\n".as_slice(),
|
||||
&[0, 0, 0, 13],
|
||||
b"IHDR".as_slice(),
|
||||
&100_000u32.to_be_bytes(),
|
||||
&100_000u32.to_be_bytes(),
|
||||
&[8, 6, 0, 0, 0],
|
||||
]
|
||||
.concat();
|
||||
assert!(
|
||||
inspect_image_for_proxy_inner(
|
||||
&png,
|
||||
ImageInspectionOptions {
|
||||
max_width: None,
|
||||
max_height: None,
|
||||
max_pixels: None,
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
import test from 'ava';
|
||||
|
||||
import { readResponseBufferWithLimit } from '../../base';
|
||||
|
||||
test('readResponseBufferWithLimit rejects timed out web streams without crashing', async t => {
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array([1, 2, 3]));
|
||||
queueMicrotask(() => {
|
||||
controller.error(
|
||||
new DOMException(
|
||||
'The operation was aborted due to timeout',
|
||||
'TimeoutError'
|
||||
)
|
||||
);
|
||||
});
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
const error = await t.throwsAsync(
|
||||
readResponseBufferWithLimit(response, 1024)
|
||||
);
|
||||
|
||||
t.is(error?.name, 'TimeoutError');
|
||||
});
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
JobQueue,
|
||||
OneMB,
|
||||
} from '../../base';
|
||||
import { ThrottlerStorage } from '../../base/throttler';
|
||||
import { SocketIoAdapter } from '../../base/websocket';
|
||||
import { AuthGuard, AuthService } from '../../core/auth';
|
||||
import { Mailer } from '../../core/mail';
|
||||
@@ -163,6 +164,10 @@ export class TestingApp extends NestApplication {
|
||||
return await this.create(MockUser, overrides);
|
||||
}
|
||||
|
||||
resetRateLimit() {
|
||||
this.get(ThrottlerStorage, { strict: false }).storage.clear();
|
||||
}
|
||||
|
||||
async signup(overrides?: Partial<MockUserInput>) {
|
||||
const user = await this.create(MockUser, overrides);
|
||||
await this.login(user);
|
||||
@@ -170,6 +175,7 @@ export class TestingApp extends NestApplication {
|
||||
}
|
||||
|
||||
async login(user: MockedUser) {
|
||||
this.resetRateLimit();
|
||||
return await this.POST('/api/auth/sign-in').send({
|
||||
email: user.email,
|
||||
password: user.password,
|
||||
@@ -195,6 +201,7 @@ export class TestingApp extends NestApplication {
|
||||
}
|
||||
|
||||
async logout(userId?: string) {
|
||||
this.resetRateLimit();
|
||||
const res = await this.POST(
|
||||
'/api/auth/sign-out' + (userId ? `?user_id=${userId}` : '')
|
||||
).expect(200);
|
||||
|
||||
@@ -92,7 +92,7 @@ test.before(async t => {
|
||||
throttle: {
|
||||
throttlers: {
|
||||
default: {
|
||||
ttl: 60,
|
||||
ttl: 60_000,
|
||||
limit: 120,
|
||||
},
|
||||
},
|
||||
@@ -109,6 +109,7 @@ test.before(async t => {
|
||||
|
||||
test.beforeEach(async t => {
|
||||
const { app } = t.context;
|
||||
t.context.storage.storage.clear();
|
||||
await app.initTestingDB();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,49 +1,79 @@
|
||||
import { LookupAddress } from 'node:dns';
|
||||
|
||||
import serverNativeModule from '@affine/server-native';
|
||||
import type { ExecutionContext, TestFn } from 'ava';
|
||||
import ava from 'ava';
|
||||
import Sinon from 'sinon';
|
||||
import type { Response } from 'supertest';
|
||||
|
||||
import {
|
||||
__resetDnsLookupForTests,
|
||||
__setDnsLookupForTests,
|
||||
type DnsLookup,
|
||||
} from '../base/utils/ssrf';
|
||||
import { createTestingApp, TestingApp } from './utils';
|
||||
import type { TestingApp } from './utils';
|
||||
|
||||
type TestContext = {
|
||||
app: TestingApp;
|
||||
};
|
||||
const test = ava as TestFn<TestContext>;
|
||||
|
||||
const LookupAddressStub = (async (_hostname, options) => {
|
||||
const result = [{ address: '76.76.21.21', family: 4 }] as LookupAddress[];
|
||||
const isOptions = options && typeof options === 'object';
|
||||
if (isOptions && 'all' in options && options.all) {
|
||||
return result;
|
||||
let safeFetchStub: Sinon.SinonStub | undefined;
|
||||
let safeFetchHandler:
|
||||
| ((request: { url: string; method?: 'get' | 'head' }) => {
|
||||
status?: number;
|
||||
finalUrl?: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: Buffer | string;
|
||||
})
|
||||
| undefined;
|
||||
|
||||
const stubSafeFetch = (
|
||||
handler: (request: { url: string; method?: 'get' | 'head' }) => {
|
||||
status?: number;
|
||||
finalUrl?: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: Buffer | string;
|
||||
}
|
||||
return result[0];
|
||||
}) as DnsLookup;
|
||||
) => {
|
||||
safeFetchHandler = handler;
|
||||
return {
|
||||
restore() {
|
||||
safeFetchHandler = undefined;
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
test.before(async t => {
|
||||
// @ts-expect-error test
|
||||
env.DEPLOYMENT_TYPE = 'selfhosted';
|
||||
|
||||
// Avoid relying on real DNS during tests. SSRF protection uses dns.lookup().
|
||||
__setDnsLookupForTests(LookupAddressStub);
|
||||
safeFetchStub = Sinon.stub(serverNativeModule, 'safeFetch').callsFake(
|
||||
async request => {
|
||||
if (!safeFetchHandler) {
|
||||
throw new Error('Unexpected safeFetch call');
|
||||
}
|
||||
const nativeRequest = request as {
|
||||
url: string;
|
||||
method?: 'get' | 'head';
|
||||
};
|
||||
const response = safeFetchHandler(nativeRequest);
|
||||
return {
|
||||
status: response.status ?? 200,
|
||||
finalUrl: response.finalUrl ?? nativeRequest.url,
|
||||
headers: response.headers ?? {},
|
||||
body: Buffer.isBuffer(response.body)
|
||||
? response.body
|
||||
: Buffer.from(response.body ?? ''),
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
const { createTestingApp } = await import('./utils');
|
||||
const app = await createTestingApp();
|
||||
|
||||
t.context.app = app;
|
||||
});
|
||||
|
||||
test.afterEach.always(() => {
|
||||
Sinon.restore();
|
||||
safeFetchHandler = undefined;
|
||||
});
|
||||
|
||||
test.after.always(async t => {
|
||||
__resetDnsLookupForTests();
|
||||
safeFetchStub?.restore();
|
||||
await t.context.app.close();
|
||||
});
|
||||
|
||||
@@ -128,15 +158,13 @@ test('should proxy image', async t => {
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jfJ8AAAAASUVORK5CYII=',
|
||||
'base64'
|
||||
);
|
||||
const fakeResponse = new Response(fakeBuffer, {
|
||||
status: 200,
|
||||
const fetchSpy = stubSafeFetch(() => ({
|
||||
body: fakeBuffer,
|
||||
headers: {
|
||||
'content-type': 'image/png',
|
||||
'content-disposition': 'inline',
|
||||
},
|
||||
});
|
||||
|
||||
const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeResponse);
|
||||
}));
|
||||
try {
|
||||
await assertAndSnapshot(
|
||||
`/api/worker/image-proxy?url=${imageUrl}`,
|
||||
@@ -146,6 +174,42 @@ test('should proxy image', async t => {
|
||||
fetchSpy.restore();
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const invalidImageUrl = `http://example.com/not-image-${Date.now()}.png`;
|
||||
const invalidFetchSpy = stubSafeFetch(() => ({
|
||||
body: 'not an image',
|
||||
headers: { 'content-type': 'image/png' },
|
||||
}));
|
||||
try {
|
||||
await t.context.app
|
||||
.GET(`/api/worker/image-proxy?url=${invalidImageUrl}`)
|
||||
.set('Origin', 'http://localhost:3010')
|
||||
.send()
|
||||
.expect(400);
|
||||
} finally {
|
||||
invalidFetchSpy.restore();
|
||||
}
|
||||
|
||||
const validImageUrl = `http://example.com/valid-image-${Date.now()}.png`;
|
||||
const fakeBuffer = Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jfJ8AAAAASUVORK5CYII=',
|
||||
'base64'
|
||||
);
|
||||
const validFetchSpy = stubSafeFetch(() => ({
|
||||
body: fakeBuffer,
|
||||
headers: { 'content-type': 'image/png' },
|
||||
}));
|
||||
try {
|
||||
await t.context.app
|
||||
.GET(`/api/worker/image-proxy?url=${validImageUrl}`)
|
||||
.set('Origin', 'http://localhost:3010')
|
||||
.send()
|
||||
.expect(200);
|
||||
} finally {
|
||||
validFetchSpy.restore();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('should preview link', async t => {
|
||||
@@ -190,7 +254,8 @@ test('should preview link', async t => {
|
||||
);
|
||||
|
||||
{
|
||||
const fakeHTML = new Response(`
|
||||
const pageUrl = `http://external.com/page-${Date.now()}`;
|
||||
const fakeHTML = `
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:title" content="Test Title" />
|
||||
@@ -201,13 +266,18 @@ test('should preview link', async t => {
|
||||
<title>Fallback Title</title>
|
||||
</body>
|
||||
</html>
|
||||
`);
|
||||
`;
|
||||
|
||||
Object.defineProperty(fakeHTML, 'url', {
|
||||
value: 'http://example.com/page',
|
||||
const fetchSpy = stubSafeFetch(request => {
|
||||
if (request.url.includes('/favicon.ico')) {
|
||||
return { status: 204, finalUrl: request.url };
|
||||
}
|
||||
return {
|
||||
body: fakeHTML,
|
||||
finalUrl: 'http://example.com/page',
|
||||
headers: { 'content-type': 'text/html;charset=UTF-8' },
|
||||
};
|
||||
});
|
||||
|
||||
const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeHTML);
|
||||
try {
|
||||
await assertAndSnapshot(
|
||||
'/api/worker/link-preview',
|
||||
@@ -215,7 +285,7 @@ test('should preview link', async t => {
|
||||
{
|
||||
status: 200,
|
||||
method: 'POST',
|
||||
body: { url: 'http://external.com/page' },
|
||||
body: { url: pageUrl },
|
||||
}
|
||||
);
|
||||
} finally {
|
||||
@@ -244,6 +314,7 @@ test('should preview link', async t => {
|
||||
];
|
||||
|
||||
for (const { content, charset } of encoded) {
|
||||
const pageUrl = `http://example.com/${charset}-${Date.now()}`;
|
||||
const before = Buffer.from(`<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=${charset}" />
|
||||
@@ -253,13 +324,18 @@ test('should preview link', async t => {
|
||||
</head>
|
||||
</html>
|
||||
`);
|
||||
const fakeHTML = new Response(Buffer.concat([before, encoded, after]));
|
||||
const fakeHTML = Buffer.concat([before, encoded, after]);
|
||||
|
||||
Object.defineProperty(fakeHTML, 'url', {
|
||||
value: `http://example.com/${charset}`,
|
||||
const fetchSpy = stubSafeFetch(request => {
|
||||
if (request.url.includes('/favicon.ico')) {
|
||||
return { status: 204, finalUrl: request.url };
|
||||
}
|
||||
return {
|
||||
body: fakeHTML,
|
||||
finalUrl: `http://example.com/${charset}`,
|
||||
headers: { 'content-type': `text/html;charset=${charset}` },
|
||||
};
|
||||
});
|
||||
|
||||
const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeHTML);
|
||||
try {
|
||||
await assertAndSnapshot(
|
||||
'/api/worker/link-preview',
|
||||
@@ -267,7 +343,7 @@ test('should preview link', async t => {
|
||||
{
|
||||
status: 200,
|
||||
method: 'POST',
|
||||
body: { url: `http://example.com/${charset}` },
|
||||
body: { url: pageUrl },
|
||||
}
|
||||
);
|
||||
} finally {
|
||||
|
||||
@@ -24,14 +24,14 @@ defineModuleConfig('throttle', {
|
||||
'throttlers.default': {
|
||||
desc: 'The config for the default throttler.',
|
||||
default: {
|
||||
ttl: 60,
|
||||
ttl: 60_000,
|
||||
limit: 120,
|
||||
},
|
||||
},
|
||||
'throttlers.strict': {
|
||||
desc: 'The config for the strict throttler.',
|
||||
default: {
|
||||
ttl: 60,
|
||||
ttl: 60_000,
|
||||
limit: 20,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -13,7 +13,7 @@ export const THROTTLER_PROTECTED = 'affine_throttler:protected';
|
||||
* it will never be rate limited.
|
||||
*
|
||||
* - default: 120 calls within 60 seconds
|
||||
* - strict: 10 calls within 60 seconds
|
||||
* - strict: 20 calls within 60 seconds
|
||||
* - authenticated: no rate limit for authenticated users, apply [default] throttler for unauthenticated users
|
||||
*
|
||||
* @example
|
||||
@@ -23,7 +23,7 @@ export const THROTTLER_PROTECTED = 'affine_throttler:protected';
|
||||
*
|
||||
* // the config call be override by the second parameter,
|
||||
* // and the call count will be calculated separately
|
||||
* \@Throttle('default', { limit: 10, ttl: 10 })
|
||||
* \@Throttle('default', { limit: 10, ttl: 10_000 })
|
||||
*
|
||||
*/
|
||||
export function Throttle(
|
||||
|
||||
@@ -1,23 +1,9 @@
|
||||
import * as dns from 'node:dns/promises';
|
||||
import { BlockList, isIP } from 'node:net';
|
||||
|
||||
import {
|
||||
assertSafeUrl as assertSafeUrlFromNative,
|
||||
safeFetch as safeFetchFromNative,
|
||||
type SafeFetchRequest,
|
||||
} from '../../native';
|
||||
import { ResponseTooLargeError, SsrfBlockedError } from '../error/errors.gen';
|
||||
import { OneMinute } from './unit';
|
||||
|
||||
const DEFAULT_ALLOWED_PROTOCOLS = new Set(['http:', 'https:']);
|
||||
const BLOCKED_IPS = new BlockList();
|
||||
const ALLOWED_IPV6 = new BlockList();
|
||||
|
||||
export type DnsLookup = typeof dns.lookup;
|
||||
let dnsLookup: DnsLookup = dns.lookup;
|
||||
|
||||
export function __setDnsLookupForTests(lookup: DnsLookup) {
|
||||
dnsLookup = lookup;
|
||||
}
|
||||
|
||||
export function __resetDnsLookupForTests() {
|
||||
dnsLookup = dns.lookup;
|
||||
}
|
||||
|
||||
export type SSRFBlockReason =
|
||||
| 'invalid_url'
|
||||
@@ -28,187 +14,55 @@ export type SSRFBlockReason =
|
||||
| 'blocked_ip'
|
||||
| 'too_many_redirects';
|
||||
|
||||
type SsrfErrorContext = { url?: string; hostname?: string; address?: string };
|
||||
const SSRF_REASONS = new Set<string>([
|
||||
'invalid_url',
|
||||
'disallowed_protocol',
|
||||
'url_has_credentials',
|
||||
'blocked_hostname',
|
||||
'unresolvable_hostname',
|
||||
'blocked_ip',
|
||||
'too_many_redirects',
|
||||
]);
|
||||
|
||||
function createSsrfBlockedError(
|
||||
reason: SSRFBlockReason,
|
||||
context?: SsrfErrorContext
|
||||
) {
|
||||
const err = new SsrfBlockedError({ reason });
|
||||
// For logging/debugging only (not part of UserFriendlyError JSON).
|
||||
(err as any).context = context;
|
||||
return err;
|
||||
function createSsrfBlockedError(reason: SSRFBlockReason) {
|
||||
return new SsrfBlockedError({ reason });
|
||||
}
|
||||
|
||||
export interface SSRFProtectionOptions {
|
||||
allowedProtocols?: ReadonlySet<string>;
|
||||
/**
|
||||
* Allow fetching private/reserved IPs when URL.origin is allowlisted.
|
||||
* Defaults to an empty allowlist (i.e. private IPs are blocked).
|
||||
*/
|
||||
allowPrivateOrigins?: ReadonlySet<string>;
|
||||
}
|
||||
|
||||
function stripZoneId(address: string) {
|
||||
const idx = address.indexOf('%');
|
||||
return idx === -1 ? address : address.slice(0, idx);
|
||||
}
|
||||
|
||||
// IPv4: RFC1918 + loopback + link-local + CGNAT + special/reserved
|
||||
for (const [network, prefix] of [
|
||||
['0.0.0.0', 8],
|
||||
['10.0.0.0', 8],
|
||||
['127.0.0.0', 8],
|
||||
['169.254.0.0', 16],
|
||||
['172.16.0.0', 12],
|
||||
['192.168.0.0', 16],
|
||||
['100.64.0.0', 10], // CGNAT
|
||||
['192.0.0.0', 24],
|
||||
['192.0.2.0', 24], // TEST-NET-1
|
||||
['198.51.100.0', 24], // TEST-NET-2
|
||||
['203.0.113.0', 24], // TEST-NET-3
|
||||
['198.18.0.0', 15], // benchmark
|
||||
['192.88.99.0', 24], // 6to4 relay
|
||||
['224.0.0.0', 4], // multicast
|
||||
['240.0.0.0', 4], // reserved (includes broadcast)
|
||||
] as const) {
|
||||
BLOCKED_IPS.addSubnet(network, prefix, 'ipv4');
|
||||
}
|
||||
|
||||
// IPv6: block loopback/unspecified/link-local/ULA/multicast/doc; allow only global unicast.
|
||||
BLOCKED_IPS.addAddress('::', 'ipv6');
|
||||
BLOCKED_IPS.addAddress('::1', 'ipv6');
|
||||
BLOCKED_IPS.addSubnet('ff00::', 8, 'ipv6'); // multicast
|
||||
BLOCKED_IPS.addSubnet('fc00::', 7, 'ipv6'); // unique local
|
||||
BLOCKED_IPS.addSubnet('fe80::', 10, 'ipv6'); // link-local
|
||||
BLOCKED_IPS.addSubnet('2001:db8::', 32, 'ipv6'); // documentation
|
||||
ALLOWED_IPV6.addSubnet('2000::', 3, 'ipv6'); // global unicast
|
||||
|
||||
function extractEmbeddedIPv4FromIPv6(address: string): string | null {
|
||||
if (!address.includes('.')) {
|
||||
return null;
|
||||
function mapNativeFetchError(error: unknown, limitBytes?: number) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const reason = [...SSRF_REASONS].find(reason => message.includes(reason));
|
||||
if (reason) {
|
||||
return createSsrfBlockedError(reason as SSRFBlockReason);
|
||||
}
|
||||
const idx = address.lastIndexOf(':');
|
||||
if (idx === -1) {
|
||||
return null;
|
||||
if (message.includes('response_too_large')) {
|
||||
return new ResponseTooLargeError({
|
||||
limitBytes: limitBytes ?? 0,
|
||||
receivedBytes: limitBytes ? limitBytes + 1 : 0,
|
||||
});
|
||||
}
|
||||
const tail = address.slice(idx + 1);
|
||||
return isIP(tail) === 4 ? tail : null;
|
||||
return error;
|
||||
}
|
||||
|
||||
function isBlockedIpAddress(address: string): boolean {
|
||||
const ip = stripZoneId(address);
|
||||
const family = isIP(ip);
|
||||
if (family === 4) {
|
||||
return BLOCKED_IPS.check(ip, 'ipv4');
|
||||
}
|
||||
if (family === 6) {
|
||||
const embeddedV4 = extractEmbeddedIPv4FromIPv6(ip);
|
||||
if (embeddedV4) {
|
||||
return isBlockedIpAddress(embeddedV4);
|
||||
}
|
||||
if (!ALLOWED_IPV6.check(ip, 'ipv6')) {
|
||||
return true;
|
||||
}
|
||||
return BLOCKED_IPS.check(ip, 'ipv6');
|
||||
}
|
||||
return true;
|
||||
export interface SafeFetchOptions {
|
||||
timeoutMs?: number;
|
||||
maxRedirects?: number;
|
||||
maxBytes?: number;
|
||||
}
|
||||
|
||||
async function resolveHostAddresses(hostname: string): Promise<string[]> {
|
||||
// Normalize common localhost aliases without DNS.
|
||||
const lowered = hostname.toLowerCase();
|
||||
if (lowered === 'localhost' || lowered.endsWith('.localhost')) {
|
||||
return ['127.0.0.1', '::1'];
|
||||
}
|
||||
|
||||
const results = await dnsLookup(hostname, {
|
||||
all: true,
|
||||
verbatim: true,
|
||||
});
|
||||
return results.map(r => r.address);
|
||||
}
|
||||
|
||||
export async function assertSsrFSafeUrl(
|
||||
rawUrl: string | URL,
|
||||
options: SSRFProtectionOptions = {}
|
||||
): Promise<URL> {
|
||||
const allowedProtocols =
|
||||
options.allowedProtocols ?? DEFAULT_ALLOWED_PROTOCOLS;
|
||||
|
||||
export async function assertSsrFSafeUrl(rawUrl: string | URL): Promise<URL> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = rawUrl instanceof URL ? rawUrl : new URL(rawUrl);
|
||||
} catch {
|
||||
throw createSsrfBlockedError('invalid_url', {
|
||||
url: typeof rawUrl === 'string' ? rawUrl : undefined,
|
||||
});
|
||||
throw createSsrfBlockedError('invalid_url');
|
||||
}
|
||||
|
||||
if (!allowedProtocols.has(url.protocol)) {
|
||||
throw createSsrfBlockedError('disallowed_protocol', {
|
||||
url: url.toString(),
|
||||
});
|
||||
}
|
||||
|
||||
if (url.username || url.password) {
|
||||
throw createSsrfBlockedError('url_has_credentials', {
|
||||
url: url.toString(),
|
||||
});
|
||||
}
|
||||
|
||||
const hostname = url.hostname;
|
||||
if (!hostname) {
|
||||
throw createSsrfBlockedError('blocked_hostname', { url: url.toString() });
|
||||
}
|
||||
|
||||
const allowPrivate =
|
||||
options.allowPrivateOrigins && options.allowPrivateOrigins.has(url.origin);
|
||||
|
||||
// IP literal
|
||||
if (isIP(hostname)) {
|
||||
if (isBlockedIpAddress(hostname) && !allowPrivate) {
|
||||
throw createSsrfBlockedError('blocked_ip', {
|
||||
url: url.toString(),
|
||||
address: hostname,
|
||||
});
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
let addresses: string[];
|
||||
try {
|
||||
addresses = await resolveHostAddresses(hostname);
|
||||
} catch {
|
||||
throw createSsrfBlockedError('unresolvable_hostname', {
|
||||
url: url.toString(),
|
||||
hostname,
|
||||
});
|
||||
assertSafeUrlFromNative({ url: url.toString() });
|
||||
return url;
|
||||
} catch (error) {
|
||||
throw mapNativeFetchError(error);
|
||||
}
|
||||
|
||||
if (addresses.length === 0) {
|
||||
throw createSsrfBlockedError('unresolvable_hostname', {
|
||||
url: url.toString(),
|
||||
hostname,
|
||||
});
|
||||
}
|
||||
|
||||
for (const address of addresses) {
|
||||
if (isBlockedIpAddress(address) && !allowPrivate) {
|
||||
throw createSsrfBlockedError('blocked_ip', {
|
||||
url: url.toString(),
|
||||
hostname,
|
||||
address,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
export interface SafeFetchOptions extends SSRFProtectionOptions {
|
||||
timeoutMs?: number;
|
||||
maxRedirects?: number;
|
||||
}
|
||||
|
||||
export async function safeFetch(
|
||||
@@ -216,153 +70,51 @@ export async function safeFetch(
|
||||
init: RequestInit = {},
|
||||
options: SafeFetchOptions = {}
|
||||
): Promise<Response> {
|
||||
const timeoutMs = options.timeoutMs ?? 10_000;
|
||||
const maxRedirects = options.maxRedirects ?? 3;
|
||||
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
const signal = init.signal
|
||||
? AbortSignal.any([init.signal, timeoutSignal])
|
||||
: timeoutSignal;
|
||||
|
||||
let current = await assertSsrFSafeUrl(rawUrl, options);
|
||||
let redirects = 0;
|
||||
|
||||
// Always handle redirects manually (SSRF-safe on each hop).
|
||||
let requestInit: RequestInit = {
|
||||
...init,
|
||||
redirect: 'manual',
|
||||
signal,
|
||||
};
|
||||
|
||||
while (true) {
|
||||
const response = await fetch(current, requestInit);
|
||||
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
const location = response.headers.get('location');
|
||||
if (!location) {
|
||||
return response;
|
||||
}
|
||||
|
||||
// Drain/cancel body before following redirect to avoid leaking resources.
|
||||
try {
|
||||
await response.body?.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
if (redirects >= maxRedirects) {
|
||||
throw createSsrfBlockedError('too_many_redirects', {
|
||||
url: current.toString(),
|
||||
});
|
||||
}
|
||||
|
||||
const next = new URL(location, current);
|
||||
current = await assertSsrFSafeUrl(next, options);
|
||||
redirects += 1;
|
||||
|
||||
// 303 forces GET semantics
|
||||
if (
|
||||
response.status === 303 &&
|
||||
requestInit.method &&
|
||||
requestInit.method !== 'GET'
|
||||
) {
|
||||
requestInit = { ...requestInit, method: 'GET', body: undefined };
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
return response;
|
||||
const url = rawUrl.toString();
|
||||
const method = String(init.method ?? 'GET').toUpperCase();
|
||||
if (method !== 'GET' && method !== 'HEAD') {
|
||||
throw new Error(`Unsupported safeFetch method: ${method}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function readResponseBufferWithLimit(
|
||||
response: Response,
|
||||
limitBytes: number
|
||||
): Promise<Buffer> {
|
||||
const rawLen = response.headers.get('content-length');
|
||||
if (rawLen) {
|
||||
const len = Number.parseInt(rawLen, 10);
|
||||
if (Number.isFinite(len) && len > limitBytes) {
|
||||
try {
|
||||
await response.body?.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
throw new ResponseTooLargeError({ limitBytes, receivedBytes: len });
|
||||
}
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
return Buffer.alloc(0);
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const chunk = value ?? new Uint8Array();
|
||||
total += chunk.byteLength;
|
||||
if (total > limitBytes) {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
throw new ResponseTooLargeError({ limitBytes, receivedBytes: total });
|
||||
}
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const response = await safeFetchFromNative({
|
||||
url,
|
||||
method: (method === 'HEAD' ? 'head' : 'get') as NonNullable<
|
||||
SafeFetchRequest['method']
|
||||
>,
|
||||
headers: normalizeHeaders(init.headers),
|
||||
timeoutMs: options.timeoutMs,
|
||||
maxRedirects: options.maxRedirects,
|
||||
maxBytes: options.maxBytes,
|
||||
});
|
||||
const body =
|
||||
method === 'HEAD' || [204, 205, 304].includes(response.status)
|
||||
? null
|
||||
: response.body;
|
||||
const webResponse = new Response(body, {
|
||||
status: response.status,
|
||||
headers: response.headers,
|
||||
});
|
||||
Object.defineProperty(webResponse, 'url', {
|
||||
value: response.finalUrl,
|
||||
});
|
||||
return webResponse;
|
||||
} catch (error) {
|
||||
try {
|
||||
await reader.cancel(error);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
reader.releaseLock();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
throw mapNativeFetchError(error, options.maxBytes);
|
||||
}
|
||||
|
||||
return Buffer.concat(
|
||||
chunks.map(chunk => Buffer.from(chunk)),
|
||||
total
|
||||
);
|
||||
}
|
||||
|
||||
type FetchBufferResult = { buffer: Buffer; type: string };
|
||||
const ATTACH_GET_PARAMS = { timeoutMs: OneMinute / 6, maxRedirects: 3 };
|
||||
|
||||
export async function fetchBuffer(
|
||||
url: string,
|
||||
limit: number,
|
||||
contentType?: string
|
||||
): Promise<FetchBufferResult> {
|
||||
const resp = url.startsWith('data:')
|
||||
? await fetch(url)
|
||||
: await safeFetch(url, { method: 'GET' }, ATTACH_GET_PARAMS);
|
||||
|
||||
if (!resp.ok) {
|
||||
throw new Error(
|
||||
`Failed to fetch attachment: ${resp.status} ${resp.statusText}`
|
||||
);
|
||||
function normalizeHeaders(headers: RequestInit['headers'] | undefined) {
|
||||
if (!headers) return undefined;
|
||||
if (headers instanceof Headers) {
|
||||
return Object.fromEntries(headers.entries());
|
||||
}
|
||||
const type = resp.headers.get('content-type') || 'application/octet-stream';
|
||||
if (contentType && !type.startsWith(contentType)) {
|
||||
throw new Error(
|
||||
`Attachment content-type mismatch: expected ${contentType} but got ${type}`
|
||||
);
|
||||
if (Array.isArray(headers)) {
|
||||
return Object.fromEntries(headers);
|
||||
}
|
||||
const buffer = await readResponseBufferWithLimit(resp, limit);
|
||||
return { buffer, type: type };
|
||||
return Object.fromEntries(
|
||||
Object.entries(headers).map(([key, value]) => [key, String(value)])
|
||||
);
|
||||
}
|
||||
|
||||
export function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import serverNativeModule, {
|
||||
type ActionEvent as NativeActionEventContract,
|
||||
type ActionRuntimeInput as NativeActionRuntimeInputContract,
|
||||
type AssertSafeUrlRequest,
|
||||
type BuiltInPromptRenderContract,
|
||||
type BuiltInPromptSessionContract,
|
||||
type BuiltInPromptSpec,
|
||||
@@ -8,6 +9,8 @@ import serverNativeModule, {
|
||||
type CanonicalStructuredRequestContract,
|
||||
type CapabilityAttachmentContract,
|
||||
type CapabilityModelCapability,
|
||||
type ImageInspection,
|
||||
type ImageInspectionOptions,
|
||||
type LlmCoreMessage,
|
||||
type LlmEmbeddingRequestContract,
|
||||
type LlmImageRequestContract,
|
||||
@@ -27,16 +30,29 @@ import serverNativeModule, {
|
||||
type PromptStructuredResponseContract,
|
||||
type PromptTokenCountContract,
|
||||
type PromptTokenCountResult,
|
||||
type RemoteAttachmentFetchRequest,
|
||||
type RemoteAttachmentFetchResponse,
|
||||
type RemoteMimeTypeRequest,
|
||||
type RequestedModelMatchResponse,
|
||||
type SafeFetchRequest,
|
||||
type SafeFetchResponse,
|
||||
type Tokenizer,
|
||||
} from '@affine/server-native';
|
||||
|
||||
export type {
|
||||
AssertSafeUrlRequest,
|
||||
CapabilityAttachmentContract,
|
||||
CapabilityModelCapability,
|
||||
ImageInspection,
|
||||
ImageInspectionOptions,
|
||||
ModelConditionsContract,
|
||||
PromptMessageContract,
|
||||
PromptStructuredResponseContract,
|
||||
RemoteAttachmentFetchRequest,
|
||||
RemoteAttachmentFetchResponse,
|
||||
RemoteMimeTypeRequest,
|
||||
SafeFetchRequest,
|
||||
SafeFetchResponse,
|
||||
};
|
||||
|
||||
export type ActionEventType =
|
||||
@@ -118,6 +134,11 @@ export function getTokenEncoder(model?: string | null): Tokenizer | null {
|
||||
}
|
||||
|
||||
export const getMime = serverNativeModule.getMime;
|
||||
export const inspectImageForProxy = serverNativeModule.inspectImageForProxy;
|
||||
export const fetchRemoteAttachment = serverNativeModule.fetchRemoteAttachment;
|
||||
export const inferRemoteMimeType = serverNativeModule.inferRemoteMimeType;
|
||||
export const assertSafeUrl = serverNativeModule.assertSafeUrl;
|
||||
export const safeFetch = serverNativeModule.safeFetch;
|
||||
export const parseDoc = serverNativeModule.parseDoc;
|
||||
export const htmlSanitize = serverNativeModule.htmlSanitize;
|
||||
export const processImage = serverNativeModule.processImage;
|
||||
|
||||
@@ -35,8 +35,6 @@ const XML_PARSER = new XMLParser({
|
||||
|
||||
const DEFAULT_REQUEST_TIMEOUT_MS = 10_000;
|
||||
const DEFAULT_MAX_REDIRECTS = 5;
|
||||
const CALDAV_ALLOWED_PROTOCOLS = new Set(['https:']);
|
||||
const CALDAV_ALLOWED_PROTOCOLS_INSECURE = new Set(['http:', 'https:']);
|
||||
|
||||
type CalDAVCredentials = {
|
||||
username: string;
|
||||
@@ -607,11 +605,7 @@ class CalDAVRequestPolicy {
|
||||
}
|
||||
|
||||
try {
|
||||
await assertSsrFSafeUrl(url, {
|
||||
allowedProtocols: this.allowInsecureHttp
|
||||
? CALDAV_ALLOWED_PROTOCOLS_INSECURE
|
||||
: CALDAV_ALLOWED_PROTOCOLS,
|
||||
});
|
||||
await assertSsrFSafeUrl(url);
|
||||
} catch (error) {
|
||||
if (error instanceof SsrfBlockedError) {
|
||||
const reason = String(error.data?.reason ?? '');
|
||||
|
||||
@@ -2,7 +2,8 @@ import { Logger } from '@nestjs/common';
|
||||
import { GoogleAuth, GoogleAuthOptions } from 'google-auth-library';
|
||||
import z from 'zod';
|
||||
|
||||
import { OneMinute, safeFetch } from '../../../base';
|
||||
import { OneMinute } from '../../../base';
|
||||
import { inferRemoteMimeType } from '../../../native';
|
||||
import { PromptAttachment, StreamObject } from './types';
|
||||
|
||||
export type VertexProviderConfig = {
|
||||
@@ -33,30 +34,7 @@ type CopilotTextStreamPart =
|
||||
}
|
||||
| { type: 'error'; error: unknown };
|
||||
|
||||
const ATTACH_HEAD_PARAMS = { timeoutMs: OneMinute / 12, maxRedirects: 3 };
|
||||
const FORMAT_INFER_MAP: Record<string, string> = {
|
||||
pdf: 'application/pdf',
|
||||
mp3: 'audio/mpeg',
|
||||
opus: 'audio/opus',
|
||||
ogg: 'audio/ogg',
|
||||
aac: 'audio/aac',
|
||||
m4a: 'audio/aac',
|
||||
flac: 'audio/flac',
|
||||
ogv: 'video/ogg',
|
||||
wav: 'audio/wav',
|
||||
png: 'image/png',
|
||||
jpeg: 'image/jpeg',
|
||||
jpg: 'image/jpeg',
|
||||
webp: 'image/webp',
|
||||
txt: 'text/plain',
|
||||
md: 'text/plain',
|
||||
mov: 'video/mov',
|
||||
mpeg: 'video/mpeg',
|
||||
mp4: 'video/mp4',
|
||||
avi: 'video/avi',
|
||||
wmv: 'video/wmv',
|
||||
flv: 'video/flv',
|
||||
};
|
||||
const ATTACH_HEAD_TIMEOUT_MS = OneMinute / 12;
|
||||
|
||||
function toBase64Data(data: string, encoding: 'base64' | 'utf8' = 'base64') {
|
||||
return encoding === 'base64'
|
||||
@@ -97,25 +75,7 @@ export async function inferMimeType(url: string) {
|
||||
if (url.startsWith('data:')) {
|
||||
return url.split(';')[0].split(':')[1];
|
||||
}
|
||||
const pathname = new URL(url).pathname;
|
||||
const extension = pathname.split('.').pop();
|
||||
if (extension) {
|
||||
const ext = FORMAT_INFER_MAP[extension];
|
||||
if (ext) {
|
||||
return ext;
|
||||
}
|
||||
}
|
||||
try {
|
||||
const mimeType = await safeFetch(
|
||||
url,
|
||||
{ method: 'HEAD' },
|
||||
ATTACH_HEAD_PARAMS
|
||||
).then(res => res.headers.get('content-type'));
|
||||
if (mimeType) return mimeType;
|
||||
} catch {
|
||||
// ignore and fallback to default
|
||||
}
|
||||
return 'application/octet-stream';
|
||||
return inferRemoteMimeType({ url, timeoutMs: ATTACH_HEAD_TIMEOUT_MS });
|
||||
}
|
||||
|
||||
type CitationIndexedEvent = {
|
||||
|
||||
+21
-29
@@ -1,10 +1,7 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
Config,
|
||||
readResponseBufferWithLimit,
|
||||
safeFetch,
|
||||
} from '../../../../base';
|
||||
import { Config } from '../../../../base';
|
||||
import { fetchRemoteAttachment } from '../../../../native';
|
||||
|
||||
type FetchRemoteAttachmentOptions = {
|
||||
signal?: AbortSignal;
|
||||
@@ -37,9 +34,12 @@ export class AttachmentMaterializer {
|
||||
constructor(private readonly config: Config) {}
|
||||
|
||||
private buildFetchOptions(url: URL, trustedHostSuffixes: string[]) {
|
||||
const baseOptions = { timeoutMs: 15_000, maxRedirects: 3 } as const;
|
||||
const baseOptions = {
|
||||
timeoutMs: 15_000,
|
||||
allowPrivateTargetOrigin: false,
|
||||
};
|
||||
if (!env.prod) {
|
||||
return { ...baseOptions, allowPrivateOrigins: new Set([url.origin]) };
|
||||
return { ...baseOptions, allowPrivateTargetOrigin: true };
|
||||
}
|
||||
|
||||
const trustedOrigins = new Set<string>();
|
||||
@@ -80,7 +80,7 @@ export class AttachmentMaterializer {
|
||||
suffix => hostname === suffix || hostname.endsWith(`.${suffix}`)
|
||||
);
|
||||
if (trustedOrigins.has(url.origin) || trustedByHost) {
|
||||
return { ...baseOptions, allowPrivateOrigins: new Set([url.origin]) };
|
||||
return { ...baseOptions, allowPrivateTargetOrigin: true };
|
||||
}
|
||||
|
||||
return baseOptions;
|
||||
@@ -91,30 +91,22 @@ export class AttachmentMaterializer {
|
||||
options: FetchRemoteAttachmentOptions
|
||||
) {
|
||||
const parsed = resolveAttachmentFetchUrl(url);
|
||||
const response = await safeFetch(
|
||||
parsed,
|
||||
{ method: 'GET', signal: options.signal },
|
||||
this.buildFetchOptions(parsed, options.trustedHostSuffixes ?? [])
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to fetch attachment: ${response.status} ${response.statusText}`
|
||||
);
|
||||
}
|
||||
|
||||
const buffer = await readResponseBufferWithLimit(
|
||||
response,
|
||||
options.maxBytes
|
||||
);
|
||||
const headerMimeType = normalizeMimeType(
|
||||
response.headers.get('content-type') || ''
|
||||
);
|
||||
options.signal?.throwIfAborted();
|
||||
const response = await fetchRemoteAttachment({
|
||||
url: parsed.toString(),
|
||||
...this.buildFetchOptions(parsed, options.trustedHostSuffixes ?? []),
|
||||
maxBytes: options.maxBytes,
|
||||
});
|
||||
options.signal?.throwIfAborted();
|
||||
|
||||
return {
|
||||
data: buffer.toString('base64'),
|
||||
data: response.body.toString('base64'),
|
||||
mimeType: options.detectMimeType
|
||||
? options.detectMimeType(buffer, headerMimeType)
|
||||
: headerMimeType,
|
||||
? options.detectMimeType(
|
||||
response.body,
|
||||
normalizeMimeType(response.mimeType)
|
||||
)
|
||||
: normalizeMimeType(response.mimeType),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import {
|
||||
BlobQuotaExceeded,
|
||||
CallMetric,
|
||||
Config,
|
||||
fetchBuffer,
|
||||
type FileUpload,
|
||||
OneMB,
|
||||
OnEvent,
|
||||
@@ -17,6 +16,7 @@ import {
|
||||
URLHelper,
|
||||
} from '../../base';
|
||||
import { QuotaService } from '../../core/quota';
|
||||
import { fetchRemoteAttachment } from '../../native';
|
||||
|
||||
const REMOTE_BLOB_MAX_BYTES = 20 * OneMB;
|
||||
|
||||
@@ -93,12 +93,15 @@ export class CopilotStorage {
|
||||
|
||||
@CallMetric('ai', 'blob_proxy_remote_url')
|
||||
async handleRemoteLink(userId: string, workspaceId: string, link: string) {
|
||||
const { buffer, type } = await fetchBuffer(
|
||||
link,
|
||||
REMOTE_BLOB_MAX_BYTES,
|
||||
'image/'
|
||||
);
|
||||
const { body, mimeType } = await fetchRemoteAttachment({
|
||||
url: link,
|
||||
maxBytes: REMOTE_BLOB_MAX_BYTES,
|
||||
expectedContentTypePrefix: 'image/',
|
||||
maxImageHeight: 4096,
|
||||
maxImageWidth: 4096,
|
||||
});
|
||||
const buffer = Buffer.from(body);
|
||||
const filename = createHash('sha256').update(buffer).digest('base64url');
|
||||
return this.put(userId, workspaceId, filename, buffer, type);
|
||||
return this.put(userId, workspaceId, filename, buffer, mimeType);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,15 +17,16 @@ import {
|
||||
applyAttachHeaders,
|
||||
BadRequest,
|
||||
Cache,
|
||||
readResponseBufferWithLimit,
|
||||
ResponseTooLargeError,
|
||||
safeFetch,
|
||||
SsrfBlockedError,
|
||||
type SSRFBlockReason,
|
||||
Throttle,
|
||||
URLHelper,
|
||||
UseNamedGuard,
|
||||
} from '../../base';
|
||||
import { Public } from '../../core/auth';
|
||||
import { inspectImageForProxy } from '../../native';
|
||||
import { WorkerService } from './service';
|
||||
import type { LinkPreviewRequest, LinkPreviewResponse } from './types';
|
||||
import {
|
||||
@@ -64,6 +65,7 @@ function toBadRequestReason(reason: SSRFBlockReason) {
|
||||
|
||||
@Public()
|
||||
@UseNamedGuard('selfhost')
|
||||
@Throttle('default', { limit: 60, ttl: 60_000 })
|
||||
@Controller('/api/worker')
|
||||
export class WorkerController {
|
||||
private readonly logger = new Logger(WorkerController.name);
|
||||
@@ -133,7 +135,8 @@ export class WorkerController {
|
||||
...(origin ? { Vary: 'Origin' } : {}),
|
||||
'Access-Control-Allow-Methods': 'GET',
|
||||
});
|
||||
applyAttachHeaders(resp, { buffer });
|
||||
const image = this.inspectProxyImage(buffer, imageURL);
|
||||
applyAttachHeaders(resp, { buffer, contentType: image.mimeType });
|
||||
const contentType = resp.getHeader('Content-Type') as string | undefined;
|
||||
if (contentType?.startsWith('image/')) {
|
||||
return resp.status(200).send(buffer);
|
||||
@@ -147,7 +150,11 @@ export class WorkerController {
|
||||
response = await safeFetch(
|
||||
targetURL.toString(),
|
||||
{ method: 'GET', headers: cloneHeader(req.headers) },
|
||||
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
|
||||
{
|
||||
timeoutMs: FETCH_TIMEOUT_MS,
|
||||
maxRedirects: MAX_REDIRECTS,
|
||||
maxBytes: IMAGE_PROXY_MAX_BYTES,
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof SsrfBlockedError) {
|
||||
@@ -155,7 +162,6 @@ export class WorkerController {
|
||||
this.logger.warn('Blocked image proxy target', {
|
||||
url: imageURL,
|
||||
reason,
|
||||
context: (error as any).context,
|
||||
});
|
||||
throw new BadRequest(toBadRequestReason(reason ?? 'invalid_url'));
|
||||
}
|
||||
@@ -175,23 +181,8 @@ export class WorkerController {
|
||||
throw new BadRequest('Failed to fetch image');
|
||||
}
|
||||
if (response.ok) {
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = await readResponseBufferWithLimit(
|
||||
response,
|
||||
IMAGE_PROXY_MAX_BYTES
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof ResponseTooLargeError) {
|
||||
this.logger.warn('Image proxy response too large', {
|
||||
url: imageURL,
|
||||
limitBytes: error.data?.limitBytes,
|
||||
receivedBytes: error.data?.receivedBytes,
|
||||
});
|
||||
throw new BadRequest('Response too large');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const image = this.inspectProxyImage(buffer, imageURL);
|
||||
await this.cache.set(cachedUrl, buffer.toString('base64'), {
|
||||
ttl: CACHE_TTL,
|
||||
});
|
||||
@@ -204,7 +195,7 @@ export class WorkerController {
|
||||
if (contentDisposition) {
|
||||
resp.setHeader('Content-Disposition', contentDisposition);
|
||||
}
|
||||
applyAttachHeaders(resp, { buffer });
|
||||
applyAttachHeaders(resp, { buffer, contentType: image.mimeType });
|
||||
const contentType = resp.getHeader('Content-Type') as string | undefined;
|
||||
if (contentType?.startsWith('image/')) {
|
||||
return resp.status(200).send(buffer);
|
||||
@@ -227,6 +218,18 @@ export class WorkerController {
|
||||
}
|
||||
}
|
||||
|
||||
private inspectProxyImage(buffer: Buffer, url: string) {
|
||||
try {
|
||||
return inspectImageForProxy(buffer);
|
||||
} catch (error) {
|
||||
this.logger.warn('Image proxy rejected invalid image', {
|
||||
url,
|
||||
error,
|
||||
});
|
||||
throw new BadRequest('Invalid image');
|
||||
}
|
||||
}
|
||||
|
||||
@Options('/link-preview')
|
||||
linkPreviewOption(
|
||||
@Req() request: ExpressRequest,
|
||||
@@ -291,7 +294,11 @@ export class WorkerController {
|
||||
const response = await safeFetch(
|
||||
targetURL.toString(),
|
||||
{ method, headers: cloneHeader(request.headers) },
|
||||
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
|
||||
{
|
||||
timeoutMs: FETCH_TIMEOUT_MS,
|
||||
maxRedirects: MAX_REDIRECTS,
|
||||
maxBytes: LINK_PREVIEW_MAX_BYTES,
|
||||
}
|
||||
);
|
||||
this.logger.debug('Fetched URL', {
|
||||
origin,
|
||||
@@ -318,10 +325,7 @@ export class WorkerController {
|
||||
};
|
||||
|
||||
if (response.body) {
|
||||
const body = await readResponseBufferWithLimit(
|
||||
response,
|
||||
LINK_PREVIEW_MAX_BYTES
|
||||
);
|
||||
const body = Buffer.from(await response.arrayBuffer());
|
||||
const limitedResponse = new Response(body, response);
|
||||
const resp = await decodeWithCharset(limitedResponse, res);
|
||||
|
||||
@@ -402,7 +406,11 @@ export class WorkerController {
|
||||
const faviconResponse = await safeFetch(
|
||||
faviconUrl.toString(),
|
||||
{ method: 'HEAD' },
|
||||
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
|
||||
{
|
||||
timeoutMs: FETCH_TIMEOUT_MS,
|
||||
maxRedirects: MAX_REDIRECTS,
|
||||
maxBytes: 0,
|
||||
}
|
||||
);
|
||||
if (faviconResponse.ok) {
|
||||
appendUrl(faviconUrl.toString(), res.favicons);
|
||||
@@ -433,7 +441,6 @@ export class WorkerController {
|
||||
origin,
|
||||
url: requestBody?.url,
|
||||
reason,
|
||||
context: (error as any).context,
|
||||
});
|
||||
throw new BadRequest(toBadRequestReason(reason ?? 'invalid_url'));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user