mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-20 19:46:32 +08:00
feat(server): native safe fetch (#14931)
This commit is contained in:
DarkSky
@@ -1,27 +0,0 @@
|
||||
import test from 'ava';
|
||||
|
||||
import { readResponseBufferWithLimit } from '../../base';
|
||||
|
||||
test('readResponseBufferWithLimit rejects timed out web streams without crashing', async t => {
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array([1, 2, 3]));
|
||||
queueMicrotask(() => {
|
||||
controller.error(
|
||||
new DOMException(
|
||||
'The operation was aborted due to timeout',
|
||||
'TimeoutError'
|
||||
)
|
||||
);
|
||||
});
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
const error = await t.throwsAsync(
|
||||
readResponseBufferWithLimit(response, 1024)
|
||||
);
|
||||
|
||||
t.is(error?.name, 'TimeoutError');
|
||||
});
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
JobQueue,
|
||||
OneMB,
|
||||
} from '../../base';
|
||||
import { ThrottlerStorage } from '../../base/throttler';
|
||||
import { SocketIoAdapter } from '../../base/websocket';
|
||||
import { AuthGuard, AuthService } from '../../core/auth';
|
||||
import { Mailer } from '../../core/mail';
|
||||
@@ -163,6 +164,10 @@ export class TestingApp extends NestApplication {
|
||||
return await this.create(MockUser, overrides);
|
||||
}
|
||||
|
||||
resetRateLimit() {
|
||||
this.get(ThrottlerStorage, { strict: false }).storage.clear();
|
||||
}
|
||||
|
||||
async signup(overrides?: Partial<MockUserInput>) {
|
||||
const user = await this.create(MockUser, overrides);
|
||||
await this.login(user);
|
||||
@@ -170,6 +175,7 @@ export class TestingApp extends NestApplication {
|
||||
}
|
||||
|
||||
async login(user: MockedUser) {
|
||||
this.resetRateLimit();
|
||||
return await this.POST('/api/auth/sign-in').send({
|
||||
email: user.email,
|
||||
password: user.password,
|
||||
@@ -195,6 +201,7 @@ export class TestingApp extends NestApplication {
|
||||
}
|
||||
|
||||
async logout(userId?: string) {
|
||||
this.resetRateLimit();
|
||||
const res = await this.POST(
|
||||
'/api/auth/sign-out' + (userId ? `?user_id=${userId}` : '')
|
||||
).expect(200);
|
||||
|
||||
@@ -92,7 +92,7 @@ test.before(async t => {
|
||||
throttle: {
|
||||
throttlers: {
|
||||
default: {
|
||||
ttl: 60,
|
||||
ttl: 60_000,
|
||||
limit: 120,
|
||||
},
|
||||
},
|
||||
@@ -109,6 +109,7 @@ test.before(async t => {
|
||||
|
||||
test.beforeEach(async t => {
|
||||
const { app } = t.context;
|
||||
t.context.storage.storage.clear();
|
||||
await app.initTestingDB();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,49 +1,79 @@
|
||||
import { LookupAddress } from 'node:dns';
|
||||
|
||||
import serverNativeModule from '@affine/server-native';
|
||||
import type { ExecutionContext, TestFn } from 'ava';
|
||||
import ava from 'ava';
|
||||
import Sinon from 'sinon';
|
||||
import type { Response } from 'supertest';
|
||||
|
||||
import {
|
||||
__resetDnsLookupForTests,
|
||||
__setDnsLookupForTests,
|
||||
type DnsLookup,
|
||||
} from '../base/utils/ssrf';
|
||||
import { createTestingApp, TestingApp } from './utils';
|
||||
import type { TestingApp } from './utils';
|
||||
|
||||
type TestContext = {
|
||||
app: TestingApp;
|
||||
};
|
||||
const test = ava as TestFn<TestContext>;
|
||||
|
||||
const LookupAddressStub = (async (_hostname, options) => {
|
||||
const result = [{ address: '76.76.21.21', family: 4 }] as LookupAddress[];
|
||||
const isOptions = options && typeof options === 'object';
|
||||
if (isOptions && 'all' in options && options.all) {
|
||||
return result;
|
||||
let safeFetchStub: Sinon.SinonStub | undefined;
|
||||
let safeFetchHandler:
|
||||
| ((request: { url: string; method?: 'get' | 'head' }) => {
|
||||
status?: number;
|
||||
finalUrl?: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: Buffer | string;
|
||||
})
|
||||
| undefined;
|
||||
|
||||
const stubSafeFetch = (
|
||||
handler: (request: { url: string; method?: 'get' | 'head' }) => {
|
||||
status?: number;
|
||||
finalUrl?: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: Buffer | string;
|
||||
}
|
||||
return result[0];
|
||||
}) as DnsLookup;
|
||||
) => {
|
||||
safeFetchHandler = handler;
|
||||
return {
|
||||
restore() {
|
||||
safeFetchHandler = undefined;
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
test.before(async t => {
|
||||
// @ts-expect-error test
|
||||
env.DEPLOYMENT_TYPE = 'selfhosted';
|
||||
|
||||
// Avoid relying on real DNS during tests. SSRF protection uses dns.lookup().
|
||||
__setDnsLookupForTests(LookupAddressStub);
|
||||
safeFetchStub = Sinon.stub(serverNativeModule, 'safeFetch').callsFake(
|
||||
async request => {
|
||||
if (!safeFetchHandler) {
|
||||
throw new Error('Unexpected safeFetch call');
|
||||
}
|
||||
const nativeRequest = request as {
|
||||
url: string;
|
||||
method?: 'get' | 'head';
|
||||
};
|
||||
const response = safeFetchHandler(nativeRequest);
|
||||
return {
|
||||
status: response.status ?? 200,
|
||||
finalUrl: response.finalUrl ?? nativeRequest.url,
|
||||
headers: response.headers ?? {},
|
||||
body: Buffer.isBuffer(response.body)
|
||||
? response.body
|
||||
: Buffer.from(response.body ?? ''),
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
const { createTestingApp } = await import('./utils');
|
||||
const app = await createTestingApp();
|
||||
|
||||
t.context.app = app;
|
||||
});
|
||||
|
||||
test.afterEach.always(() => {
|
||||
Sinon.restore();
|
||||
safeFetchHandler = undefined;
|
||||
});
|
||||
|
||||
test.after.always(async t => {
|
||||
__resetDnsLookupForTests();
|
||||
safeFetchStub?.restore();
|
||||
await t.context.app.close();
|
||||
});
|
||||
|
||||
@@ -128,15 +158,13 @@ test('should proxy image', async t => {
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jfJ8AAAAASUVORK5CYII=',
|
||||
'base64'
|
||||
);
|
||||
const fakeResponse = new Response(fakeBuffer, {
|
||||
status: 200,
|
||||
const fetchSpy = stubSafeFetch(() => ({
|
||||
body: fakeBuffer,
|
||||
headers: {
|
||||
'content-type': 'image/png',
|
||||
'content-disposition': 'inline',
|
||||
},
|
||||
});
|
||||
|
||||
const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeResponse);
|
||||
}));
|
||||
try {
|
||||
await assertAndSnapshot(
|
||||
`/api/worker/image-proxy?url=${imageUrl}`,
|
||||
@@ -146,6 +174,42 @@ test('should proxy image', async t => {
|
||||
fetchSpy.restore();
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const invalidImageUrl = `http://example.com/not-image-${Date.now()}.png`;
|
||||
const invalidFetchSpy = stubSafeFetch(() => ({
|
||||
body: 'not an image',
|
||||
headers: { 'content-type': 'image/png' },
|
||||
}));
|
||||
try {
|
||||
await t.context.app
|
||||
.GET(`/api/worker/image-proxy?url=${invalidImageUrl}`)
|
||||
.set('Origin', 'http://localhost:3010')
|
||||
.send()
|
||||
.expect(400);
|
||||
} finally {
|
||||
invalidFetchSpy.restore();
|
||||
}
|
||||
|
||||
const validImageUrl = `http://example.com/valid-image-${Date.now()}.png`;
|
||||
const fakeBuffer = Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jfJ8AAAAASUVORK5CYII=',
|
||||
'base64'
|
||||
);
|
||||
const validFetchSpy = stubSafeFetch(() => ({
|
||||
body: fakeBuffer,
|
||||
headers: { 'content-type': 'image/png' },
|
||||
}));
|
||||
try {
|
||||
await t.context.app
|
||||
.GET(`/api/worker/image-proxy?url=${validImageUrl}`)
|
||||
.set('Origin', 'http://localhost:3010')
|
||||
.send()
|
||||
.expect(200);
|
||||
} finally {
|
||||
validFetchSpy.restore();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('should preview link', async t => {
|
||||
@@ -190,7 +254,8 @@ test('should preview link', async t => {
|
||||
);
|
||||
|
||||
{
|
||||
const fakeHTML = new Response(`
|
||||
const pageUrl = `http://external.com/page-${Date.now()}`;
|
||||
const fakeHTML = `
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:title" content="Test Title" />
|
||||
@@ -201,13 +266,18 @@ test('should preview link', async t => {
|
||||
<title>Fallback Title</title>
|
||||
</body>
|
||||
</html>
|
||||
`);
|
||||
`;
|
||||
|
||||
Object.defineProperty(fakeHTML, 'url', {
|
||||
value: 'http://example.com/page',
|
||||
const fetchSpy = stubSafeFetch(request => {
|
||||
if (request.url.includes('/favicon.ico')) {
|
||||
return { status: 204, finalUrl: request.url };
|
||||
}
|
||||
return {
|
||||
body: fakeHTML,
|
||||
finalUrl: 'http://example.com/page',
|
||||
headers: { 'content-type': 'text/html;charset=UTF-8' },
|
||||
};
|
||||
});
|
||||
|
||||
const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeHTML);
|
||||
try {
|
||||
await assertAndSnapshot(
|
||||
'/api/worker/link-preview',
|
||||
@@ -215,7 +285,7 @@ test('should preview link', async t => {
|
||||
{
|
||||
status: 200,
|
||||
method: 'POST',
|
||||
body: { url: 'http://external.com/page' },
|
||||
body: { url: pageUrl },
|
||||
}
|
||||
);
|
||||
} finally {
|
||||
@@ -244,6 +314,7 @@ test('should preview link', async t => {
|
||||
];
|
||||
|
||||
for (const { content, charset } of encoded) {
|
||||
const pageUrl = `http://example.com/${charset}-${Date.now()}`;
|
||||
const before = Buffer.from(`<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=${charset}" />
|
||||
@@ -253,13 +324,18 @@ test('should preview link', async t => {
|
||||
</head>
|
||||
</html>
|
||||
`);
|
||||
const fakeHTML = new Response(Buffer.concat([before, encoded, after]));
|
||||
const fakeHTML = Buffer.concat([before, encoded, after]);
|
||||
|
||||
Object.defineProperty(fakeHTML, 'url', {
|
||||
value: `http://example.com/${charset}`,
|
||||
const fetchSpy = stubSafeFetch(request => {
|
||||
if (request.url.includes('/favicon.ico')) {
|
||||
return { status: 204, finalUrl: request.url };
|
||||
}
|
||||
return {
|
||||
body: fakeHTML,
|
||||
finalUrl: `http://example.com/${charset}`,
|
||||
headers: { 'content-type': `text/html;charset=${charset}` },
|
||||
};
|
||||
});
|
||||
|
||||
const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeHTML);
|
||||
try {
|
||||
await assertAndSnapshot(
|
||||
'/api/worker/link-preview',
|
||||
@@ -267,7 +343,7 @@ test('should preview link', async t => {
|
||||
{
|
||||
status: 200,
|
||||
method: 'POST',
|
||||
body: { url: `http://example.com/${charset}` },
|
||||
body: { url: pageUrl },
|
||||
}
|
||||
);
|
||||
} finally {
|
||||
|
||||
@@ -24,14 +24,14 @@ defineModuleConfig('throttle', {
|
||||
'throttlers.default': {
|
||||
desc: 'The config for the default throttler.',
|
||||
default: {
|
||||
ttl: 60,
|
||||
ttl: 60_000,
|
||||
limit: 120,
|
||||
},
|
||||
},
|
||||
'throttlers.strict': {
|
||||
desc: 'The config for the strict throttler.',
|
||||
default: {
|
||||
ttl: 60,
|
||||
ttl: 60_000,
|
||||
limit: 20,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -13,7 +13,7 @@ export const THROTTLER_PROTECTED = 'affine_throttler:protected';
|
||||
* it will never be rate limited.
|
||||
*
|
||||
* - default: 120 calls within 60 seconds
|
||||
* - strict: 10 calls within 60 seconds
|
||||
* - strict: 20 calls within 60 seconds
|
||||
* - authenticated: no rate limit for authenticated users, apply [default] throttler for unauthenticated users
|
||||
*
|
||||
* @example
|
||||
@@ -23,7 +23,7 @@ export const THROTTLER_PROTECTED = 'affine_throttler:protected';
|
||||
*
|
||||
* // the config call be override by the second parameter,
|
||||
* // and the call count will be calculated separately
|
||||
* \@Throttle('default', { limit: 10, ttl: 10 })
|
||||
* \@Throttle('default', { limit: 10, ttl: 10_000 })
|
||||
*
|
||||
*/
|
||||
export function Throttle(
|
||||
|
||||
@@ -1,23 +1,9 @@
|
||||
import * as dns from 'node:dns/promises';
|
||||
import { BlockList, isIP } from 'node:net';
|
||||
|
||||
import {
|
||||
assertSafeUrl as assertSafeUrlFromNative,
|
||||
safeFetch as safeFetchFromNative,
|
||||
type SafeFetchRequest,
|
||||
} from '../../native';
|
||||
import { ResponseTooLargeError, SsrfBlockedError } from '../error/errors.gen';
|
||||
import { OneMinute } from './unit';
|
||||
|
||||
const DEFAULT_ALLOWED_PROTOCOLS = new Set(['http:', 'https:']);
|
||||
const BLOCKED_IPS = new BlockList();
|
||||
const ALLOWED_IPV6 = new BlockList();
|
||||
|
||||
export type DnsLookup = typeof dns.lookup;
|
||||
let dnsLookup: DnsLookup = dns.lookup;
|
||||
|
||||
export function __setDnsLookupForTests(lookup: DnsLookup) {
|
||||
dnsLookup = lookup;
|
||||
}
|
||||
|
||||
export function __resetDnsLookupForTests() {
|
||||
dnsLookup = dns.lookup;
|
||||
}
|
||||
|
||||
export type SSRFBlockReason =
|
||||
| 'invalid_url'
|
||||
@@ -28,187 +14,55 @@ export type SSRFBlockReason =
|
||||
| 'blocked_ip'
|
||||
| 'too_many_redirects';
|
||||
|
||||
type SsrfErrorContext = { url?: string; hostname?: string; address?: string };
|
||||
const SSRF_REASONS = new Set<string>([
|
||||
'invalid_url',
|
||||
'disallowed_protocol',
|
||||
'url_has_credentials',
|
||||
'blocked_hostname',
|
||||
'unresolvable_hostname',
|
||||
'blocked_ip',
|
||||
'too_many_redirects',
|
||||
]);
|
||||
|
||||
function createSsrfBlockedError(
|
||||
reason: SSRFBlockReason,
|
||||
context?: SsrfErrorContext
|
||||
) {
|
||||
const err = new SsrfBlockedError({ reason });
|
||||
// For logging/debugging only (not part of UserFriendlyError JSON).
|
||||
(err as any).context = context;
|
||||
return err;
|
||||
function createSsrfBlockedError(reason: SSRFBlockReason) {
|
||||
return new SsrfBlockedError({ reason });
|
||||
}
|
||||
|
||||
export interface SSRFProtectionOptions {
|
||||
allowedProtocols?: ReadonlySet<string>;
|
||||
/**
|
||||
* Allow fetching private/reserved IPs when URL.origin is allowlisted.
|
||||
* Defaults to an empty allowlist (i.e. private IPs are blocked).
|
||||
*/
|
||||
allowPrivateOrigins?: ReadonlySet<string>;
|
||||
}
|
||||
|
||||
function stripZoneId(address: string) {
|
||||
const idx = address.indexOf('%');
|
||||
return idx === -1 ? address : address.slice(0, idx);
|
||||
}
|
||||
|
||||
// IPv4: RFC1918 + loopback + link-local + CGNAT + special/reserved
|
||||
for (const [network, prefix] of [
|
||||
['0.0.0.0', 8],
|
||||
['10.0.0.0', 8],
|
||||
['127.0.0.0', 8],
|
||||
['169.254.0.0', 16],
|
||||
['172.16.0.0', 12],
|
||||
['192.168.0.0', 16],
|
||||
['100.64.0.0', 10], // CGNAT
|
||||
['192.0.0.0', 24],
|
||||
['192.0.2.0', 24], // TEST-NET-1
|
||||
['198.51.100.0', 24], // TEST-NET-2
|
||||
['203.0.113.0', 24], // TEST-NET-3
|
||||
['198.18.0.0', 15], // benchmark
|
||||
['192.88.99.0', 24], // 6to4 relay
|
||||
['224.0.0.0', 4], // multicast
|
||||
['240.0.0.0', 4], // reserved (includes broadcast)
|
||||
] as const) {
|
||||
BLOCKED_IPS.addSubnet(network, prefix, 'ipv4');
|
||||
}
|
||||
|
||||
// IPv6: block loopback/unspecified/link-local/ULA/multicast/doc; allow only global unicast.
|
||||
BLOCKED_IPS.addAddress('::', 'ipv6');
|
||||
BLOCKED_IPS.addAddress('::1', 'ipv6');
|
||||
BLOCKED_IPS.addSubnet('ff00::', 8, 'ipv6'); // multicast
|
||||
BLOCKED_IPS.addSubnet('fc00::', 7, 'ipv6'); // unique local
|
||||
BLOCKED_IPS.addSubnet('fe80::', 10, 'ipv6'); // link-local
|
||||
BLOCKED_IPS.addSubnet('2001:db8::', 32, 'ipv6'); // documentation
|
||||
ALLOWED_IPV6.addSubnet('2000::', 3, 'ipv6'); // global unicast
|
||||
|
||||
function extractEmbeddedIPv4FromIPv6(address: string): string | null {
|
||||
if (!address.includes('.')) {
|
||||
return null;
|
||||
function mapNativeFetchError(error: unknown, limitBytes?: number) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const reason = [...SSRF_REASONS].find(reason => message.includes(reason));
|
||||
if (reason) {
|
||||
return createSsrfBlockedError(reason as SSRFBlockReason);
|
||||
}
|
||||
const idx = address.lastIndexOf(':');
|
||||
if (idx === -1) {
|
||||
return null;
|
||||
if (message.includes('response_too_large')) {
|
||||
return new ResponseTooLargeError({
|
||||
limitBytes: limitBytes ?? 0,
|
||||
receivedBytes: limitBytes ? limitBytes + 1 : 0,
|
||||
});
|
||||
}
|
||||
const tail = address.slice(idx + 1);
|
||||
return isIP(tail) === 4 ? tail : null;
|
||||
return error;
|
||||
}
|
||||
|
||||
function isBlockedIpAddress(address: string): boolean {
|
||||
const ip = stripZoneId(address);
|
||||
const family = isIP(ip);
|
||||
if (family === 4) {
|
||||
return BLOCKED_IPS.check(ip, 'ipv4');
|
||||
}
|
||||
if (family === 6) {
|
||||
const embeddedV4 = extractEmbeddedIPv4FromIPv6(ip);
|
||||
if (embeddedV4) {
|
||||
return isBlockedIpAddress(embeddedV4);
|
||||
}
|
||||
if (!ALLOWED_IPV6.check(ip, 'ipv6')) {
|
||||
return true;
|
||||
}
|
||||
return BLOCKED_IPS.check(ip, 'ipv6');
|
||||
}
|
||||
return true;
|
||||
export interface SafeFetchOptions {
|
||||
timeoutMs?: number;
|
||||
maxRedirects?: number;
|
||||
maxBytes?: number;
|
||||
}
|
||||
|
||||
async function resolveHostAddresses(hostname: string): Promise<string[]> {
|
||||
// Normalize common localhost aliases without DNS.
|
||||
const lowered = hostname.toLowerCase();
|
||||
if (lowered === 'localhost' || lowered.endsWith('.localhost')) {
|
||||
return ['127.0.0.1', '::1'];
|
||||
}
|
||||
|
||||
const results = await dnsLookup(hostname, {
|
||||
all: true,
|
||||
verbatim: true,
|
||||
});
|
||||
return results.map(r => r.address);
|
||||
}
|
||||
|
||||
export async function assertSsrFSafeUrl(
|
||||
rawUrl: string | URL,
|
||||
options: SSRFProtectionOptions = {}
|
||||
): Promise<URL> {
|
||||
const allowedProtocols =
|
||||
options.allowedProtocols ?? DEFAULT_ALLOWED_PROTOCOLS;
|
||||
|
||||
export async function assertSsrFSafeUrl(rawUrl: string | URL): Promise<URL> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = rawUrl instanceof URL ? rawUrl : new URL(rawUrl);
|
||||
} catch {
|
||||
throw createSsrfBlockedError('invalid_url', {
|
||||
url: typeof rawUrl === 'string' ? rawUrl : undefined,
|
||||
});
|
||||
throw createSsrfBlockedError('invalid_url');
|
||||
}
|
||||
|
||||
if (!allowedProtocols.has(url.protocol)) {
|
||||
throw createSsrfBlockedError('disallowed_protocol', {
|
||||
url: url.toString(),
|
||||
});
|
||||
}
|
||||
|
||||
if (url.username || url.password) {
|
||||
throw createSsrfBlockedError('url_has_credentials', {
|
||||
url: url.toString(),
|
||||
});
|
||||
}
|
||||
|
||||
const hostname = url.hostname;
|
||||
if (!hostname) {
|
||||
throw createSsrfBlockedError('blocked_hostname', { url: url.toString() });
|
||||
}
|
||||
|
||||
const allowPrivate =
|
||||
options.allowPrivateOrigins && options.allowPrivateOrigins.has(url.origin);
|
||||
|
||||
// IP literal
|
||||
if (isIP(hostname)) {
|
||||
if (isBlockedIpAddress(hostname) && !allowPrivate) {
|
||||
throw createSsrfBlockedError('blocked_ip', {
|
||||
url: url.toString(),
|
||||
address: hostname,
|
||||
});
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
let addresses: string[];
|
||||
try {
|
||||
addresses = await resolveHostAddresses(hostname);
|
||||
} catch {
|
||||
throw createSsrfBlockedError('unresolvable_hostname', {
|
||||
url: url.toString(),
|
||||
hostname,
|
||||
});
|
||||
assertSafeUrlFromNative({ url: url.toString() });
|
||||
return url;
|
||||
} catch (error) {
|
||||
throw mapNativeFetchError(error);
|
||||
}
|
||||
|
||||
if (addresses.length === 0) {
|
||||
throw createSsrfBlockedError('unresolvable_hostname', {
|
||||
url: url.toString(),
|
||||
hostname,
|
||||
});
|
||||
}
|
||||
|
||||
for (const address of addresses) {
|
||||
if (isBlockedIpAddress(address) && !allowPrivate) {
|
||||
throw createSsrfBlockedError('blocked_ip', {
|
||||
url: url.toString(),
|
||||
hostname,
|
||||
address,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
export interface SafeFetchOptions extends SSRFProtectionOptions {
|
||||
timeoutMs?: number;
|
||||
maxRedirects?: number;
|
||||
}
|
||||
|
||||
export async function safeFetch(
|
||||
@@ -216,153 +70,51 @@ export async function safeFetch(
|
||||
init: RequestInit = {},
|
||||
options: SafeFetchOptions = {}
|
||||
): Promise<Response> {
|
||||
const timeoutMs = options.timeoutMs ?? 10_000;
|
||||
const maxRedirects = options.maxRedirects ?? 3;
|
||||
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
const signal = init.signal
|
||||
? AbortSignal.any([init.signal, timeoutSignal])
|
||||
: timeoutSignal;
|
||||
|
||||
let current = await assertSsrFSafeUrl(rawUrl, options);
|
||||
let redirects = 0;
|
||||
|
||||
// Always handle redirects manually (SSRF-safe on each hop).
|
||||
let requestInit: RequestInit = {
|
||||
...init,
|
||||
redirect: 'manual',
|
||||
signal,
|
||||
};
|
||||
|
||||
while (true) {
|
||||
const response = await fetch(current, requestInit);
|
||||
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
const location = response.headers.get('location');
|
||||
if (!location) {
|
||||
return response;
|
||||
}
|
||||
|
||||
// Drain/cancel body before following redirect to avoid leaking resources.
|
||||
try {
|
||||
await response.body?.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
if (redirects >= maxRedirects) {
|
||||
throw createSsrfBlockedError('too_many_redirects', {
|
||||
url: current.toString(),
|
||||
});
|
||||
}
|
||||
|
||||
const next = new URL(location, current);
|
||||
current = await assertSsrFSafeUrl(next, options);
|
||||
redirects += 1;
|
||||
|
||||
// 303 forces GET semantics
|
||||
if (
|
||||
response.status === 303 &&
|
||||
requestInit.method &&
|
||||
requestInit.method !== 'GET'
|
||||
) {
|
||||
requestInit = { ...requestInit, method: 'GET', body: undefined };
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
return response;
|
||||
const url = rawUrl.toString();
|
||||
const method = String(init.method ?? 'GET').toUpperCase();
|
||||
if (method !== 'GET' && method !== 'HEAD') {
|
||||
throw new Error(`Unsupported safeFetch method: ${method}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function readResponseBufferWithLimit(
|
||||
response: Response,
|
||||
limitBytes: number
|
||||
): Promise<Buffer> {
|
||||
const rawLen = response.headers.get('content-length');
|
||||
if (rawLen) {
|
||||
const len = Number.parseInt(rawLen, 10);
|
||||
if (Number.isFinite(len) && len > limitBytes) {
|
||||
try {
|
||||
await response.body?.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
throw new ResponseTooLargeError({ limitBytes, receivedBytes: len });
|
||||
}
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
return Buffer.alloc(0);
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const chunk = value ?? new Uint8Array();
|
||||
total += chunk.byteLength;
|
||||
if (total > limitBytes) {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
throw new ResponseTooLargeError({ limitBytes, receivedBytes: total });
|
||||
}
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const response = await safeFetchFromNative({
|
||||
url,
|
||||
method: (method === 'HEAD' ? 'head' : 'get') as NonNullable<
|
||||
SafeFetchRequest['method']
|
||||
>,
|
||||
headers: normalizeHeaders(init.headers),
|
||||
timeoutMs: options.timeoutMs,
|
||||
maxRedirects: options.maxRedirects,
|
||||
maxBytes: options.maxBytes,
|
||||
});
|
||||
const body =
|
||||
method === 'HEAD' || [204, 205, 304].includes(response.status)
|
||||
? null
|
||||
: response.body;
|
||||
const webResponse = new Response(body, {
|
||||
status: response.status,
|
||||
headers: response.headers,
|
||||
});
|
||||
Object.defineProperty(webResponse, 'url', {
|
||||
value: response.finalUrl,
|
||||
});
|
||||
return webResponse;
|
||||
} catch (error) {
|
||||
try {
|
||||
await reader.cancel(error);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
reader.releaseLock();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
throw mapNativeFetchError(error, options.maxBytes);
|
||||
}
|
||||
|
||||
return Buffer.concat(
|
||||
chunks.map(chunk => Buffer.from(chunk)),
|
||||
total
|
||||
);
|
||||
}
|
||||
|
||||
type FetchBufferResult = { buffer: Buffer; type: string };
|
||||
const ATTACH_GET_PARAMS = { timeoutMs: OneMinute / 6, maxRedirects: 3 };
|
||||
|
||||
export async function fetchBuffer(
|
||||
url: string,
|
||||
limit: number,
|
||||
contentType?: string
|
||||
): Promise<FetchBufferResult> {
|
||||
const resp = url.startsWith('data:')
|
||||
? await fetch(url)
|
||||
: await safeFetch(url, { method: 'GET' }, ATTACH_GET_PARAMS);
|
||||
|
||||
if (!resp.ok) {
|
||||
throw new Error(
|
||||
`Failed to fetch attachment: ${resp.status} ${resp.statusText}`
|
||||
);
|
||||
function normalizeHeaders(headers: RequestInit['headers'] | undefined) {
|
||||
if (!headers) return undefined;
|
||||
if (headers instanceof Headers) {
|
||||
return Object.fromEntries(headers.entries());
|
||||
}
|
||||
const type = resp.headers.get('content-type') || 'application/octet-stream';
|
||||
if (contentType && !type.startsWith(contentType)) {
|
||||
throw new Error(
|
||||
`Attachment content-type mismatch: expected ${contentType} but got ${type}`
|
||||
);
|
||||
if (Array.isArray(headers)) {
|
||||
return Object.fromEntries(headers);
|
||||
}
|
||||
const buffer = await readResponseBufferWithLimit(resp, limit);
|
||||
return { buffer, type: type };
|
||||
return Object.fromEntries(
|
||||
Object.entries(headers).map(([key, value]) => [key, String(value)])
|
||||
);
|
||||
}
|
||||
|
||||
export function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import serverNativeModule, {
|
||||
type ActionEvent as NativeActionEventContract,
|
||||
type ActionRuntimeInput as NativeActionRuntimeInputContract,
|
||||
type AssertSafeUrlRequest,
|
||||
type BuiltInPromptRenderContract,
|
||||
type BuiltInPromptSessionContract,
|
||||
type BuiltInPromptSpec,
|
||||
@@ -8,6 +9,8 @@ import serverNativeModule, {
|
||||
type CanonicalStructuredRequestContract,
|
||||
type CapabilityAttachmentContract,
|
||||
type CapabilityModelCapability,
|
||||
type ImageInspection,
|
||||
type ImageInspectionOptions,
|
||||
type LlmCoreMessage,
|
||||
type LlmEmbeddingRequestContract,
|
||||
type LlmImageRequestContract,
|
||||
@@ -27,16 +30,29 @@ import serverNativeModule, {
|
||||
type PromptStructuredResponseContract,
|
||||
type PromptTokenCountContract,
|
||||
type PromptTokenCountResult,
|
||||
type RemoteAttachmentFetchRequest,
|
||||
type RemoteAttachmentFetchResponse,
|
||||
type RemoteMimeTypeRequest,
|
||||
type RequestedModelMatchResponse,
|
||||
type SafeFetchRequest,
|
||||
type SafeFetchResponse,
|
||||
type Tokenizer,
|
||||
} from '@affine/server-native';
|
||||
|
||||
export type {
|
||||
AssertSafeUrlRequest,
|
||||
CapabilityAttachmentContract,
|
||||
CapabilityModelCapability,
|
||||
ImageInspection,
|
||||
ImageInspectionOptions,
|
||||
ModelConditionsContract,
|
||||
PromptMessageContract,
|
||||
PromptStructuredResponseContract,
|
||||
RemoteAttachmentFetchRequest,
|
||||
RemoteAttachmentFetchResponse,
|
||||
RemoteMimeTypeRequest,
|
||||
SafeFetchRequest,
|
||||
SafeFetchResponse,
|
||||
};
|
||||
|
||||
export type ActionEventType =
|
||||
@@ -118,6 +134,11 @@ export function getTokenEncoder(model?: string | null): Tokenizer | null {
|
||||
}
|
||||
|
||||
export const getMime = serverNativeModule.getMime;
|
||||
export const inspectImageForProxy = serverNativeModule.inspectImageForProxy;
|
||||
export const fetchRemoteAttachment = serverNativeModule.fetchRemoteAttachment;
|
||||
export const inferRemoteMimeType = serverNativeModule.inferRemoteMimeType;
|
||||
export const assertSafeUrl = serverNativeModule.assertSafeUrl;
|
||||
export const safeFetch = serverNativeModule.safeFetch;
|
||||
export const parseDoc = serverNativeModule.parseDoc;
|
||||
export const htmlSanitize = serverNativeModule.htmlSanitize;
|
||||
export const processImage = serverNativeModule.processImage;
|
||||
|
||||
@@ -35,8 +35,6 @@ const XML_PARSER = new XMLParser({
|
||||
|
||||
const DEFAULT_REQUEST_TIMEOUT_MS = 10_000;
|
||||
const DEFAULT_MAX_REDIRECTS = 5;
|
||||
const CALDAV_ALLOWED_PROTOCOLS = new Set(['https:']);
|
||||
const CALDAV_ALLOWED_PROTOCOLS_INSECURE = new Set(['http:', 'https:']);
|
||||
|
||||
type CalDAVCredentials = {
|
||||
username: string;
|
||||
@@ -607,11 +605,7 @@ class CalDAVRequestPolicy {
|
||||
}
|
||||
|
||||
try {
|
||||
await assertSsrFSafeUrl(url, {
|
||||
allowedProtocols: this.allowInsecureHttp
|
||||
? CALDAV_ALLOWED_PROTOCOLS_INSECURE
|
||||
: CALDAV_ALLOWED_PROTOCOLS,
|
||||
});
|
||||
await assertSsrFSafeUrl(url);
|
||||
} catch (error) {
|
||||
if (error instanceof SsrfBlockedError) {
|
||||
const reason = String(error.data?.reason ?? '');
|
||||
|
||||
@@ -2,7 +2,8 @@ import { Logger } from '@nestjs/common';
|
||||
import { GoogleAuth, GoogleAuthOptions } from 'google-auth-library';
|
||||
import z from 'zod';
|
||||
|
||||
import { OneMinute, safeFetch } from '../../../base';
|
||||
import { OneMinute } from '../../../base';
|
||||
import { inferRemoteMimeType } from '../../../native';
|
||||
import { PromptAttachment, StreamObject } from './types';
|
||||
|
||||
export type VertexProviderConfig = {
|
||||
@@ -33,30 +34,7 @@ type CopilotTextStreamPart =
|
||||
}
|
||||
| { type: 'error'; error: unknown };
|
||||
|
||||
const ATTACH_HEAD_PARAMS = { timeoutMs: OneMinute / 12, maxRedirects: 3 };
|
||||
const FORMAT_INFER_MAP: Record<string, string> = {
|
||||
pdf: 'application/pdf',
|
||||
mp3: 'audio/mpeg',
|
||||
opus: 'audio/opus',
|
||||
ogg: 'audio/ogg',
|
||||
aac: 'audio/aac',
|
||||
m4a: 'audio/aac',
|
||||
flac: 'audio/flac',
|
||||
ogv: 'video/ogg',
|
||||
wav: 'audio/wav',
|
||||
png: 'image/png',
|
||||
jpeg: 'image/jpeg',
|
||||
jpg: 'image/jpeg',
|
||||
webp: 'image/webp',
|
||||
txt: 'text/plain',
|
||||
md: 'text/plain',
|
||||
mov: 'video/mov',
|
||||
mpeg: 'video/mpeg',
|
||||
mp4: 'video/mp4',
|
||||
avi: 'video/avi',
|
||||
wmv: 'video/wmv',
|
||||
flv: 'video/flv',
|
||||
};
|
||||
const ATTACH_HEAD_TIMEOUT_MS = OneMinute / 12;
|
||||
|
||||
function toBase64Data(data: string, encoding: 'base64' | 'utf8' = 'base64') {
|
||||
return encoding === 'base64'
|
||||
@@ -97,25 +75,7 @@ export async function inferMimeType(url: string) {
|
||||
if (url.startsWith('data:')) {
|
||||
return url.split(';')[0].split(':')[1];
|
||||
}
|
||||
const pathname = new URL(url).pathname;
|
||||
const extension = pathname.split('.').pop();
|
||||
if (extension) {
|
||||
const ext = FORMAT_INFER_MAP[extension];
|
||||
if (ext) {
|
||||
return ext;
|
||||
}
|
||||
}
|
||||
try {
|
||||
const mimeType = await safeFetch(
|
||||
url,
|
||||
{ method: 'HEAD' },
|
||||
ATTACH_HEAD_PARAMS
|
||||
).then(res => res.headers.get('content-type'));
|
||||
if (mimeType) return mimeType;
|
||||
} catch {
|
||||
// ignore and fallback to default
|
||||
}
|
||||
return 'application/octet-stream';
|
||||
return inferRemoteMimeType({ url, timeoutMs: ATTACH_HEAD_TIMEOUT_MS });
|
||||
}
|
||||
|
||||
type CitationIndexedEvent = {
|
||||
|
||||
+21
-29
@@ -1,10 +1,7 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
Config,
|
||||
readResponseBufferWithLimit,
|
||||
safeFetch,
|
||||
} from '../../../../base';
|
||||
import { Config } from '../../../../base';
|
||||
import { fetchRemoteAttachment } from '../../../../native';
|
||||
|
||||
type FetchRemoteAttachmentOptions = {
|
||||
signal?: AbortSignal;
|
||||
@@ -37,9 +34,12 @@ export class AttachmentMaterializer {
|
||||
constructor(private readonly config: Config) {}
|
||||
|
||||
private buildFetchOptions(url: URL, trustedHostSuffixes: string[]) {
|
||||
const baseOptions = { timeoutMs: 15_000, maxRedirects: 3 } as const;
|
||||
const baseOptions = {
|
||||
timeoutMs: 15_000,
|
||||
allowPrivateTargetOrigin: false,
|
||||
};
|
||||
if (!env.prod) {
|
||||
return { ...baseOptions, allowPrivateOrigins: new Set([url.origin]) };
|
||||
return { ...baseOptions, allowPrivateTargetOrigin: true };
|
||||
}
|
||||
|
||||
const trustedOrigins = new Set<string>();
|
||||
@@ -80,7 +80,7 @@ export class AttachmentMaterializer {
|
||||
suffix => hostname === suffix || hostname.endsWith(`.${suffix}`)
|
||||
);
|
||||
if (trustedOrigins.has(url.origin) || trustedByHost) {
|
||||
return { ...baseOptions, allowPrivateOrigins: new Set([url.origin]) };
|
||||
return { ...baseOptions, allowPrivateTargetOrigin: true };
|
||||
}
|
||||
|
||||
return baseOptions;
|
||||
@@ -91,30 +91,22 @@ export class AttachmentMaterializer {
|
||||
options: FetchRemoteAttachmentOptions
|
||||
) {
|
||||
const parsed = resolveAttachmentFetchUrl(url);
|
||||
const response = await safeFetch(
|
||||
parsed,
|
||||
{ method: 'GET', signal: options.signal },
|
||||
this.buildFetchOptions(parsed, options.trustedHostSuffixes ?? [])
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to fetch attachment: ${response.status} ${response.statusText}`
|
||||
);
|
||||
}
|
||||
|
||||
const buffer = await readResponseBufferWithLimit(
|
||||
response,
|
||||
options.maxBytes
|
||||
);
|
||||
const headerMimeType = normalizeMimeType(
|
||||
response.headers.get('content-type') || ''
|
||||
);
|
||||
options.signal?.throwIfAborted();
|
||||
const response = await fetchRemoteAttachment({
|
||||
url: parsed.toString(),
|
||||
...this.buildFetchOptions(parsed, options.trustedHostSuffixes ?? []),
|
||||
maxBytes: options.maxBytes,
|
||||
});
|
||||
options.signal?.throwIfAborted();
|
||||
|
||||
return {
|
||||
data: buffer.toString('base64'),
|
||||
data: response.body.toString('base64'),
|
||||
mimeType: options.detectMimeType
|
||||
? options.detectMimeType(buffer, headerMimeType)
|
||||
: headerMimeType,
|
||||
? options.detectMimeType(
|
||||
response.body,
|
||||
normalizeMimeType(response.mimeType)
|
||||
)
|
||||
: normalizeMimeType(response.mimeType),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import {
|
||||
BlobQuotaExceeded,
|
||||
CallMetric,
|
||||
Config,
|
||||
fetchBuffer,
|
||||
type FileUpload,
|
||||
OneMB,
|
||||
OnEvent,
|
||||
@@ -17,6 +16,7 @@ import {
|
||||
URLHelper,
|
||||
} from '../../base';
|
||||
import { QuotaService } from '../../core/quota';
|
||||
import { fetchRemoteAttachment } from '../../native';
|
||||
|
||||
const REMOTE_BLOB_MAX_BYTES = 20 * OneMB;
|
||||
|
||||
@@ -93,12 +93,15 @@ export class CopilotStorage {
|
||||
|
||||
@CallMetric('ai', 'blob_proxy_remote_url')
|
||||
async handleRemoteLink(userId: string, workspaceId: string, link: string) {
|
||||
const { buffer, type } = await fetchBuffer(
|
||||
link,
|
||||
REMOTE_BLOB_MAX_BYTES,
|
||||
'image/'
|
||||
);
|
||||
const { body, mimeType } = await fetchRemoteAttachment({
|
||||
url: link,
|
||||
maxBytes: REMOTE_BLOB_MAX_BYTES,
|
||||
expectedContentTypePrefix: 'image/',
|
||||
maxImageHeight: 4096,
|
||||
maxImageWidth: 4096,
|
||||
});
|
||||
const buffer = Buffer.from(body);
|
||||
const filename = createHash('sha256').update(buffer).digest('base64url');
|
||||
return this.put(userId, workspaceId, filename, buffer, type);
|
||||
return this.put(userId, workspaceId, filename, buffer, mimeType);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,15 +17,16 @@ import {
|
||||
applyAttachHeaders,
|
||||
BadRequest,
|
||||
Cache,
|
||||
readResponseBufferWithLimit,
|
||||
ResponseTooLargeError,
|
||||
safeFetch,
|
||||
SsrfBlockedError,
|
||||
type SSRFBlockReason,
|
||||
Throttle,
|
||||
URLHelper,
|
||||
UseNamedGuard,
|
||||
} from '../../base';
|
||||
import { Public } from '../../core/auth';
|
||||
import { inspectImageForProxy } from '../../native';
|
||||
import { WorkerService } from './service';
|
||||
import type { LinkPreviewRequest, LinkPreviewResponse } from './types';
|
||||
import {
|
||||
@@ -64,6 +65,7 @@ function toBadRequestReason(reason: SSRFBlockReason) {
|
||||
|
||||
@Public()
|
||||
@UseNamedGuard('selfhost')
|
||||
@Throttle('default', { limit: 60, ttl: 60_000 })
|
||||
@Controller('/api/worker')
|
||||
export class WorkerController {
|
||||
private readonly logger = new Logger(WorkerController.name);
|
||||
@@ -133,7 +135,8 @@ export class WorkerController {
|
||||
...(origin ? { Vary: 'Origin' } : {}),
|
||||
'Access-Control-Allow-Methods': 'GET',
|
||||
});
|
||||
applyAttachHeaders(resp, { buffer });
|
||||
const image = this.inspectProxyImage(buffer, imageURL);
|
||||
applyAttachHeaders(resp, { buffer, contentType: image.mimeType });
|
||||
const contentType = resp.getHeader('Content-Type') as string | undefined;
|
||||
if (contentType?.startsWith('image/')) {
|
||||
return resp.status(200).send(buffer);
|
||||
@@ -147,7 +150,11 @@ export class WorkerController {
|
||||
response = await safeFetch(
|
||||
targetURL.toString(),
|
||||
{ method: 'GET', headers: cloneHeader(req.headers) },
|
||||
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
|
||||
{
|
||||
timeoutMs: FETCH_TIMEOUT_MS,
|
||||
maxRedirects: MAX_REDIRECTS,
|
||||
maxBytes: IMAGE_PROXY_MAX_BYTES,
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof SsrfBlockedError) {
|
||||
@@ -155,7 +162,6 @@ export class WorkerController {
|
||||
this.logger.warn('Blocked image proxy target', {
|
||||
url: imageURL,
|
||||
reason,
|
||||
context: (error as any).context,
|
||||
});
|
||||
throw new BadRequest(toBadRequestReason(reason ?? 'invalid_url'));
|
||||
}
|
||||
@@ -175,23 +181,8 @@ export class WorkerController {
|
||||
throw new BadRequest('Failed to fetch image');
|
||||
}
|
||||
if (response.ok) {
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = await readResponseBufferWithLimit(
|
||||
response,
|
||||
IMAGE_PROXY_MAX_BYTES
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof ResponseTooLargeError) {
|
||||
this.logger.warn('Image proxy response too large', {
|
||||
url: imageURL,
|
||||
limitBytes: error.data?.limitBytes,
|
||||
receivedBytes: error.data?.receivedBytes,
|
||||
});
|
||||
throw new BadRequest('Response too large');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const image = this.inspectProxyImage(buffer, imageURL);
|
||||
await this.cache.set(cachedUrl, buffer.toString('base64'), {
|
||||
ttl: CACHE_TTL,
|
||||
});
|
||||
@@ -204,7 +195,7 @@ export class WorkerController {
|
||||
if (contentDisposition) {
|
||||
resp.setHeader('Content-Disposition', contentDisposition);
|
||||
}
|
||||
applyAttachHeaders(resp, { buffer });
|
||||
applyAttachHeaders(resp, { buffer, contentType: image.mimeType });
|
||||
const contentType = resp.getHeader('Content-Type') as string | undefined;
|
||||
if (contentType?.startsWith('image/')) {
|
||||
return resp.status(200).send(buffer);
|
||||
@@ -227,6 +218,18 @@ export class WorkerController {
|
||||
}
|
||||
}
|
||||
|
||||
private inspectProxyImage(buffer: Buffer, url: string) {
|
||||
try {
|
||||
return inspectImageForProxy(buffer);
|
||||
} catch (error) {
|
||||
this.logger.warn('Image proxy rejected invalid image', {
|
||||
url,
|
||||
error,
|
||||
});
|
||||
throw new BadRequest('Invalid image');
|
||||
}
|
||||
}
|
||||
|
||||
@Options('/link-preview')
|
||||
linkPreviewOption(
|
||||
@Req() request: ExpressRequest,
|
||||
@@ -291,7 +294,11 @@ export class WorkerController {
|
||||
const response = await safeFetch(
|
||||
targetURL.toString(),
|
||||
{ method, headers: cloneHeader(request.headers) },
|
||||
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
|
||||
{
|
||||
timeoutMs: FETCH_TIMEOUT_MS,
|
||||
maxRedirects: MAX_REDIRECTS,
|
||||
maxBytes: LINK_PREVIEW_MAX_BYTES,
|
||||
}
|
||||
);
|
||||
this.logger.debug('Fetched URL', {
|
||||
origin,
|
||||
@@ -318,10 +325,7 @@ export class WorkerController {
|
||||
};
|
||||
|
||||
if (response.body) {
|
||||
const body = await readResponseBufferWithLimit(
|
||||
response,
|
||||
LINK_PREVIEW_MAX_BYTES
|
||||
);
|
||||
const body = Buffer.from(await response.arrayBuffer());
|
||||
const limitedResponse = new Response(body, response);
|
||||
const resp = await decodeWithCharset(limitedResponse, res);
|
||||
|
||||
@@ -402,7 +406,11 @@ export class WorkerController {
|
||||
const faviconResponse = await safeFetch(
|
||||
faviconUrl.toString(),
|
||||
{ method: 'HEAD' },
|
||||
{ timeoutMs: FETCH_TIMEOUT_MS, maxRedirects: MAX_REDIRECTS }
|
||||
{
|
||||
timeoutMs: FETCH_TIMEOUT_MS,
|
||||
maxRedirects: MAX_REDIRECTS,
|
||||
maxBytes: 0,
|
||||
}
|
||||
);
|
||||
if (faviconResponse.ok) {
|
||||
appendUrl(faviconUrl.toString(), res.favicons);
|
||||
@@ -433,7 +441,6 @@ export class WorkerController {
|
||||
origin,
|
||||
url: requestBody?.url,
|
||||
reason,
|
||||
context: (error as any).context,
|
||||
});
|
||||
throw new BadRequest(toBadRequestReason(reason ?? 'invalid_url'));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user