diff --git a/oxlint.json b/oxlint.json index 27117edfc3..91da9d2d11 100644 --- a/oxlint.json +++ b/oxlint.json @@ -30,6 +30,7 @@ "tools/cli/src/webpack/error-handler.js", "packages/backend/native/index.d.ts", "packages/backend/server/src/__tests__/__snapshots__", + "packages/common/native/fixtures/**", "packages/frontend/native/index.d.ts", "packages/frontend/native/index.js", "packages/frontend/graphql/src/graphql/index.ts", diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index 3a005ef291..bdf4630d2e 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -70,6 +70,7 @@ "graphql-scalars": "^1.24.0", "graphql-upload": "^17.0.0", "html-validate": "^9.0.0", + "htmlrewriter": "^0.0.12", "ioredis": "^5.4.1", "is-mobile": "^5.0.0", "keyv": "^5.2.2", @@ -92,6 +93,7 @@ "ses": "^1.10.0", "socket.io": "^4.8.1", "stripe": "^17.4.0", + "tldts": "^6.1.68", "ts-node": "^10.9.2", "typescript": "^5.7.2", "winston": "^3.17.0", diff --git a/packages/backend/server/src/__tests__/__snapshots__/worker.e2e.ts.md b/packages/backend/server/src/__tests__/__snapshots__/worker.e2e.ts.md new file mode 100644 index 0000000000..bd6979b853 --- /dev/null +++ b/packages/backend/server/src/__tests__/__snapshots__/worker.e2e.ts.md @@ -0,0 +1,78 @@ +# Snapshot report for `src/__tests__/worker.e2e.ts` + +The actual snapshot is saved in `worker.e2e.ts.snap`. + +Generated by [AVA](https://avajs.dev). + +## should proxy image + +> Snapshot 1 + + {} + +> Snapshot 2 + + { + code: 'Bad Request', + message: 'Missing "url" parameter', + name: 'BAD_REQUEST', + status: 400, + type: 'BAD_REQUEST', + } + +> Snapshot 3 + + { + code: 'Bad Request', + message: 'Invalid header', + name: 'BAD_REQUEST', + status: 400, + type: 'BAD_REQUEST', + } + +> Snapshot 4 + + Buffer @Uint8Array [ + 66616b65 20696d61 6765 + ] + +## should preview link + +> Snapshot 1 + + {} + +> Snapshot 2 + + { + code: 'Bad Request', + message: 'Invalid URL', + name: 'BAD_REQUEST', + status: 400, + type: 'BAD_REQUEST', + } + +> Snapshot 3 + + { + code: 'Bad Request', + message: 'Invalid URL', + name: 'BAD_REQUEST', + status: 400, + type: 'BAD_REQUEST', + } + +> Snapshot 4 + + { + description: 'Test Description', + favicons: [ + 'http://localhost:3010/api/worker/image-proxy?url=https%3A%2F%2Fexample.com%2Ffavicon.ico', + ], + images: [ + 'http://localhost:3010/api/worker/image-proxy?url=https%3A%2F%2Fexample.com%2Fimage.png', + ], + title: 'Test Title', + url: 'http://example.com/page', + videos: [], + } diff --git a/packages/backend/server/src/__tests__/__snapshots__/worker.e2e.ts.snap b/packages/backend/server/src/__tests__/__snapshots__/worker.e2e.ts.snap new file mode 100644 index 0000000000..7d2ba02596 Binary files /dev/null and b/packages/backend/server/src/__tests__/__snapshots__/worker.e2e.ts.snap differ diff --git a/packages/backend/server/src/__tests__/utils/testing-app.ts b/packages/backend/server/src/__tests__/utils/testing-app.ts index fe2e0bca16..2271c547df 100644 --- a/packages/backend/server/src/__tests__/utils/testing-app.ts +++ b/packages/backend/server/src/__tests__/utils/testing-app.ts @@ -95,7 +95,7 @@ export class TestingApp extends ApplyType() { } request( - method: 'get' | 'post' | 'put' | 'delete' | 'patch', + method: 'options' | 'get' | 'post' | 'put' | 'delete' | 'patch', path: string ): supertest.Test { return supertest(this.getHttpServer()) @@ -106,6 +106,10 @@ export class TestingApp extends ApplyType() { ]); } + OPTIONS(path: string): supertest.Test { + return this.request('options', path); + } + GET(path: string): supertest.Test { return this.request('get', path); } diff --git a/packages/backend/server/src/__tests__/worker.e2e.ts b/packages/backend/server/src/__tests__/worker.e2e.ts new file mode 100644 index 0000000000..f03a6c2008 --- /dev/null +++ b/packages/backend/server/src/__tests__/worker.e2e.ts @@ -0,0 +1,174 @@ +import type { ExecutionContext, TestFn } from 'ava'; +import ava from 'ava'; +import Sinon from 'sinon'; +import type { Response } from 'supertest'; + +import { WorkerModule } from '../plugins/worker'; +import { createTestingApp, TestingApp } from './utils'; + +type TestContext = { + app: TestingApp; +}; + +const test = ava as TestFn; + +test.before(async t => { + const app = await createTestingApp({ + imports: [WorkerModule], + }); + + t.context.app = app; +}); + +test.after.always(async t => { + await t.context.app.close(); +}); + +const assertAndSnapshotRaw = async ( + t: ExecutionContext, + route: string, + message: string, + options?: { + status?: number; + origin?: string; + method?: 'GET' | 'OPTIONS' | 'POST'; + body?: any; + checker?: (res: Response) => any; + } +) => { + const { + status = 200, + origin = 'http://localhost', + method = 'GET', + checker = () => {}, + } = options || {}; + const { app } = t.context; + const res = app[method](route) + .set('Origin', origin) + .send(options?.body) + .expect(status) + .expect(checker); + t.notThrowsAsync(res, message); + t.snapshot((await res).body); +}; + +test('should proxy image', async t => { + const assertAndSnapshot = assertAndSnapshotRaw.bind(null, t); + + await assertAndSnapshot( + '/api/worker/image-proxy', + 'should return proper CORS headers on OPTIONS request', + { + status: 204, + method: 'OPTIONS', + checker: (res: Response) => { + if (!res.headers['access-control-allow-methods']) { + throw new Error('Missing CORS headers'); + } + }, + } + ); + + { + await assertAndSnapshot( + '/api/worker/image-proxy', + 'should return 400 if "url" query parameter is missing', + { status: 400 } + ); + } + + { + await assertAndSnapshot( + '/api/worker/image-proxy?url=http://example.com/image.png', + 'should return 400 for invalid origin header', + { status: 400, origin: 'http://invalid.com' } + ); + } + + { + const fakeBuffer = Buffer.from('fake image'); + const fakeResponse = { + ok: true, + headers: { + get: (header: string) => { + if (header.toLowerCase() === 'content-type') return 'image/png'; + if (header.toLowerCase() === 'content-disposition') return 'inline'; + return null; + }, + }, + arrayBuffer: async () => fakeBuffer, + } as any; + + const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeResponse); + + await assertAndSnapshot( + '/api/worker/image-proxy?url=http://example.com/image.png', + 'should return image buffer' + ); + + fetchSpy.restore(); + } +}); + +test('should preview link', async t => { + const assertAndSnapshot = assertAndSnapshotRaw.bind(null, t); + + await assertAndSnapshot( + '/api/worker/link-preview', + 'should return proper CORS headers on OPTIONS request', + { + status: 204, + method: 'OPTIONS', + checker: (res: Response) => { + if (!res.headers['access-control-allow-methods']) { + throw new Error('Missing CORS headers'); + } + }, + } + ); + + await assertAndSnapshot( + '/api/worker/link-preview', + 'should return 400 if request body is invalid', + { status: 400, method: 'POST' } + ); + + await assertAndSnapshot( + '/api/worker/link-preview', + 'should return 400 if provided URL is from the same origin', + { status: 400, method: 'POST', body: { url: 'http://localhost/somepage' } } + ); + + { + const fakeHTML = new Response(` + + + + + + + + Fallback Title + + + `); + + Object.defineProperty(fakeHTML, 'url', { + value: 'http://example.com/page', + }); + + const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeHTML); + + await assertAndSnapshot( + '/api/worker/link-preview', + 'should process a valid external URL and return link preview data', + { + status: 200, + method: 'POST', + body: { url: 'http://external.com/page' }, + } + ); + + fetchSpy.restore(); + } +}); diff --git a/packages/backend/server/src/base/error/def.ts b/packages/backend/server/src/base/error/def.ts index e147188a3e..52aafcec71 100644 --- a/packages/backend/server/src/base/error/def.ts +++ b/packages/backend/server/src/base/error/def.ts @@ -247,6 +247,10 @@ export const USER_FRIENDLY_ERRORS = { type: 'resource_not_found', message: 'Resource not found.', }, + bad_request: { + type: 'bad_request', + message: 'Bad request.', + }, // Input errors query_too_long: { diff --git a/packages/backend/server/src/base/error/errors.gen.ts b/packages/backend/server/src/base/error/errors.gen.ts index fca1390412..6196c06491 100644 --- a/packages/backend/server/src/base/error/errors.gen.ts +++ b/packages/backend/server/src/base/error/errors.gen.ts @@ -21,6 +21,12 @@ export class NotFound extends UserFriendlyError { super('resource_not_found', 'not_found', message); } } + +export class BadRequest extends UserFriendlyError { + constructor(message?: string) { + super('bad_request', 'bad_request', message); + } +} @ObjectType() class QueryTooLongDataType { @Field() max!: number @@ -770,6 +776,7 @@ export enum ErrorNames { INTERNAL_SERVER_ERROR, TOO_MANY_REQUEST, NOT_FOUND, + BAD_REQUEST, QUERY_TOO_LONG, USER_NOT_FOUND, USER_AVATAR_NOT_FOUND, diff --git a/packages/backend/server/src/config/affine.self.ts b/packages/backend/server/src/config/affine.self.ts index 3069ef60c3..af82f3957e 100644 --- a/packages/backend/server/src/config/affine.self.ts +++ b/packages/backend/server/src/config/affine.self.ts @@ -89,4 +89,7 @@ if (AFFiNE.deploy) { }; AFFiNE.use('gcloud'); +} else { + // only enable dev mode + AFFiNE.use('worker'); } diff --git a/packages/backend/server/src/config/affine.ts b/packages/backend/server/src/config/affine.ts index c948d6861a..fc34e43102 100644 --- a/packages/backend/server/src/config/affine.ts +++ b/packages/backend/server/src/config/affine.ts @@ -161,3 +161,8 @@ AFFiNE.server.port = 3010; // bucket: 'copilot', // } // }) +// +// /* AFFiNE Link Preview & Image Proxy API */ +// AFFiNE.use('worker', { +// allowedOrigin: ['example.com'], +// }); diff --git a/packages/backend/server/src/plugins/index.ts b/packages/backend/server/src/plugins/index.ts index e0453a6b81..57fca3a307 100644 --- a/packages/backend/server/src/plugins/index.ts +++ b/packages/backend/server/src/plugins/index.ts @@ -4,6 +4,7 @@ import './gcloud'; import './oauth'; import './payment'; import './storage'; +import './worker'; export { enablePlugin, diff --git a/packages/backend/server/src/plugins/worker/config.ts b/packages/backend/server/src/plugins/worker/config.ts new file mode 100644 index 0000000000..c37c07946a --- /dev/null +++ b/packages/backend/server/src/plugins/worker/config.ts @@ -0,0 +1,15 @@ +import { defineStartupConfig, ModuleConfig } from '../../base/config'; + +export interface WorkerStartupConfigurations { + allowedOrigin: string[]; +} + +declare module '../config' { + interface PluginsConfig { + worker: ModuleConfig; + } +} + +defineStartupConfig('plugins.worker', { + allowedOrigin: ['localhost', '127.0.0.1'], +}); diff --git a/packages/backend/server/src/plugins/worker/controller.ts b/packages/backend/server/src/plugins/worker/controller.ts new file mode 100644 index 0000000000..636a7d2f4e --- /dev/null +++ b/packages/backend/server/src/plugins/worker/controller.ts @@ -0,0 +1,279 @@ +import { + Controller, + Get, + Logger, + Options, + Post, + Req, + Res, +} from '@nestjs/common'; +import type { Request, Response } from 'express'; +import { HTMLRewriter } from 'htmlrewriter'; + +import { BadRequest, Cache, Config, URLHelper } from '../../base'; +import { Public } from '../../core/auth'; +import type { LinkPreviewRequest, LinkPreviewResponse } from './types'; +import { + appendUrl, + cloneHeader, + fixUrl, + getCorsHeaders, + isOriginAllowed, + isRefererAllowed, + OriginRules, + parseJson, + reduceUrls, +} from './utils'; + +@Public() +@Controller('/api/worker') +export class WorkerController { + private readonly logger = new Logger(WorkerController.name); + private readonly allowedOrigin: OriginRules; + + constructor( + config: Config, + private readonly cache: Cache, + private readonly url: URLHelper + ) { + this.allowedOrigin = [ + ...config.plugins.worker.allowedOrigin + .map(u => fixUrl(u)?.origin as string) + .filter(v => !!v), + url.origin, + ]; + } + + @Get('/image-proxy') + async imageProxy(@Req() req: Request, @Res() resp: Response) { + const origin = req.headers.origin ?? ''; + const referer = req.headers.referer; + if ( + (origin && !isOriginAllowed(origin, this.allowedOrigin)) || + (referer && !isRefererAllowed(referer, this.allowedOrigin)) + ) { + this.logger.error('Invalid Origin', 'ERROR', { origin, referer }); + throw new BadRequest('Invalid header'); + } + const url = new URL(req.url, this.url.baseUrl); + const imageURL = url.searchParams.get('url'); + if (!imageURL) { + throw new BadRequest('Missing "url" parameter'); + } + + const targetURL = fixUrl(imageURL); + if (!targetURL) { + this.logger.error(`Invalid URL: ${url}`); + throw new BadRequest(`Invalid URL`); + } + + const response = await fetch( + new Request(targetURL.toString(), { + method: 'GET', + headers: cloneHeader(req.headers), + }) + ); + if (response.ok) { + const contentType = response.headers.get('Content-Type'); + const contentDisposition = response.headers.get('Content-Disposition'); + if (contentType?.startsWith('image/')) { + return resp + .status(200) + .header({ + 'Access-Control-Allow-Origin': origin ?? 'null', + Vary: 'Origin', + 'Access-Control-Allow-Methods': 'GET', + 'Content-Type': contentType, + 'Content-Disposition': contentDisposition, + }) + .send(Buffer.from(await response.arrayBuffer())); + } else { + throw new BadRequest('Invalid content type'); + } + } else { + this.logger.error('Failed to fetch image', { + origin, + url: imageURL, + status: resp.status, + }); + throw new BadRequest('Failed to fetch image'); + } + } + + @Options('/link-preview') + linkPreviewOption(@Req() request: Request, @Res() resp: Response) { + const origin = request.headers.origin; + return resp + .status(200) + .header({ + ...getCorsHeaders(origin), + 'Access-Control-Allow-Methods': 'POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type', + }) + .send(); + } + + @Post('/link-preview') + async linkPreview( + @Req() request: Request, + @Res() resp: Response + ): Promise { + const origin = request.headers.origin; + const referer = request.headers.referer; + if ( + (origin && !isOriginAllowed(origin, this.allowedOrigin)) || + (referer && !isRefererAllowed(referer, this.allowedOrigin)) + ) { + this.logger.error('Invalid Origin', { origin, referer }); + throw new BadRequest('Invalid header'); + } + + this.logger.debug('Received request', { origin, method: request.method }); + + const targetBody = parseJson(request.body); + const targetURL = fixUrl(targetBody?.url); + // not allow same site preview + if (!targetURL || isOriginAllowed(targetURL.origin, this.allowedOrigin)) { + this.logger.error('Invalid URL', { origin, url: targetBody?.url }); + throw new BadRequest('Invalid URL'); + } + + this.logger.debug('Processing request', { origin, url: targetURL }); + + try { + const cachedResponse = await this.cache.get(targetURL.toString()); + if (cachedResponse) { + return resp + .status(200) + .header({ + 'content-type': 'application/json;charset=UTF-8', + ...getCorsHeaders(origin), + }) + .send(cachedResponse); + } + + const response = await fetch(targetURL, { + headers: cloneHeader(request.headers), + }); + this.logger.error('Fetched URL', { + origin, + url: targetURL, + status: response.status, + }); + + const res: LinkPreviewResponse = { + url: response.url, + images: [], + videos: [], + favicons: [], + }; + const baseUrl = new URL(request.url, this.url.baseUrl).toString(); + + if (response.body) { + const rewriter = new HTMLRewriter() + .on('meta', { + element(element) { + const property = + element.getAttribute('property') ?? + element.getAttribute('name'); + const content = element.getAttribute('content'); + if (property && content) { + switch (property.toLowerCase()) { + case 'og:title': + res.title = content; + break; + case 'og:site_name': + res.siteName = content; + break; + case 'og:description': + res.description = content; + break; + case 'og:image': + appendUrl(content, res.images); + break; + case 'og:video': + appendUrl(content, res.videos); + break; + case 'og:type': + res.mediaType = content; + break; + case 'description': + if (!res.description) { + res.description = content; + } + } + } + }, + }) + .on('link', { + element(element) { + if (element.getAttribute('rel')?.toLowerCase().includes('icon')) { + appendUrl(element.getAttribute('href'), res.favicons); + } + }, + }) + .on('title', { + text(text) { + if (!res.title) { + res.title = text.text; + } + }, + }) + .on('img', { + element(element) { + appendUrl(element.getAttribute('src'), res.images); + }, + }) + .on('video', { + element(element) { + appendUrl(element.getAttribute('src'), res.videos); + }, + }); + + await rewriter.transform(response).text(); + + res.images = await reduceUrls(baseUrl, res.images); + + this.logger.error('Processed response with HTMLRewriter', { + origin, + url: response.url, + }); + } + + // fix favicon + { + // head default path of favicon + const faviconUrl = new URL('/favicon.ico', response.url); + const faviconResponse = await fetch(faviconUrl, { method: 'HEAD' }); + if (faviconResponse.ok) { + appendUrl(faviconUrl.toString(), res.favicons); + } + + res.favicons = await reduceUrls(baseUrl, res.favicons); + } + + const json = JSON.stringify(res); + this.logger.debug('Sending response', { + origin, + url: res.url, + responseSize: json.length, + }); + + await this.cache.set(targetURL.toString(), res); + return resp + .status(200) + .header({ + 'content-type': 'application/json;charset=UTF-8', + ...getCorsHeaders(origin), + }) + .send(json); + } catch (error) { + this.logger.error('Error fetching URL', { + origin, + url: targetURL, + error, + }); + throw new BadRequest('Error fetching URL'); + } + } +} diff --git a/packages/backend/server/src/plugins/worker/index.ts b/packages/backend/server/src/plugins/worker/index.ts new file mode 100644 index 0000000000..1878539675 --- /dev/null +++ b/packages/backend/server/src/plugins/worker/index.ts @@ -0,0 +1,11 @@ +import './config'; + +import { Plugin } from '../registry'; +import { WorkerController } from './controller'; + +@Plugin({ + name: 'worker', + controllers: [WorkerController], + if: config => config.isSelfhosted || config.node.dev || config.node.test, +}) +export class WorkerModule {} diff --git a/packages/backend/server/src/plugins/worker/types.ts b/packages/backend/server/src/plugins/worker/types.ts new file mode 100644 index 0000000000..27765946c9 --- /dev/null +++ b/packages/backend/server/src/plugins/worker/types.ts @@ -0,0 +1,16 @@ +export type LinkPreviewRequest = { + url: string; +}; + +export type LinkPreviewResponse = { + url: string; + title?: string; + siteName?: string; + description?: string; + images?: string[]; + mediaType?: string; + contentType?: string; + charset?: string; + videos?: string[]; + favicons?: string[]; +}; diff --git a/packages/backend/server/src/plugins/worker/utils/headers.ts b/packages/backend/server/src/plugins/worker/utils/headers.ts new file mode 100644 index 0000000000..caedad54c5 --- /dev/null +++ b/packages/backend/server/src/plugins/worker/utils/headers.ts @@ -0,0 +1,63 @@ +import { IncomingHttpHeaders } from 'node:http'; + +export type OriginRule = string | RegExp | ((origin: string) => boolean); +export type OriginRules = OriginRule | OriginRule[]; + +function isString(s: OriginRule): s is string { + return typeof s === 'string' || s instanceof String; +} + +export function isOriginAllowed(origin: string, allowedOrigin: OriginRules) { + if (Array.isArray(allowedOrigin)) { + for (const allowed of allowedOrigin) { + if (isOriginAllowed(origin, allowed)) { + return true; + } + } + return false; + } else if (isString(allowedOrigin)) { + return origin === allowedOrigin; + } else if (allowedOrigin instanceof RegExp) { + return allowedOrigin.test(origin); + } + return allowedOrigin(origin); +} + +export function isRefererAllowed(referer: string, allowedOrigin: OriginRules) { + try { + const origin = new URL(referer).origin; + return isOriginAllowed(origin, allowedOrigin); + } catch { + return false; + } +} + +const headerFilters = [/^Sec-/i, /^Accept/i, /^User-Agent$/i]; + +export function cloneHeader(source: IncomingHttpHeaders) { + const headers: Record = {}; + + Object.entries(source).forEach(([key, value]) => { + if (headerFilters.some(filter => filter.test(key))) { + if (Array.isArray(value)) { + headers[key] = value.join(','); + } else if (value) { + headers[key] = value; + } + } + }); + + return headers; +} + +export function getCorsHeaders(origin?: string | null): { + [key: string]: string; +} { + if (origin) { + return { + 'Access-Control-Allow-Origin': origin, + }; + } else { + return {}; + } +} diff --git a/packages/backend/server/src/plugins/worker/utils/index.ts b/packages/backend/server/src/plugins/worker/utils/index.ts new file mode 100644 index 0000000000..bcef68ae46 --- /dev/null +++ b/packages/backend/server/src/plugins/worker/utils/index.ts @@ -0,0 +1,12 @@ +export * from './headers'; +export * from './proxy'; +export * from './url'; + +export function parseJson(data: string): T | null { + try { + if (data && typeof data === 'object') return data; + return JSON.parse(data); + } catch { + return null; + } +} diff --git a/packages/backend/server/src/plugins/worker/utils/proxy.ts b/packages/backend/server/src/plugins/worker/utils/proxy.ts new file mode 100644 index 0000000000..f52902dc0b --- /dev/null +++ b/packages/backend/server/src/plugins/worker/utils/proxy.ts @@ -0,0 +1,54 @@ +const IMAGE_PROXY = '/api/worker/image-proxy'; + +const httpsDomain = new Set(); + +async function checkHttpsSupport(url: URL): Promise { + const httpsUrl = new URL(url.toString()); + httpsUrl.protocol = 'https:'; + try { + const response = await fetch(httpsUrl, { + method: 'HEAD', + redirect: 'manual', + }); + + if (response.ok || (response.status >= 400 && response.status < 600)) { + return true; + } + } catch {} + return false; +} + +async function fixProtocol(url: string): Promise { + const targetUrl = new URL(url); + if (targetUrl.protocol !== 'http:') { + return targetUrl; + } else if (httpsDomain.has(targetUrl.hostname)) { + targetUrl.protocol = 'https:'; + return targetUrl; + } else if (await checkHttpsSupport(targetUrl)) { + httpsDomain.add(targetUrl.hostname); + targetUrl.protocol = 'https:'; + return targetUrl; + } + return targetUrl; +} + +export function imageProxyBuilder( + url: string +): (url: string) => Promise { + try { + const proxy = new URL(url); + proxy.pathname = IMAGE_PROXY; + + return async url => { + try { + const targetUrl = await fixProtocol(url); + proxy.searchParams.set('url', targetUrl.toString()); + return proxy.toString(); + } catch {} + return; + }; + } catch { + return async url => url.toString(); + } +} diff --git a/packages/backend/server/src/plugins/worker/utils/url.ts b/packages/backend/server/src/plugins/worker/utils/url.ts new file mode 100644 index 0000000000..3253cc8779 --- /dev/null +++ b/packages/backend/server/src/plugins/worker/utils/url.ts @@ -0,0 +1,54 @@ +import { getDomain, getSubdomain } from 'tldts'; + +import { imageProxyBuilder } from './proxy'; + +const localhost = new Set(['localhost', '127.0.0.1']); + +export function fixUrl(url?: string): URL | null { + if (typeof url !== 'string') { + return null; + } + + let fullUrl = url; + + // don't require // prefix, URL can handle protocol:domain + if (!url.startsWith('http:') && !url.startsWith('https:')) { + fullUrl = 'http://' + url; + } + + try { + const parsed = new URL(fullUrl); + + const subDomain = getSubdomain(url); + const mainDomain = getDomain(url); + const fullDomain = subDomain ? `${subDomain}.${mainDomain}` : mainDomain; + + if ( + ['http:', 'https:'].includes(parsed.protocol) && + // check hostname is a valid domain + (fullDomain === parsed.hostname || localhost.has(parsed.hostname)) + ) { + return parsed; + } + } catch {} + + return null; +} + +export function appendUrl(url: string | null, array?: string[]) { + if (url) { + const fixedUrl = fixUrl(url); + if (fixedUrl) { + array?.push(fixedUrl.toString()); + } + } +} + +export async function reduceUrls(baseUrl: string, urls?: string[]) { + if (urls && urls.length > 0) { + const imageProxy = imageProxyBuilder(baseUrl); + const newUrls = await Promise.all(urls.map(imageProxy)); + return newUrls.filter((x): x is string => !!x); + } + return []; +} diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index 3a213c47e0..9456710ab1 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -319,6 +319,7 @@ enum ErrorNames { ACTION_FORBIDDEN_ON_NON_TEAM_WORKSPACE ALREADY_IN_SPACE AUTHENTICATION_REQUIRED + BAD_REQUEST BLOB_NOT_FOUND BLOB_QUOTA_EXCEEDED CANNOT_DELETE_ALL_ADMIN_ACCOUNT diff --git a/packages/frontend/graphql/src/schema.ts b/packages/frontend/graphql/src/schema.ts index e85572927e..27b5229418 100644 --- a/packages/frontend/graphql/src/schema.ts +++ b/packages/frontend/graphql/src/schema.ts @@ -425,6 +425,7 @@ export enum ErrorNames { ACTION_FORBIDDEN_ON_NON_TEAM_WORKSPACE = 'ACTION_FORBIDDEN_ON_NON_TEAM_WORKSPACE', ALREADY_IN_SPACE = 'ALREADY_IN_SPACE', AUTHENTICATION_REQUIRED = 'AUTHENTICATION_REQUIRED', + BAD_REQUEST = 'BAD_REQUEST', BLOB_NOT_FOUND = 'BLOB_NOT_FOUND', BLOB_QUOTA_EXCEEDED = 'BLOB_QUOTA_EXCEEDED', CANNOT_DELETE_ALL_ADMIN_ACCOUNT = 'CANNOT_DELETE_ALL_ADMIN_ACCOUNT', diff --git a/tools/cli/src/webpack/index.ts b/tools/cli/src/webpack/index.ts index 50aea19622..56c5c4180d 100644 --- a/tools/cli/src/webpack/index.ts +++ b/tools/cli/src/webpack/index.ts @@ -361,12 +361,6 @@ export function createWebpackConfig( }, ], proxy: [ - { - context: '/api/worker/', - target: 'https://affine.fail', - changeOrigin: true, - secure: false, - }, { context: '/api', target: 'http://localhost:3010' }, { context: '/socket.io', target: 'http://localhost:3010', ws: true }, { context: '/graphql', target: 'http://localhost:3010' }, diff --git a/yarn.lock b/yarn.lock index 492572e3b7..a2672a38ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -839,6 +839,7 @@ __metadata: graphql-scalars: "npm:^1.24.0" graphql-upload: "npm:^17.0.0" html-validate: "npm:^9.0.0" + htmlrewriter: "npm:^0.0.12" ioredis: "npm:^5.4.1" is-mobile: "npm:^5.0.0" keyv: "npm:^5.2.2" @@ -865,6 +866,7 @@ __metadata: socket.io: "npm:^4.8.1" stripe: "npm:^17.4.0" supertest: "npm:^7.0.0" + tldts: "npm:^6.1.68" ts-node: "npm:^10.9.2" typescript: "npm:^5.7.2" winston: "npm:^3.17.0" @@ -23193,6 +23195,13 @@ __metadata: languageName: node linkType: hard +"htmlrewriter@npm:^0.0.12": + version: 0.0.12 + resolution: "htmlrewriter@npm:0.0.12" + checksum: 10/8e1b39c52957c42fbf989f805504c92821fddcd7d3b42a0de0ac87dedaf3f45df1c2d4e4d47c371ff0395471faf6fe67c70d0c9b871787842a4ce19261dfd8a1 + languageName: node + linkType: hard + "http-cache-semantics@npm:^4.0.0, http-cache-semantics@npm:^4.1.0, http-cache-semantics@npm:^4.1.1": version: 4.1.1 resolution: "http-cache-semantics@npm:4.1.1" @@ -24502,12 +24511,12 @@ __metadata: languageName: node linkType: hard -"jsesc@npm:^3.0.2": - version: 3.1.0 - resolution: "jsesc@npm:3.1.0" +"jsesc@npm:^3.0.2, jsesc@npm:~3.0.2": + version: 3.0.2 + resolution: "jsesc@npm:3.0.2" bin: jsesc: bin/jsesc - checksum: 10/20bd37a142eca5d1794f354db8f1c9aeb54d85e1f5c247b371de05d23a9751ecd7bd3a9c4fc5298ea6fa09a100dafb4190fa5c98c6610b75952c3487f3ce7967 + checksum: 10/8e5a7de6b70a8bd71f9cb0b5a7ade6a73ae6ab55e697c74cc997cede97417a3a65ed86c36f7dd6125fe49766e8386c845023d9e213916ca92c9dfdd56e2babf3 languageName: node linkType: hard @@ -24520,15 +24529,6 @@ __metadata: languageName: node linkType: hard -"jsesc@npm:~3.0.2": - version: 3.0.2 - resolution: "jsesc@npm:3.0.2" - bin: - jsesc: bin/jsesc - checksum: 10/8e5a7de6b70a8bd71f9cb0b5a7ade6a73ae6ab55e697c74cc997cede97417a3a65ed86c36f7dd6125fe49766e8386c845023d9e213916ca92c9dfdd56e2babf3 - languageName: node - linkType: hard - "json-bigint@npm:^1.0.0": version: 1.0.0 resolution: "json-bigint@npm:1.0.0" @@ -32658,6 +32658,24 @@ __metadata: languageName: node linkType: hard +"tldts-core@npm:^6.1.68": + version: 6.1.68 + resolution: "tldts-core@npm:6.1.68" + checksum: 10/6cd30acd54a6cd402afb75d4d034ca008ab06b8d254efaa976e38814b7d0095f2fdfda2e33c162085d2f45b2b7b8b5724384192c2268930e63bee886241e399f + languageName: node + linkType: hard + +"tldts@npm:^6.1.68": + version: 6.1.68 + resolution: "tldts@npm:6.1.68" + dependencies: + tldts-core: "npm:^6.1.68" + bin: + tldts: bin/cli.js + checksum: 10/5e28d274ba7364c80f4d81d922427cfae6081b2f33b27a81eab05f5e62f650daef7f8037aae489407f6b1cf997cb3aa353d41cd6bbce75758af44b1ae8b3cfc5 + languageName: node + linkType: hard + "tmp-promise@npm:^3.0.2": version: 3.0.3 resolution: "tmp-promise@npm:3.0.3"