feat(server): self-hosted worker (#10085)

oxlint.json

@@ -30,6 +30,7 @@
     "tools/cli/src/webpack/error-handler.js",
     "packages/backend/native/index.d.ts",
     "packages/backend/server/src/__tests__/__snapshots__",
+    "packages/common/native/fixtures/**",
     "packages/frontend/native/index.d.ts",
     "packages/frontend/native/index.js",
     "packages/frontend/graphql/src/graphql/index.ts",

packages/backend/server/package.json

@@ -70,6 +70,7 @@
     "graphql-scalars": "^1.24.0",
     "graphql-upload": "^17.0.0",
     "html-validate": "^9.0.0",
+    "htmlrewriter": "^0.0.12",
     "ioredis": "^5.4.1",
     "is-mobile": "^5.0.0",
     "keyv": "^5.2.2",
@@ -92,6 +93,7 @@
     "ses": "^1.10.0",
     "socket.io": "^4.8.1",
     "stripe": "^17.4.0",
+    "tldts": "^6.1.68",
     "ts-node": "^10.9.2",
     "typescript": "^5.7.2",
     "winston": "^3.17.0",

packages/backend/server/src/__tests__/__snapshots__/worker.e2e.ts.md

@@ -0,0 +1,78 @@
+# Snapshot report for `src/__tests__/worker.e2e.ts`
+
+The actual snapshot is saved in `worker.e2e.ts.snap`.
+
+Generated by [AVA](https://avajs.dev).
+
+## should proxy image
+
+> Snapshot 1
+
+    {}
+
+> Snapshot 2
+
+    {
+      code: 'Bad Request',
+      message: 'Missing "url" parameter',
+      name: 'BAD_REQUEST',
+      status: 400,
+      type: 'BAD_REQUEST',
+    }
+
+> Snapshot 3
+
+    {
+      code: 'Bad Request',
+      message: 'Invalid header',
+      name: 'BAD_REQUEST',
+      status: 400,
+      type: 'BAD_REQUEST',
+    }
+
+> Snapshot 4
+
+    Buffer @Uint8Array [
+      66616b65 20696d61 6765
+    ]
+
+## should preview link
+
+> Snapshot 1
+
+    {}
+
+> Snapshot 2
+
+    {
+      code: 'Bad Request',
+      message: 'Invalid URL',
+      name: 'BAD_REQUEST',
+      status: 400,
+      type: 'BAD_REQUEST',
+    }
+
+> Snapshot 3
+
+    {
+      code: 'Bad Request',
+      message: 'Invalid URL',
+      name: 'BAD_REQUEST',
+      status: 400,
+      type: 'BAD_REQUEST',
+    }
+
+> Snapshot 4
+
+    {
+      description: 'Test Description',
+      favicons: [
+        'http://localhost:3010/api/worker/image-proxy?url=https%3A%2F%2Fexample.com%2Ffavicon.ico',
+      ],
+      images: [
+        'http://localhost:3010/api/worker/image-proxy?url=https%3A%2F%2Fexample.com%2Fimage.png',
+      ],
+      title: 'Test Title',
+      url: 'http://example.com/page',
+      videos: [],
+    }

packages/backend/server/src/__tests__/utils/testing-app.ts

@@ -95,7 +95,7 @@ export class TestingApp extends ApplyType<INestApplication>() {
   }
 
   request(
-    method: 'get' | 'post' | 'put' | 'delete' | 'patch',
+    method: 'options' | 'get' | 'post' | 'put' | 'delete' | 'patch',
     path: string
   ): supertest.Test {
     return supertest(this.getHttpServer())
@@ -106,6 +106,10 @@ export class TestingApp extends ApplyType<INestApplication>() {
       ]);
   }
 
+  OPTIONS(path: string): supertest.Test {
+    return this.request('options', path);
+  }
+
   GET(path: string): supertest.Test {
     return this.request('get', path);
   }

packages/backend/server/src/__tests__/worker.e2e.ts

@@ -0,0 +1,174 @@
+import type { ExecutionContext, TestFn } from 'ava';
+import ava from 'ava';
+import Sinon from 'sinon';
+import type { Response } from 'supertest';
+
+import { WorkerModule } from '../plugins/worker';
+import { createTestingApp, TestingApp } from './utils';
+
+type TestContext = {
+  app: TestingApp;
+};
+
+const test = ava as TestFn<TestContext>;
+
+test.before(async t => {
+  const app = await createTestingApp({
+    imports: [WorkerModule],
+  });
+
+  t.context.app = app;
+});
+
+test.after.always(async t => {
+  await t.context.app.close();
+});
+
+const assertAndSnapshotRaw = async (
+  t: ExecutionContext<TestContext>,
+  route: string,
+  message: string,
+  options?: {
+    status?: number;
+    origin?: string;
+    method?: 'GET' | 'OPTIONS' | 'POST';
+    body?: any;
+    checker?: (res: Response) => any;
+  }
+) => {
+  const {
+    status = 200,
+    origin = 'http://localhost',
+    method = 'GET',
+    checker = () => {},
+  } = options || {};
+  const { app } = t.context;
+  const res = app[method](route)
+    .set('Origin', origin)
+    .send(options?.body)
+    .expect(status)
+    .expect(checker);
+  t.notThrowsAsync(res, message);
+  t.snapshot((await res).body);
+};
+
+test('should proxy image', async t => {
+  const assertAndSnapshot = assertAndSnapshotRaw.bind(null, t);
+
+  await assertAndSnapshot(
+    '/api/worker/image-proxy',
+    'should return proper CORS headers on OPTIONS request',
+    {
+      status: 204,
+      method: 'OPTIONS',
+      checker: (res: Response) => {
+        if (!res.headers['access-control-allow-methods']) {
+          throw new Error('Missing CORS headers');
+        }
+      },
+    }
+  );
+
+  {
+    await assertAndSnapshot(
+      '/api/worker/image-proxy',
+      'should return 400 if "url" query parameter is missing',
+      { status: 400 }
+    );
+  }
+
+  {
+    await assertAndSnapshot(
+      '/api/worker/image-proxy?url=http://example.com/image.png',
+      'should return 400 for invalid origin header',
+      { status: 400, origin: 'http://invalid.com' }
+    );
+  }
+
+  {
+    const fakeBuffer = Buffer.from('fake image');
+    const fakeResponse = {
+      ok: true,
+      headers: {
+        get: (header: string) => {
+          if (header.toLowerCase() === 'content-type') return 'image/png';
+          if (header.toLowerCase() === 'content-disposition') return 'inline';
+          return null;
+        },
+      },
+      arrayBuffer: async () => fakeBuffer,
+    } as any;
+
+    const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeResponse);
+
+    await assertAndSnapshot(
+      '/api/worker/image-proxy?url=http://example.com/image.png',
+      'should return image buffer'
+    );
+
+    fetchSpy.restore();
+  }
+});
+
+test('should preview link', async t => {
+  const assertAndSnapshot = assertAndSnapshotRaw.bind(null, t);
+
+  await assertAndSnapshot(
+    '/api/worker/link-preview',
+    'should return proper CORS headers on OPTIONS request',
+    {
+      status: 204,
+      method: 'OPTIONS',
+      checker: (res: Response) => {
+        if (!res.headers['access-control-allow-methods']) {
+          throw new Error('Missing CORS headers');
+        }
+      },
+    }
+  );
+
+  await assertAndSnapshot(
+    '/api/worker/link-preview',
+    'should return 400 if request body is invalid',
+    { status: 400, method: 'POST' }
+  );
+
+  await assertAndSnapshot(
+    '/api/worker/link-preview',
+    'should return 400 if provided URL is from the same origin',
+    { status: 400, method: 'POST', body: { url: 'http://localhost/somepage' } }
+  );
+
+  {
+    const fakeHTML = new Response(`
+        <html>
+          <head>
+            <meta property="og:title" content="Test Title" />
+            <meta property="og:description" content="Test Description" />
+            <meta property="og:image" content="http://example.com/image.png" />
+          </head>
+          <body>
+            <title>Fallback Title</title>
+          </body>
+        </html>
+      `);
+
+    Object.defineProperty(fakeHTML, 'url', {
+      value: 'http://example.com/page',
+    });
+
+    const fetchSpy = Sinon.stub(global, 'fetch').resolves(fakeHTML);
+
+    await assertAndSnapshot(
+      '/api/worker/link-preview',
+      'should process a valid external URL and return link preview data',
+      {
+        status: 200,
+        method: 'POST',
+        body: { url: 'http://external.com/page' },
+      }
+    );
+
+    fetchSpy.restore();
+  }
+});

packages/backend/server/src/base/error/def.ts

@@ -247,6 +247,10 @@ export const USER_FRIENDLY_ERRORS = {
     type: 'resource_not_found',
     message: 'Resource not found.',
   },
+  bad_request: {
+    type: 'bad_request',
+    message: 'Bad request.',
+  },
 
   // Input errors
   query_too_long: {

packages/backend/server/src/base/error/errors.gen.ts

@@ -21,6 +21,12 @@ export class NotFound extends UserFriendlyError {
     super('resource_not_found', 'not_found', message);
   }
 }
+
+export class BadRequest extends UserFriendlyError {
+  constructor(message?: string) {
+    super('bad_request', 'bad_request', message);
+  }
+}
 @ObjectType()
 class QueryTooLongDataType {
   @Field() max!: number
@@ -770,6 +776,7 @@ export enum ErrorNames {
   INTERNAL_SERVER_ERROR,
   TOO_MANY_REQUEST,
   NOT_FOUND,
+  BAD_REQUEST,
   QUERY_TOO_LONG,
   USER_NOT_FOUND,
   USER_AVATAR_NOT_FOUND,

packages/backend/server/src/config/affine.self.ts

@@ -89,4 +89,7 @@ if (AFFiNE.deploy) {
   };
 
   AFFiNE.use('gcloud');
+} else {
+  // only enable dev mode
+  AFFiNE.use('worker');
 }

packages/backend/server/src/config/affine.ts

@@ -161,3 +161,8 @@ AFFiNE.server.port = 3010;
 //     bucket: 'copilot',
 //   }
 // })
+//
+// /* AFFiNE Link Preview & Image Proxy API */
+// AFFiNE.use('worker', {
+//   allowedOrigin: ['example.com'],
+// });

packages/backend/server/src/plugins/index.ts

@@ -4,6 +4,7 @@ import './gcloud';
 import './oauth';
 import './payment';
 import './storage';
+import './worker';
 
 export {
   enablePlugin,

packages/backend/server/src/plugins/worker/config.ts

@@ -0,0 +1,15 @@
+import { defineStartupConfig, ModuleConfig } from '../../base/config';
+
+export interface WorkerStartupConfigurations {
+  allowedOrigin: string[];
+}
+
+declare module '../config' {
+  interface PluginsConfig {
+    worker: ModuleConfig<WorkerStartupConfigurations>;
+  }
+}
+
+defineStartupConfig('plugins.worker', {
+  allowedOrigin: ['localhost', '127.0.0.1'],
+});

packages/backend/server/src/plugins/worker/controller.ts

@@ -0,0 +1,279 @@
+import {
+  Controller,
+  Get,
+  Logger,
+  Options,
+  Post,
+  Req,
+  Res,
+} from '@nestjs/common';
+import type { Request, Response } from 'express';
+import { HTMLRewriter } from 'htmlrewriter';
+
+import { BadRequest, Cache, Config, URLHelper } from '../../base';
+import { Public } from '../../core/auth';
+import type { LinkPreviewRequest, LinkPreviewResponse } from './types';
+import {
+  appendUrl,
+  cloneHeader,
+  fixUrl,
+  getCorsHeaders,
+  isOriginAllowed,
+  isRefererAllowed,
+  OriginRules,
+  parseJson,
+  reduceUrls,
+} from './utils';
+
+@Public()
+@Controller('/api/worker')
+export class WorkerController {
+  private readonly logger = new Logger(WorkerController.name);
+  private readonly allowedOrigin: OriginRules;
+
+  constructor(
+    config: Config,
+    private readonly cache: Cache,
+    private readonly url: URLHelper
+  ) {
+    this.allowedOrigin = [
+      ...config.plugins.worker.allowedOrigin
+        .map(u => fixUrl(u)?.origin as string)
+        .filter(v => !!v),
+      url.origin,
+    ];
+  }
+
+  @Get('/image-proxy')
+  async imageProxy(@Req() req: Request, @Res() resp: Response) {
+    const origin = req.headers.origin ?? '';
+    const referer = req.headers.referer;
+    if (
+      (origin && !isOriginAllowed(origin, this.allowedOrigin)) ||
+      (referer && !isRefererAllowed(referer, this.allowedOrigin))
+    ) {
+      this.logger.error('Invalid Origin', 'ERROR', { origin, referer });
+      throw new BadRequest('Invalid header');
+    }
+    const url = new URL(req.url, this.url.baseUrl);
+    const imageURL = url.searchParams.get('url');
+    if (!imageURL) {
+      throw new BadRequest('Missing "url" parameter');
+    }
+
+    const targetURL = fixUrl(imageURL);
+    if (!targetURL) {
+      this.logger.error(`Invalid URL: ${url}`);
+      throw new BadRequest(`Invalid URL`);
+    }
+
+    const response = await fetch(
+      new Request(targetURL.toString(), {
+        method: 'GET',
+        headers: cloneHeader(req.headers),
+      })
+    );
+    if (response.ok) {
+      const contentType = response.headers.get('Content-Type');
+      const contentDisposition = response.headers.get('Content-Disposition');
+      if (contentType?.startsWith('image/')) {
+        return resp
+          .status(200)
+          .header({
+            'Access-Control-Allow-Origin': origin ?? 'null',
+            Vary: 'Origin',
+            'Access-Control-Allow-Methods': 'GET',
+            'Content-Type': contentType,
+            'Content-Disposition': contentDisposition,
+          })
+          .send(Buffer.from(await response.arrayBuffer()));
+      } else {
+        throw new BadRequest('Invalid content type');
+      }
+    } else {
+      this.logger.error('Failed to fetch image', {
+        origin,
+        url: imageURL,
+        status: resp.status,
+      });
+      throw new BadRequest('Failed to fetch image');
+    }
+  }
+
+  @Options('/link-preview')
+  linkPreviewOption(@Req() request: Request, @Res() resp: Response) {
+    const origin = request.headers.origin;
+    return resp
+      .status(200)
+      .header({
+        ...getCorsHeaders(origin),
+        'Access-Control-Allow-Methods': 'POST, OPTIONS',
+        'Access-Control-Allow-Headers': 'Content-Type',
+      })
+      .send();
+  }
+
+  @Post('/link-preview')
+  async linkPreview(
+    @Req() request: Request,
+    @Res() resp: Response
+  ): Promise<Response> {
+    const origin = request.headers.origin;
+    const referer = request.headers.referer;
+    if (
+      (origin && !isOriginAllowed(origin, this.allowedOrigin)) ||
+      (referer && !isRefererAllowed(referer, this.allowedOrigin))
+    ) {
+      this.logger.error('Invalid Origin', { origin, referer });
+      throw new BadRequest('Invalid header');
+    }
+
+    this.logger.debug('Received request', { origin, method: request.method });
+
+    const targetBody = parseJson<LinkPreviewRequest>(request.body);
+    const targetURL = fixUrl(targetBody?.url);
+    // not allow same site preview
+    if (!targetURL || isOriginAllowed(targetURL.origin, this.allowedOrigin)) {
+      this.logger.error('Invalid URL', { origin, url: targetBody?.url });
+      throw new BadRequest('Invalid URL');
+    }
+
+    this.logger.debug('Processing request', { origin, url: targetURL });
+
+    try {
+      const cachedResponse = await this.cache.get<string>(targetURL.toString());
+      if (cachedResponse) {
+        return resp
+          .status(200)
+          .header({
+            'content-type': 'application/json;charset=UTF-8',
+            ...getCorsHeaders(origin),
+          })
+          .send(cachedResponse);
+      }
+
+      const response = await fetch(targetURL, {
+        headers: cloneHeader(request.headers),
+      });
+      this.logger.error('Fetched URL', {
+        origin,
+        url: targetURL,
+        status: response.status,
+      });
+
+      const res: LinkPreviewResponse = {
+        url: response.url,
+        images: [],
+        videos: [],
+        favicons: [],
+      };
+      const baseUrl = new URL(request.url, this.url.baseUrl).toString();
+
+      if (response.body) {
+        const rewriter = new HTMLRewriter()
+          .on('meta', {
+            element(element) {
+              const property =
+                element.getAttribute('property') ??
+                element.getAttribute('name');
+              const content = element.getAttribute('content');
+              if (property && content) {
+                switch (property.toLowerCase()) {
+                  case 'og:title':
+                    res.title = content;
+                    break;
+                  case 'og:site_name':
+                    res.siteName = content;
+                    break;
+                  case 'og:description':
+                    res.description = content;
+                    break;
+                  case 'og:image':
+                    appendUrl(content, res.images);
+                    break;
+                  case 'og:video':
+                    appendUrl(content, res.videos);
+                    break;
+                  case 'og:type':
+                    res.mediaType = content;
+                    break;
+                  case 'description':
+                    if (!res.description) {
+                      res.description = content;
+                    }
+                }
+              }
+            },
+          })
+          .on('link', {
+            element(element) {
+              if (element.getAttribute('rel')?.toLowerCase().includes('icon')) {
+                appendUrl(element.getAttribute('href'), res.favicons);
+              }
+            },
+          })
+          .on('title', {
+            text(text) {
+              if (!res.title) {
+                res.title = text.text;
+              }
+            },
+          })
+          .on('img', {
+            element(element) {
+              appendUrl(element.getAttribute('src'), res.images);
+            },
+          })
+          .on('video', {
+            element(element) {
+              appendUrl(element.getAttribute('src'), res.videos);
+            },
+          });
+
+        await rewriter.transform(response).text();
+
+        res.images = await reduceUrls(baseUrl, res.images);
+
+        this.logger.error('Processed response with HTMLRewriter', {
+          origin,
+          url: response.url,
+        });
+      }
+
+      // fix favicon
+      {
+        // head default path of favicon
+        const faviconUrl = new URL('/favicon.ico', response.url);
+        const faviconResponse = await fetch(faviconUrl, { method: 'HEAD' });
+        if (faviconResponse.ok) {
+          appendUrl(faviconUrl.toString(), res.favicons);
+        }
+
+        res.favicons = await reduceUrls(baseUrl, res.favicons);
+      }
+
+      const json = JSON.stringify(res);
+      this.logger.debug('Sending response', {
+        origin,
+        url: res.url,
+        responseSize: json.length,
+      });
+
+      await this.cache.set(targetURL.toString(), res);
+      return resp
+        .status(200)
+        .header({
+          'content-type': 'application/json;charset=UTF-8',
+          ...getCorsHeaders(origin),
+        })
+        .send(json);
+    } catch (error) {
+      this.logger.error('Error fetching URL', {
+        origin,
+        url: targetURL,
+        error,
+      });
+      throw new BadRequest('Error fetching URL');
+    }
+  }
+}

packages/backend/server/src/plugins/worker/index.ts

@@ -0,0 +1,11 @@
+import './config';
+
+import { Plugin } from '../registry';
+import { WorkerController } from './controller';
+
+@Plugin({
+  name: 'worker',
+  controllers: [WorkerController],
+  if: config => config.isSelfhosted || config.node.dev || config.node.test,
+})
+export class WorkerModule {}

packages/backend/server/src/plugins/worker/types.ts

@@ -0,0 +1,16 @@
+export type LinkPreviewRequest = {
+  url: string;
+};
+
+export type LinkPreviewResponse = {
+  url: string;
+  title?: string;
+  siteName?: string;
+  description?: string;
+  images?: string[];
+  mediaType?: string;
+  contentType?: string;
+  charset?: string;
+  videos?: string[];
+  favicons?: string[];
+};

packages/backend/server/src/plugins/worker/utils/headers.ts

@@ -0,0 +1,63 @@
+import { IncomingHttpHeaders } from 'node:http';
+
+export type OriginRule = string | RegExp | ((origin: string) => boolean);
+export type OriginRules = OriginRule | OriginRule[];
+
+function isString(s: OriginRule): s is string {
+  return typeof s === 'string' || s instanceof String;
+}
+
+export function isOriginAllowed(origin: string, allowedOrigin: OriginRules) {
+  if (Array.isArray(allowedOrigin)) {
+    for (const allowed of allowedOrigin) {
+      if (isOriginAllowed(origin, allowed)) {
+        return true;
+      }
+    }
+    return false;
+  } else if (isString(allowedOrigin)) {
+    return origin === allowedOrigin;
+  } else if (allowedOrigin instanceof RegExp) {
+    return allowedOrigin.test(origin);
+  }
+  return allowedOrigin(origin);
+}
+
+export function isRefererAllowed(referer: string, allowedOrigin: OriginRules) {
+  try {
+    const origin = new URL(referer).origin;
+    return isOriginAllowed(origin, allowedOrigin);
+  } catch {
+    return false;
+  }
+}
+
+const headerFilters = [/^Sec-/i, /^Accept/i, /^User-Agent$/i];
+
+export function cloneHeader(source: IncomingHttpHeaders) {
+  const headers: Record<string, string> = {};
+
+  Object.entries(source).forEach(([key, value]) => {
+    if (headerFilters.some(filter => filter.test(key))) {
+      if (Array.isArray(value)) {
+        headers[key] = value.join(',');
+      } else if (value) {
+        headers[key] = value;
+      }
+    }
+  });
+
+  return headers;
+}
+
+export function getCorsHeaders(origin?: string | null): {
+  [key: string]: string;
+} {
+  if (origin) {
+    return {
+      'Access-Control-Allow-Origin': origin,
+    };
+  } else {
+    return {};
+  }
+}

packages/backend/server/src/plugins/worker/utils/index.ts

@@ -0,0 +1,12 @@
+export * from './headers';
+export * from './proxy';
+export * from './url';
+
+export function parseJson<T>(data: string): T | null {
+  try {
+    if (data && typeof data === 'object') return data;
+    return JSON.parse(data);
+  } catch {
+    return null;
+  }
+}

packages/backend/server/src/plugins/worker/utils/proxy.ts

@@ -0,0 +1,54 @@
+const IMAGE_PROXY = '/api/worker/image-proxy';
+
+const httpsDomain = new Set();
+
+async function checkHttpsSupport(url: URL): Promise<boolean> {
+  const httpsUrl = new URL(url.toString());
+  httpsUrl.protocol = 'https:';
+  try {
+    const response = await fetch(httpsUrl, {
+      method: 'HEAD',
+      redirect: 'manual',
+    });
+
+    if (response.ok || (response.status >= 400 && response.status < 600)) {
+      return true;
+    }
+  } catch {}
+  return false;
+}
+
+async function fixProtocol(url: string): Promise<URL> {
+  const targetUrl = new URL(url);
+  if (targetUrl.protocol !== 'http:') {
+    return targetUrl;
+  } else if (httpsDomain.has(targetUrl.hostname)) {
+    targetUrl.protocol = 'https:';
+    return targetUrl;
+  } else if (await checkHttpsSupport(targetUrl)) {
+    httpsDomain.add(targetUrl.hostname);
+    targetUrl.protocol = 'https:';
+    return targetUrl;
+  }
+  return targetUrl;
+}
+
+export function imageProxyBuilder(
+  url: string
+): (url: string) => Promise<string | undefined> {
+  try {
+    const proxy = new URL(url);
+    proxy.pathname = IMAGE_PROXY;
+
+    return async url => {
+      try {
+        const targetUrl = await fixProtocol(url);
+        proxy.searchParams.set('url', targetUrl.toString());
+        return proxy.toString();
+      } catch {}
+      return;
+    };
+  } catch {
+    return async url => url.toString();
+  }
+}

packages/backend/server/src/plugins/worker/utils/url.ts

@@ -0,0 +1,54 @@
+import { getDomain, getSubdomain } from 'tldts';
+
+import { imageProxyBuilder } from './proxy';
+
+const localhost = new Set(['localhost', '127.0.0.1']);
+
+export function fixUrl(url?: string): URL | null {
+  if (typeof url !== 'string') {
+    return null;
+  }
+
+  let fullUrl = url;
+
+  // don't require // prefix, URL can handle protocol:domain
+  if (!url.startsWith('http:') && !url.startsWith('https:')) {
+    fullUrl = 'http://' + url;
+  }
+
+  try {
+    const parsed = new URL(fullUrl);
+
+    const subDomain = getSubdomain(url);
+    const mainDomain = getDomain(url);
+    const fullDomain = subDomain ? `${subDomain}.${mainDomain}` : mainDomain;
+
+    if (
+      ['http:', 'https:'].includes(parsed.protocol) &&
+      // check hostname is a valid domain
+      (fullDomain === parsed.hostname || localhost.has(parsed.hostname))
+    ) {
+      return parsed;
+    }
+  } catch {}
+
+  return null;
+}
+
+export function appendUrl(url: string | null, array?: string[]) {
+  if (url) {
+    const fixedUrl = fixUrl(url);
+    if (fixedUrl) {
+      array?.push(fixedUrl.toString());
+    }
+  }
+}
+
+export async function reduceUrls(baseUrl: string, urls?: string[]) {
+  if (urls && urls.length > 0) {
+    const imageProxy = imageProxyBuilder(baseUrl);
+    const newUrls = await Promise.all(urls.map(imageProxy));
+    return newUrls.filter((x): x is string => !!x);
+  }
+  return [];
+}

packages/backend/server/src/schema.gql

@@ -319,6 +319,7 @@ enum ErrorNames {
   ACTION_FORBIDDEN_ON_NON_TEAM_WORKSPACE
   ALREADY_IN_SPACE
   AUTHENTICATION_REQUIRED
+  BAD_REQUEST
   BLOB_NOT_FOUND
   BLOB_QUOTA_EXCEEDED
   CANNOT_DELETE_ALL_ADMIN_ACCOUNT

packages/frontend/graphql/src/schema.ts

@@ -425,6 +425,7 @@ export enum ErrorNames {
   ACTION_FORBIDDEN_ON_NON_TEAM_WORKSPACE = 'ACTION_FORBIDDEN_ON_NON_TEAM_WORKSPACE',
   ALREADY_IN_SPACE = 'ALREADY_IN_SPACE',
   AUTHENTICATION_REQUIRED = 'AUTHENTICATION_REQUIRED',
+  BAD_REQUEST = 'BAD_REQUEST',
   BLOB_NOT_FOUND = 'BLOB_NOT_FOUND',
   BLOB_QUOTA_EXCEEDED = 'BLOB_QUOTA_EXCEEDED',
   CANNOT_DELETE_ALL_ADMIN_ACCOUNT = 'CANNOT_DELETE_ALL_ADMIN_ACCOUNT',

tools/cli/src/webpack/index.ts

@@ -361,12 +361,6 @@ export function createWebpackConfig(
         },
       ],
       proxy: [
-        {
-          context: '/api/worker/',
-          target: 'https://affine.fail',
-          changeOrigin: true,
-          secure: false,
-        },
         { context: '/api', target: 'http://localhost:3010' },
         { context: '/socket.io', target: 'http://localhost:3010', ws: true },
         { context: '/graphql', target: 'http://localhost:3010' },

yarn.lock

@@ -839,6 +839,7 @@ __metadata:
     graphql-scalars: "npm:^1.24.0"
     graphql-upload: "npm:^17.0.0"
     html-validate: "npm:^9.0.0"
+    htmlrewriter: "npm:^0.0.12"
     ioredis: "npm:^5.4.1"
     is-mobile: "npm:^5.0.0"
     keyv: "npm:^5.2.2"
@@ -865,6 +866,7 @@ __metadata:
     socket.io: "npm:^4.8.1"
     stripe: "npm:^17.4.0"
     supertest: "npm:^7.0.0"
+    tldts: "npm:^6.1.68"
     ts-node: "npm:^10.9.2"
     typescript: "npm:^5.7.2"
     winston: "npm:^3.17.0"
@@ -23193,6 +23195,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"htmlrewriter@npm:^0.0.12":
+  version: 0.0.12
+  resolution: "htmlrewriter@npm:0.0.12"
+  checksum: 10/8e1b39c52957c42fbf989f805504c92821fddcd7d3b42a0de0ac87dedaf3f45df1c2d4e4d47c371ff0395471faf6fe67c70d0c9b871787842a4ce19261dfd8a1
+  languageName: node
+  linkType: hard
+
 "http-cache-semantics@npm:^4.0.0, http-cache-semantics@npm:^4.1.0, http-cache-semantics@npm:^4.1.1":
   version: 4.1.1
   resolution: "http-cache-semantics@npm:4.1.1"
@@ -24502,12 +24511,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"jsesc@npm:^3.0.2":
-  version: 3.1.0
-  resolution: "jsesc@npm:3.1.0"
+"jsesc@npm:^3.0.2, jsesc@npm:~3.0.2":
+  version: 3.0.2
+  resolution: "jsesc@npm:3.0.2"
   bin:
     jsesc: bin/jsesc
-  checksum: 10/20bd37a142eca5d1794f354db8f1c9aeb54d85e1f5c247b371de05d23a9751ecd7bd3a9c4fc5298ea6fa09a100dafb4190fa5c98c6610b75952c3487f3ce7967
+  checksum: 10/8e5a7de6b70a8bd71f9cb0b5a7ade6a73ae6ab55e697c74cc997cede97417a3a65ed86c36f7dd6125fe49766e8386c845023d9e213916ca92c9dfdd56e2babf3
   languageName: node
   linkType: hard
 
@@ -24520,15 +24529,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"jsesc@npm:~3.0.2":
-  version: 3.0.2
-  resolution: "jsesc@npm:3.0.2"
-  bin:
-    jsesc: bin/jsesc
-  checksum: 10/8e5a7de6b70a8bd71f9cb0b5a7ade6a73ae6ab55e697c74cc997cede97417a3a65ed86c36f7dd6125fe49766e8386c845023d9e213916ca92c9dfdd56e2babf3
-  languageName: node
-  linkType: hard
-
 "json-bigint@npm:^1.0.0":
   version: 1.0.0
   resolution: "json-bigint@npm:1.0.0"
@@ -32658,6 +32658,24 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tldts-core@npm:^6.1.68":
+  version: 6.1.68
+  resolution: "tldts-core@npm:6.1.68"
+  checksum: 10/6cd30acd54a6cd402afb75d4d034ca008ab06b8d254efaa976e38814b7d0095f2fdfda2e33c162085d2f45b2b7b8b5724384192c2268930e63bee886241e399f
+  languageName: node
+  linkType: hard
+
+"tldts@npm:^6.1.68":
+  version: 6.1.68
+  resolution: "tldts@npm:6.1.68"
+  dependencies:
+    tldts-core: "npm:^6.1.68"
+  bin:
+    tldts: bin/cli.js
+  checksum: 10/5e28d274ba7364c80f4d81d922427cfae6081b2f33b27a81eab05f5e62f650daef7f8037aae489407f6b1cf997cb3aa353d41cd6bbce75758af44b1ae8b3cfc5
+  languageName: node
+  linkType: hard
+
 "tmp-promise@npm:^3.0.2":
   version: 3.0.3
   resolution: "tmp-promise@npm:3.0.3"