feat(server): self-hosted worker (#10085)

packages/backend/server/src/plugins/index.ts

@@ -4,6 +4,7 @@ import './gcloud';
 import './oauth';
 import './payment';
 import './storage';
+import './worker';
 
 export {
   enablePlugin,

packages/backend/server/src/plugins/worker/config.ts

@@ -0,0 +1,15 @@
+import { defineStartupConfig, ModuleConfig } from '../../base/config';
+
+export interface WorkerStartupConfigurations {
+  allowedOrigin: string[];
+}
+
+declare module '../config' {
+  interface PluginsConfig {
+    worker: ModuleConfig<WorkerStartupConfigurations>;
+  }
+}
+
+defineStartupConfig('plugins.worker', {
+  allowedOrigin: ['localhost', '127.0.0.1'],
+});

packages/backend/server/src/plugins/worker/controller.ts

@@ -0,0 +1,279 @@
+import {
+  Controller,
+  Get,
+  Logger,
+  Options,
+  Post,
+  Req,
+  Res,
+} from '@nestjs/common';
+import type { Request, Response } from 'express';
+import { HTMLRewriter } from 'htmlrewriter';
+
+import { BadRequest, Cache, Config, URLHelper } from '../../base';
+import { Public } from '../../core/auth';
+import type { LinkPreviewRequest, LinkPreviewResponse } from './types';
+import {
+  appendUrl,
+  cloneHeader,
+  fixUrl,
+  getCorsHeaders,
+  isOriginAllowed,
+  isRefererAllowed,
+  OriginRules,
+  parseJson,
+  reduceUrls,
+} from './utils';
+
+@Public()
+@Controller('/api/worker')
+export class WorkerController {
+  private readonly logger = new Logger(WorkerController.name);
+  private readonly allowedOrigin: OriginRules;
+
+  constructor(
+    config: Config,
+    private readonly cache: Cache,
+    private readonly url: URLHelper
+  ) {
+    this.allowedOrigin = [
+      ...config.plugins.worker.allowedOrigin
+        .map(u => fixUrl(u)?.origin as string)
+        .filter(v => !!v),
+      url.origin,
+    ];
+  }
+
+  @Get('/image-proxy')
+  async imageProxy(@Req() req: Request, @Res() resp: Response) {
+    const origin = req.headers.origin ?? '';
+    const referer = req.headers.referer;
+    if (
+      (origin && !isOriginAllowed(origin, this.allowedOrigin)) ||
+      (referer && !isRefererAllowed(referer, this.allowedOrigin))
+    ) {
+      this.logger.error('Invalid Origin', 'ERROR', { origin, referer });
+      throw new BadRequest('Invalid header');
+    }
+    const url = new URL(req.url, this.url.baseUrl);
+    const imageURL = url.searchParams.get('url');
+    if (!imageURL) {
+      throw new BadRequest('Missing "url" parameter');
+    }
+
+    const targetURL = fixUrl(imageURL);
+    if (!targetURL) {
+      this.logger.error(`Invalid URL: ${url}`);
+      throw new BadRequest(`Invalid URL`);
+    }
+
+    const response = await fetch(
+      new Request(targetURL.toString(), {
+        method: 'GET',
+        headers: cloneHeader(req.headers),
+      })
+    );
+    if (response.ok) {
+      const contentType = response.headers.get('Content-Type');
+      const contentDisposition = response.headers.get('Content-Disposition');
+      if (contentType?.startsWith('image/')) {
+        return resp
+          .status(200)
+          .header({
+            'Access-Control-Allow-Origin': origin ?? 'null',
+            Vary: 'Origin',
+            'Access-Control-Allow-Methods': 'GET',
+            'Content-Type': contentType,
+            'Content-Disposition': contentDisposition,
+          })
+          .send(Buffer.from(await response.arrayBuffer()));
+      } else {
+        throw new BadRequest('Invalid content type');
+      }
+    } else {
+      this.logger.error('Failed to fetch image', {
+        origin,
+        url: imageURL,
+        status: resp.status,
+      });
+      throw new BadRequest('Failed to fetch image');
+    }
+  }
+
+  @Options('/link-preview')
+  linkPreviewOption(@Req() request: Request, @Res() resp: Response) {
+    const origin = request.headers.origin;
+    return resp
+      .status(200)
+      .header({
+        ...getCorsHeaders(origin),
+        'Access-Control-Allow-Methods': 'POST, OPTIONS',
+        'Access-Control-Allow-Headers': 'Content-Type',
+      })
+      .send();
+  }
+
+  @Post('/link-preview')
+  async linkPreview(
+    @Req() request: Request,
+    @Res() resp: Response
+  ): Promise<Response> {
+    const origin = request.headers.origin;
+    const referer = request.headers.referer;
+    if (
+      (origin && !isOriginAllowed(origin, this.allowedOrigin)) ||
+      (referer && !isRefererAllowed(referer, this.allowedOrigin))
+    ) {
+      this.logger.error('Invalid Origin', { origin, referer });
+      throw new BadRequest('Invalid header');
+    }
+
+    this.logger.debug('Received request', { origin, method: request.method });
+
+    const targetBody = parseJson<LinkPreviewRequest>(request.body);
+    const targetURL = fixUrl(targetBody?.url);
+    // not allow same site preview
+    if (!targetURL || isOriginAllowed(targetURL.origin, this.allowedOrigin)) {
+      this.logger.error('Invalid URL', { origin, url: targetBody?.url });
+      throw new BadRequest('Invalid URL');
+    }
+
+    this.logger.debug('Processing request', { origin, url: targetURL });
+
+    try {
+      const cachedResponse = await this.cache.get<string>(targetURL.toString());
+      if (cachedResponse) {
+        return resp
+          .status(200)
+          .header({
+            'content-type': 'application/json;charset=UTF-8',
+            ...getCorsHeaders(origin),
+          })
+          .send(cachedResponse);
+      }
+
+      const response = await fetch(targetURL, {
+        headers: cloneHeader(request.headers),
+      });
+      this.logger.error('Fetched URL', {
+        origin,
+        url: targetURL,
+        status: response.status,
+      });
+
+      const res: LinkPreviewResponse = {
+        url: response.url,
+        images: [],
+        videos: [],
+        favicons: [],
+      };
+      const baseUrl = new URL(request.url, this.url.baseUrl).toString();
+
+      if (response.body) {
+        const rewriter = new HTMLRewriter()
+          .on('meta', {
+            element(element) {
+              const property =
+                element.getAttribute('property') ??
+                element.getAttribute('name');
+              const content = element.getAttribute('content');
+              if (property && content) {
+                switch (property.toLowerCase()) {
+                  case 'og:title':
+                    res.title = content;
+                    break;
+                  case 'og:site_name':
+                    res.siteName = content;
+                    break;
+                  case 'og:description':
+                    res.description = content;
+                    break;
+                  case 'og:image':
+                    appendUrl(content, res.images);
+                    break;
+                  case 'og:video':
+                    appendUrl(content, res.videos);
+                    break;
+                  case 'og:type':
+                    res.mediaType = content;
+                    break;
+                  case 'description':
+                    if (!res.description) {
+                      res.description = content;
+                    }
+                }
+              }
+            },
+          })
+          .on('link', {
+            element(element) {
+              if (element.getAttribute('rel')?.toLowerCase().includes('icon')) {
+                appendUrl(element.getAttribute('href'), res.favicons);
+              }
+            },
+          })
+          .on('title', {
+            text(text) {
+              if (!res.title) {
+                res.title = text.text;
+              }
+            },
+          })
+          .on('img', {
+            element(element) {
+              appendUrl(element.getAttribute('src'), res.images);
+            },
+          })
+          .on('video', {
+            element(element) {
+              appendUrl(element.getAttribute('src'), res.videos);
+            },
+          });
+
+        await rewriter.transform(response).text();
+
+        res.images = await reduceUrls(baseUrl, res.images);
+
+        this.logger.error('Processed response with HTMLRewriter', {
+          origin,
+          url: response.url,
+        });
+      }
+
+      // fix favicon
+      {
+        // head default path of favicon
+        const faviconUrl = new URL('/favicon.ico', response.url);
+        const faviconResponse = await fetch(faviconUrl, { method: 'HEAD' });
+        if (faviconResponse.ok) {
+          appendUrl(faviconUrl.toString(), res.favicons);
+        }
+
+        res.favicons = await reduceUrls(baseUrl, res.favicons);
+      }
+
+      const json = JSON.stringify(res);
+      this.logger.debug('Sending response', {
+        origin,
+        url: res.url,
+        responseSize: json.length,
+      });
+
+      await this.cache.set(targetURL.toString(), res);
+      return resp
+        .status(200)
+        .header({
+          'content-type': 'application/json;charset=UTF-8',
+          ...getCorsHeaders(origin),
+        })
+        .send(json);
+    } catch (error) {
+      this.logger.error('Error fetching URL', {
+        origin,
+        url: targetURL,
+        error,
+      });
+      throw new BadRequest('Error fetching URL');
+    }
+  }
+}

packages/backend/server/src/plugins/worker/index.ts

@@ -0,0 +1,11 @@
+import './config';
+
+import { Plugin } from '../registry';
+import { WorkerController } from './controller';
+
+@Plugin({
+  name: 'worker',
+  controllers: [WorkerController],
+  if: config => config.isSelfhosted || config.node.dev || config.node.test,
+})
+export class WorkerModule {}

packages/backend/server/src/plugins/worker/types.ts

@@ -0,0 +1,16 @@
+export type LinkPreviewRequest = {
+  url: string;
+};
+
+export type LinkPreviewResponse = {
+  url: string;
+  title?: string;
+  siteName?: string;
+  description?: string;
+  images?: string[];
+  mediaType?: string;
+  contentType?: string;
+  charset?: string;
+  videos?: string[];
+  favicons?: string[];
+};

packages/backend/server/src/plugins/worker/utils/headers.ts

@@ -0,0 +1,63 @@
+import { IncomingHttpHeaders } from 'node:http';
+
+export type OriginRule = string | RegExp | ((origin: string) => boolean);
+export type OriginRules = OriginRule | OriginRule[];
+
+function isString(s: OriginRule): s is string {
+  return typeof s === 'string' || s instanceof String;
+}
+
+export function isOriginAllowed(origin: string, allowedOrigin: OriginRules) {
+  if (Array.isArray(allowedOrigin)) {
+    for (const allowed of allowedOrigin) {
+      if (isOriginAllowed(origin, allowed)) {
+        return true;
+      }
+    }
+    return false;
+  } else if (isString(allowedOrigin)) {
+    return origin === allowedOrigin;
+  } else if (allowedOrigin instanceof RegExp) {
+    return allowedOrigin.test(origin);
+  }
+  return allowedOrigin(origin);
+}
+
+export function isRefererAllowed(referer: string, allowedOrigin: OriginRules) {
+  try {
+    const origin = new URL(referer).origin;
+    return isOriginAllowed(origin, allowedOrigin);
+  } catch {
+    return false;
+  }
+}
+
+const headerFilters = [/^Sec-/i, /^Accept/i, /^User-Agent$/i];
+
+export function cloneHeader(source: IncomingHttpHeaders) {
+  const headers: Record<string, string> = {};
+
+  Object.entries(source).forEach(([key, value]) => {
+    if (headerFilters.some(filter => filter.test(key))) {
+      if (Array.isArray(value)) {
+        headers[key] = value.join(',');
+      } else if (value) {
+        headers[key] = value;
+      }
+    }
+  });
+
+  return headers;
+}
+
+export function getCorsHeaders(origin?: string | null): {
+  [key: string]: string;
+} {
+  if (origin) {
+    return {
+      'Access-Control-Allow-Origin': origin,
+    };
+  } else {
+    return {};
+  }
+}

packages/backend/server/src/plugins/worker/utils/index.ts

@@ -0,0 +1,12 @@
+export * from './headers';
+export * from './proxy';
+export * from './url';
+
+export function parseJson<T>(data: string): T | null {
+  try {
+    if (data && typeof data === 'object') return data;
+    return JSON.parse(data);
+  } catch {
+    return null;
+  }
+}

packages/backend/server/src/plugins/worker/utils/proxy.ts

@@ -0,0 +1,54 @@
+const IMAGE_PROXY = '/api/worker/image-proxy';
+
+const httpsDomain = new Set();
+
+async function checkHttpsSupport(url: URL): Promise<boolean> {
+  const httpsUrl = new URL(url.toString());
+  httpsUrl.protocol = 'https:';
+  try {
+    const response = await fetch(httpsUrl, {
+      method: 'HEAD',
+      redirect: 'manual',
+    });
+
+    if (response.ok || (response.status >= 400 && response.status < 600)) {
+      return true;
+    }
+  } catch {}
+  return false;
+}
+
+async function fixProtocol(url: string): Promise<URL> {
+  const targetUrl = new URL(url);
+  if (targetUrl.protocol !== 'http:') {
+    return targetUrl;
+  } else if (httpsDomain.has(targetUrl.hostname)) {
+    targetUrl.protocol = 'https:';
+    return targetUrl;
+  } else if (await checkHttpsSupport(targetUrl)) {
+    httpsDomain.add(targetUrl.hostname);
+    targetUrl.protocol = 'https:';
+    return targetUrl;
+  }
+  return targetUrl;
+}
+
+export function imageProxyBuilder(
+  url: string
+): (url: string) => Promise<string | undefined> {
+  try {
+    const proxy = new URL(url);
+    proxy.pathname = IMAGE_PROXY;
+
+    return async url => {
+      try {
+        const targetUrl = await fixProtocol(url);
+        proxy.searchParams.set('url', targetUrl.toString());
+        return proxy.toString();
+      } catch {}
+      return;
+    };
+  } catch {
+    return async url => url.toString();
+  }
+}

packages/backend/server/src/plugins/worker/utils/url.ts

@@ -0,0 +1,54 @@
+import { getDomain, getSubdomain } from 'tldts';
+
+import { imageProxyBuilder } from './proxy';
+
+const localhost = new Set(['localhost', '127.0.0.1']);
+
+export function fixUrl(url?: string): URL | null {
+  if (typeof url !== 'string') {
+    return null;
+  }
+
+  let fullUrl = url;
+
+  // don't require // prefix, URL can handle protocol:domain
+  if (!url.startsWith('http:') && !url.startsWith('https:')) {
+    fullUrl = 'http://' + url;
+  }
+
+  try {
+    const parsed = new URL(fullUrl);
+
+    const subDomain = getSubdomain(url);
+    const mainDomain = getDomain(url);
+    const fullDomain = subDomain ? `${subDomain}.${mainDomain}` : mainDomain;
+
+    if (
+      ['http:', 'https:'].includes(parsed.protocol) &&
+      // check hostname is a valid domain
+      (fullDomain === parsed.hostname || localhost.has(parsed.hostname))
+    ) {
+      return parsed;
+    }
+  } catch {}
+
+  return null;
+}
+
+export function appendUrl(url: string | null, array?: string[]) {
+  if (url) {
+    const fixedUrl = fixUrl(url);
+    if (fixedUrl) {
+      array?.push(fixedUrl.toString());
+    }
+  }
+}
+
+export async function reduceUrls(baseUrl: string, urls?: string[]) {
+  if (urls && urls.length > 0) {
+    const imageProxy = imageProxyBuilder(baseUrl);
+    const newUrls = await Promise.all(urls.map(imageProxy));
+    return newUrls.filter((x): x is string => !!x);
+  }
+  return [];
+}