feat: support get dynamic page meta data (#2632)

apps/electron/layers/main/src/ui/get-meta-data/index.ts

@@ -1,23 +1,15 @@
 import type { CheerioAPI, Element } from 'cheerio';
 import { load } from 'cheerio';
-import got from 'got';
 
 import type { Context, MetaData, Options, RuleSet } from './types';
 
 export * from './types';
 
+import { getHTMLByURL } from './get-html';
 import { metaDataRules } from './rules';
+import type { GetMetaDataOptions } from './types';
 
-const defaultOptions = {
-  maxRedirects: 5,
-  ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',
-  lang: '*',
-  timeout: 10000,
-  forceImageHttps: true,
-  customRules: {},
-};
-
-const runRule = function (ruleSet: RuleSet, $: CheerioAPI, context: Context) {
+function runRule(ruleSet: RuleSet, $: CheerioAPI, context: Context) {
   let maxScore = 0;
   let value;
 
@@ -58,61 +50,31 @@ const runRule = function (ruleSet: RuleSet, $: CheerioAPI, context: Context) {
   }
 
   return undefined;
-};
+}
 
-const getMetaData = async function (
-  input: string | Partial<Options>,
-  inputOptions: Partial<Options> = {}
+async function getMetaDataByHTML(
+  html: string,
+  url: string,
+  options: GetMetaDataOptions
 ) {
-  let url;
-  if (typeof input === 'object') {
-    inputOptions = input;
-    url = input.url || '';
-  } else {
-    url = input;
-  }
-
-  const options = Object.assign({}, defaultOptions, inputOptions);
-
+  const { customRules = {} } = options;
   const rules: Record<string, RuleSet> = { ...metaDataRules };
-  Object.keys(options.customRules).forEach((key: string) => {
+  Object.keys(customRules).forEach((key: string) => {
     rules[key] = {
-      rules: [...metaDataRules[key].rules, ...options.customRules[key].rules],
+      rules: [...metaDataRules[key].rules, ...customRules[key].rules],
       defaultValue:
-        options.customRules[key].defaultValue ||
-        metaDataRules[key].defaultValue,
-      processor:
-        options.customRules[key].processor || metaDataRules[key].processor,
+        customRules[key].defaultValue || metaDataRules[key].defaultValue,
+      processor: customRules[key].processor || metaDataRules[key].processor,
     };
   });
 
-  let html;
-  if (!options.html) {
-    const response = await got(url, {
-      headers: {
-        'User-Agent': options.ua,
-        'Accept-Language': options.lang,
-      },
-      timeout: options.timeout,
-      ...(options.maxRedirects === 0
-        ? { followRedirect: false }
-        : { maxRedirects: options.maxRedirects }),
-    });
-    html = response.body;
-  } else {
-    html = options.html;
-  }
-
   const metadata: MetaData = {};
   const context: Context = {
     url,
-    options,
+    ...options,
   };
 
   const $ = load(html);
-  // console.log('===============================');
-  // console.log('html');
-  // console.log(doc);
 
   Object.keys(rules).forEach((key: string) => {
     const ruleSet = rules[key];
@@ -120,6 +82,26 @@ const getMetaData = async function (
   });
 
   return metadata;
-};
+}
 
-export { getMetaData };
+export async function getMetaData(url: string, options: Options = {}) {
+  const { customRules, forceImageHttps, shouldReGetHTML, ...other } = options;
+  const html = await getHTMLByURL(url, {
+    ...other,
+    shouldReGetHTML: async html => {
+      const meta = await getMetaDataByHTML(html, url, {
+        customRules,
+        forceImageHttps,
+      });
+      return shouldReGetHTML ? await shouldReGetHTML(meta) : false;
+    },
+  }).catch(() => {
+    //   TODO: report error
+    return '';
+  });
+
+  return await getMetaDataByHTML(html, url, {
+    customRules,
+    forceImageHttps,
+  });
+}