refactor(i18n): lazy load languages (#8456)

closes AF-1397

packages/frontend/i18n/src/i18n-completenesses.json

@@ -0,0 +1,22 @@
+{
+  "ar": 87,
+  "ca": 6,
+  "da": 7,
+  "de": 33,
+  "en": 100,
+  "es-AR": 14,
+  "es-CL": 16,
+  "es": 14,
+  "fr": 78,
+  "hi": 2,
+  "it": 1,
+  "ja": 28,
+  "ko": 92,
+  "pl": 0,
+  "pt-BR": 100,
+  "ru": 85,
+  "sv-SE": 5,
+  "ur": 3,
+  "zh-Hans": 99,
+  "zh-Hant": 21
+}

packages/frontend/i18n/src/i18n.ts

@@ -1,86 +0,0 @@
-import { useMemo } from 'react';
-import { getI18n, useTranslation } from 'react-i18next';
-
-import type { useAFFiNEI18N } from './i18n-generated';
-
-export type I18nFuncs = ReturnType<typeof useAFFiNEI18N>;
-
-export type I18nInfos = {
-  [K in keyof I18nFuncs]: I18nFuncs[K] extends (...a: infer Opt) => any
-    ? Opt[0]
-    : never;
-};
-
-export type I18nKeys = keyof I18nInfos;
-
-export type I18nString =
-  | {
-      [K in I18nKeys]: {
-        key: K;
-      } & (I18nInfos[K] extends undefined
-        ? unknown
-        : { options: I18nInfos[K] });
-    }[I18nKeys]
-  | string;
-
-export const isI18nString = (value: any): value is I18nString => {
-  return (
-    typeof value === 'string' || (typeof value === 'object' && 'key' in value)
-  );
-};
-
-function createI18nWrapper(
-  getI18nFn: () => ReturnType<typeof getI18n>,
-  getI18nT: () => ReturnType<typeof getI18n>['t']
-) {
-  const I18nMethod = {
-    t(i18nStr: I18nString) {
-      const i18n = getI18nFn();
-      if (typeof i18nStr === 'object') {
-        return i18n.t(i18nStr.key, 'options' in i18nStr ? i18nStr.options : {});
-      }
-      return i18nStr;
-    },
-    get language() {
-      const i18n = getI18nFn();
-      return i18n.language;
-    },
-    changeLanguage(lng?: string | undefined) {
-      const i18n = getI18nFn();
-      return i18n.changeLanguage(lng);
-    },
-    get on() {
-      const i18n = getI18nFn();
-      return i18n.on.bind(i18n);
-    },
-  };
-
-  return new Proxy(I18nMethod, {
-    get(self, key) {
-      const i18n = getI18nFn();
-      if (typeof key === 'string' && i18n.exists(key)) {
-        return getI18nT().bind(null, key as string);
-      } else {
-        return (self as any)[key as string] as any;
-      }
-    },
-  }) as I18nFuncs & typeof I18nMethod;
-}
-
-export const useI18n = () => {
-  const { i18n, t } = useTranslation();
-  return useMemo(
-    () =>
-      createI18nWrapper(
-        () => i18n,
-        () => t
-      ),
-    [i18n, t]
-  );
-};
-
-/**
- * I18n['com.affine.xxx']({ arg1: 'hello' }) -> '中文 hello'
- */
-export const I18n = createI18nWrapper(getI18n, () => getI18n().t);
-export type I18n = typeof I18n;

packages/frontend/i18n/src/i18next.ts

@@ -0,0 +1,161 @@
+import { DebugLogger } from '@affine/debug';
+import type { BackendModule, i18n } from 'i18next';
+import i18next from 'i18next';
+import { initReactI18next } from 'react-i18next';
+
+import type { useAFFiNEI18N } from './i18n-generated';
+import type { Language } from './resources';
+import { SUPPORTED_LANGUAGES } from './resources';
+
+const logger = new DebugLogger('i18n');
+
+const defaultLng: Language = 'en';
+
+let _instance: i18n | null = null;
+export const getOrCreateI18n = (): i18n => {
+  if (!_instance) {
+    _instance = i18next.createInstance();
+    _instance
+      .use(initReactI18next)
+      .use({
+        type: 'backend',
+        init: () => {},
+        read: (lng: Language, _ns: string, callback) => {
+          const resource = SUPPORTED_LANGUAGES[lng].resource;
+          if (typeof resource === 'function') {
+            resource()
+              .then(data => {
+                logger.info(`Loaded i18n ${lng} resource`);
+                callback(null, data.default);
+              })
+              .catch(err => {
+                logger.error(`Failed to load i18n ${lng} resource`, err);
+                callback(null, null);
+              });
+          } else {
+            callback(null, resource);
+          }
+        },
+      } as BackendModule)
+      .init({
+        lng: defaultLng,
+        fallbackLng: code => {
+          // always fallback to english
+          const fallbacks: string[] = [defaultLng];
+          const langPart = code.split('-')[0];
+
+          // fallback xx-YY to xx, e.g. es-AR to es
+          // fallback zh-Hant to zh-Hans
+          if (langPart === 'cn') {
+            fallbacks.push('zh-Hans');
+          } else if (
+            langPart !== code &&
+            SUPPORTED_LANGUAGES[code as Language]
+          ) {
+            fallbacks.unshift(langPart);
+          }
+
+          return fallbacks;
+        },
+        supportedLngs: Object.keys(SUPPORTED_LANGUAGES),
+        debug: false,
+        partialBundledLanguages: true,
+        resources: {
+          [defaultLng]: {
+            translation: SUPPORTED_LANGUAGES[defaultLng].resource,
+          },
+        },
+        interpolation: {
+          escapeValue: false, // not needed for react as it escapes by default
+        },
+      })
+      .then(() => {
+        logger.info('i18n initialized');
+      })
+      .catch(() => {});
+  }
+
+  return _instance;
+};
+
+declare module 'i18next' {
+  interface CustomTypeOptions {
+    // NOTE(@forehalo):
+    //   DO NOT ENABLE THIS
+    //   This could bring typecheck for <Trans /> component,
+    //   but it will make typecheck of whole codebase so laggy!
+    //   check [./react.ts]
+    // resources: {
+    //   translation: LanguageResource;
+    // };
+  }
+}
+
+export type I18nFuncs = ReturnType<typeof useAFFiNEI18N>;
+type KnownI18nKey = keyof I18nFuncs;
+
+export type I18nString =
+  | KnownI18nKey
+  | string
+  | { i18nKey: string; options?: Record<string, any> };
+
+export function isI18nString(value: unknown): value is I18nString {
+  if (typeof value === 'string') {
+    return true;
+  }
+
+  if (typeof value === 'object' && value !== null) {
+    return 'i18nKey' in value;
+  }
+
+  return false;
+}
+
+export function createI18nWrapper(getI18nFn: () => i18n) {
+  const I18nMethod = {
+    t(key: I18nString, options?: Record<string, any>) {
+      if (typeof key === 'object' && 'i18nKey' in key) {
+        options = key.options;
+        key = key.i18nKey as string;
+      }
+
+      const i18n = getI18nFn();
+      if (i18n.exists(key)) {
+        return i18n.t(key, options);
+      } else {
+        // unknown translate key 'xxx.xxx' returns itself
+        return key;
+      }
+    },
+    get language() {
+      const i18n = getI18nFn();
+      return i18n.language;
+    },
+    changeLanguage(lng?: string | undefined) {
+      const i18n = getI18nFn();
+      return i18n.changeLanguage(lng);
+    },
+    get on() {
+      const i18n = getI18nFn();
+      return i18n.on.bind(i18n);
+    },
+  };
+
+  return new Proxy(I18nMethod, {
+    get(self, key: string) {
+      if (key in self) {
+        // @ts-expect-error allow
+        return self[key];
+      }
+
+      return I18nMethod.t.bind(null, key);
+    },
+  }) as typeof I18nMethod &
+    ReturnType<typeof useAFFiNEI18N> & { [unknownKey: string]: () => string };
+}
+
+/**
+ * I18n['com.affine.xxx']({ arg1: 'hello' }) -> '中文 hello'
+ */
+export const I18n = createI18nWrapper(getOrCreateI18n);
+export type I18n = typeof I18n;

packages/frontend/i18n/src/index.ts

@@ -1,122 +1,7 @@
-import type { i18n, Resource } from 'i18next';
-import i18next from 'i18next';
-import type { I18nextProviderProps } from 'react-i18next';
-import { I18nextProvider, initReactI18next, Trans } from 'react-i18next';
-
-import { LOCALES } from './resources';
-import type en_US from './resources/en.json';
-
-export * from './i18n';
+export * from './i18next';
+export * from './react';
+export * from './resources';
 export * from './utils';
+import completenesses from './i18n-completenesses.json';
 
-declare module 'i18next' {
-  // Refs: https://www.i18next.com/overview/typescript#argument-of-type-defaulttfuncreturn-is-not-assignable-to-parameter-of-type-xyz
-  interface CustomTypeOptions {
-    returnNull: false;
-  }
-}
-
-// const localStorage = {
-//   getItem() {
-//     return undefined;
-//   },
-//   setItem() {},
-// };
-// See https://react.i18next.com/latest/typescript
-declare module 'react-i18next' {
-  interface CustomTypeOptions {
-    // custom namespace type if you changed it
-    // defaultNS: 'ns1';
-    // custom resources type
-    allowObjectInHTMLChildren: true;
-    resources: {
-      en: typeof en_US;
-    };
-  }
-}
-
-const STORAGE_KEY = 'i18n_lng';
-
-export { I18nextProvider, LOCALES, Trans };
-
-const resources = LOCALES.reduce<Resource>((acc, { tag, res }) => {
-  return Object.assign(acc, { [tag]: { translation: res } });
-}, {});
-
-const fallbackLng = 'en';
-const standardizeLocale = (language: string) => {
-  if (language === 'zh-CN' || language === 'zh' || language === 'zh-Hans') {
-    language = 'zh-Hans';
-  } else if (language.slice(0, 2).toLowerCase() === 'zh') {
-    language = 'zh-Hant';
-  }
-  if (LOCALES.some(locale => locale.tag === language)) return language;
-  if (
-    LOCALES.some(locale => locale.tag === language.slice(0, 2).toLowerCase())
-  ) {
-    return language.slice(0, 2).toLowerCase();
-  }
-
-  return fallbackLng;
-};
-
-export const createI18n = (): I18nextProviderProps['i18n'] => {
-  const i18n: I18nextProviderProps['i18n'] = i18next.createInstance();
-  i18n
-    .use(initReactI18next)
-    .init({
-      lng: 'en',
-      fallbackLng,
-      debug: false,
-      resources,
-      interpolation: {
-        escapeValue: false, // not needed for react as it escapes by default
-      },
-    })
-    .then(() => {
-      console.info('i18n init success');
-    })
-    .catch(() => {
-      console.error('i18n init failed');
-    });
-
-  if (globalThis.localStorage) {
-    i18n.on('languageChanged', lng => {
-      localStorage.setItem(STORAGE_KEY, lng);
-    });
-  }
-  return i18n;
-};
-
-export function setUpLanguage(i: i18n) {
-  let language;
-  const localStorageLanguage = localStorage.getItem(STORAGE_KEY);
-  if (localStorageLanguage) {
-    language = standardizeLocale(localStorageLanguage);
-  } else {
-    language = standardizeLocale(navigator.language);
-  }
-  return i.changeLanguage(language);
-}
-
-const cachedCompleteness: Record<string, number> = {};
-export const calcLocaleCompleteness = (
-  locale: (typeof LOCALES)[number]['tag']
-) => {
-  if (cachedCompleteness[locale]) {
-    return cachedCompleteness[locale];
-  }
-  const base = LOCALES.find(item => item.base);
-  if (!base) {
-    throw new Error('Base language not found');
-  }
-  const target = LOCALES.find(item => item.tag === locale);
-  if (!target) {
-    throw new Error('Locale not found');
-  }
-  const baseKeyCount = Object.keys(base.res).length;
-  const translatedKeyCount = Object.keys(target.res).length;
-  const completeness = translatedKeyCount / baseKeyCount;
-  cachedCompleteness[target.tag] = completeness;
-  return completeness;
-};
+export const i18nCompletenesses = completenesses;

packages/frontend/i18n/src/react.ts

@@ -0,0 +1,12 @@
+import { useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+
+import { createI18nWrapper } from './i18next';
+
+export const useI18n = () => {
+  const { i18n } = useTranslation('translation');
+
+  return useMemo(() => createI18nWrapper(() => i18n), [i18n]);
+};
+
+export { I18nextProvider, Trans } from 'react-i18next';

packages/frontend/i18n/src/resources/index.ts

@@ -1,174 +1,153 @@
-import ar from './ar.json';
-import ca from './ca.json';
-import da from './da.json';
-import de from './de.json';
 import en from './en.json';
-import es from './es.json';
-import es_AR from './es-AR.json';
-import es_CL from './es-CL.json';
-import fr from './fr.json';
-import hi from './hi.json';
-import it from './it.json';
-import ja from './ja.json';
-import ko from './ko.json';
-import pt_BR from './pt-BR.json';
-import ru from './ru.json';
-import sv_SE from './sv-SE.json';
-import ur from './ur.json';
-import zh_Hans from './zh-Hans.json';
-import zh_Hant from './zh-Hant.json';
 
-export const LOCALES = [
-  {
-    name: 'Korean (South Korea)',
-    tag: 'ko',
-    originalName: '한국어(대한민국)',
-    flagEmoji: '🇰🇷',
-    base: false,
-    res: ko,
-  },
-  {
-    name: 'Portuguese (Brazil)',
-    tag: 'pt-BR',
-    originalName: 'português (Brasil)',
-    flagEmoji: '🇧🇷',
-    base: false,
-    res: pt_BR,
-  },
+export type Language =
+  | 'en'
+  | 'zh-Hans'
+  | 'zh-Hant'
+  | 'fr'
+  | 'es'
+  | 'es-AR'
+  | 'es-CL'
+  | 'de'
+  | 'ru'
+  | 'ja'
+  | 'it'
+  | 'ca'
+  | 'da'
+  | 'hi'
+  | 'sv-SE'
+  | 'ur'
+  | 'ar'
+  | 'ko'
+  | 'pt-BR';
+
+export type LanguageResource = typeof en;
+export const SUPPORTED_LANGUAGES: Record<
+  Language,
   {
+    name: string;
+    originalName: string;
+    flagEmoji: string;
+    resource:
+      | LanguageResource
+      | (() => Promise<{ default: Partial<LanguageResource> }>);
+  }
+> = {
+  en: {
     name: 'English',
-    tag: 'en',
     originalName: 'English',
     flagEmoji: '🇬🇧',
-    base: true,
-    res: en,
+    resource: en,
   },
-  {
-    name: 'Traditional Chinese',
-    tag: 'zh-Hant',
-    originalName: '繁體中文',
-    flagEmoji: '🇭🇰',
-    base: false,
-    res: zh_Hant,
+  ko: {
+    name: 'Korean (South Korea)',
+    originalName: '한국어(대한민국)',
+    flagEmoji: '🇰🇷',
+    resource: () => /* webpackChunkName "i18n-ko" */ import('./ko.json'),
   },
-  {
+  'pt-BR': {
+    name: 'Portuguese (Brazil)',
+    originalName: 'português (Brasil)',
+    flagEmoji: '🇧🇷',
+    resource: () => /* webpackChunkName "i18n-pt_BR" */ import('./pt-BR.json'),
+  },
+  'zh-Hans': {
     name: 'Simplified Chinese',
-    tag: 'zh-Hans',
     originalName: '简体中文',
     flagEmoji: '🇨🇳',
-    base: false,
-    res: zh_Hans,
+    resource: () =>
+      /* webpackChunkName "i18n-zh_Hans" */ import('./zh-Hans.json'),
   },
-  {
+  'zh-Hant': {
+    name: 'Traditional Chinese',
+    originalName: '繁體中文',
+    flagEmoji: '🇭🇰',
+    resource: () =>
+      /* webpackChunkName "i18n-zh_Hant" */ import('./zh-Hant.json'),
+  },
+  fr: {
     name: 'French',
-    tag: 'fr',
     originalName: 'français',
     flagEmoji: '🇫🇷',
-    base: false,
-    res: fr,
+    resource: () => /* webpackChunkName "i18n-fr" */ import('./fr.json'),
   },
-  {
+  es: {
     name: 'Spanish',
-    tag: 'es',
     originalName: 'español',
     flagEmoji: '🇪🇸',
-    base: false,
-    res: es,
+    resource: () => /* webpackChunkName "i18n-es" */ import('./es.json'),
   },
-  {
-    name: 'German',
-    tag: 'de',
-    originalName: 'Deutsch',
-    flagEmoji: '🇩🇪',
-    base: false,
-    res: de,
-  },
-  {
-    name: 'Russian',
-    tag: 'ru',
-    originalName: 'русский',
-    flagEmoji: '🇷🇺',
-    base: false,
-    res: ru,
-  },
-  {
-    name: 'Japanese',
-    tag: 'ja',
-    originalName: '日本語',
-    flagEmoji: '🇯🇵',
-    base: false,
-    res: ja,
-  },
-  {
-    name: 'Italian',
-    tag: 'it',
-    originalName: 'italiano',
-    flagEmoji: '🇮🇹',
-    base: false,
-    res: it,
-  },
-  {
-    name: 'Catalan',
-    tag: 'ca',
-    originalName: 'català',
-    flagEmoji: '🇦🇩',
-    base: false,
-    res: ca,
-  },
-  {
-    name: 'Danish',
-    tag: 'da',
-    originalName: 'dansk',
-    flagEmoji: '🇩🇰',
-    base: false,
-    res: da,
-  },
-  {
-    name: 'Spanish (Chile)',
-    tag: 'es-CL',
-    originalName: 'español (Chile)',
-    flagEmoji: '🇨🇱',
-    base: false,
-    res: es_CL,
-  },
-  {
-    name: 'Hindi',
-    tag: 'hi',
-    originalName: 'हिन्दी',
-    flagEmoji: '🇮🇳',
-    base: false,
-    res: hi,
-  },
-  {
-    name: 'Swedish (Sweden)',
-    tag: 'sv-SE',
-    originalName: 'svenska (Sverige)',
-    flagEmoji: '🇸🇪',
-    base: false,
-    res: sv_SE,
-  },
-  {
+  'es-AR': {
     name: 'Spanish (Argentina)',
-    tag: 'es-AR',
     originalName: 'español (Argentina)',
     flagEmoji: '🇦🇷',
-    base: false,
-    res: es_AR,
+    resource: () => /* webpackChunkName "i18n-es_AR" */ import('./es-AR.json'),
   },
-  {
+  'es-CL': {
+    name: 'Spanish (Chile)',
+    originalName: 'español (Chile)',
+    flagEmoji: '🇨🇱',
+    resource: () => /* webpackChunkName "i18n-es_CL" */ import('./es-CL.json'),
+  },
+  de: {
+    name: 'German',
+    originalName: 'Deutsch',
+    flagEmoji: '🇩🇪',
+    resource: () => /* webpackChunkName "i18n-de" */ import('./de.json'),
+  },
+  ru: {
+    name: 'Russian',
+    originalName: 'русский',
+    flagEmoji: '🇷🇺',
+    resource: () => /* webpackChunkName "i18n-ru" */ import('./ru.json'),
+  },
+  ja: {
+    name: 'Japanese',
+    originalName: '日本語',
+    flagEmoji: '🇯🇵',
+    resource: () => /* webpackChunkName "i18n-ja" */ import('./ja.json'),
+  },
+  it: {
+    name: 'Italian',
+    originalName: 'italiano',
+    flagEmoji: '🇮🇹',
+    resource: () => /* webpackChunkName "i18n-it" */ import('./it.json'),
+  },
+  ca: {
+    name: 'Catalan',
+    originalName: 'català',
+    flagEmoji: '🇦🇩',
+    resource: () => /* webpackChunkName "i18n-ca" */ import('./ca.json'),
+  },
+  da: {
+    name: 'Danish',
+    originalName: 'dansk',
+    flagEmoji: '🇩🇰',
+    resource: () => /* webpackChunkName "i18n-da" */ import('./da.json'),
+  },
+  hi: {
+    name: 'Hindi',
+    originalName: 'हिन्दी',
+    flagEmoji: '🇮🇳',
+    resource: () => /* webpackChunkName "i18n-hi" */ import('./hi.json'),
+  },
+  'sv-SE': {
+    name: 'Swedish (Sweden)',
+    originalName: 'svenska (Sverige)',
+    flagEmoji: '🇸🇪',
+    resource: () => /* webpackChunkName "i18n-sv_SE" */ import('./sv-SE.json'),
+  },
+
+  ur: {
     name: 'Urdu',
-    tag: 'ur',
     originalName: 'اردو',
     flagEmoji: '🇵🇰',
-    base: false,
-    res: ur,
+    resource: () => /* webpackChunkName "i18n-ur" */ import('./ur.json'),
   },
-  {
+  ar: {
     name: 'Arabic',
-    tag: 'ar',
     originalName: 'العربية',
     flagEmoji: '🇸🇦',
-    base: false,
-    res: ar,
+    resource: () => /* webpackChunkName "i18n-ar" */ import('./ar.json'),
   },
-] as const;
+};

packages/frontend/i18n/src/utils/__tests__/time.spec.ts

@@ -1,12 +1,12 @@
 import { describe, expect, test } from 'vitest';
 
-import { createI18n, I18n } from '../../';
+import { getOrCreateI18n, I18n } from '../../';
 import { i18nTime } from '../time';
 
 // Intl api is not available in github action, skip the test
 describe('humanTime', () => {
   test('absolute', async () => {
-    createI18n();
+    getOrCreateI18n();
     expect(i18nTime('2024-10-10 13:30:28')).toBe('Oct 10, 2024, 1:30:28 PM');
     expect(
       i18nTime('2024-10-10 13:30:28', {
@@ -48,7 +48,7 @@ describe('humanTime', () => {
   });
 
   test('relative', async () => {
-    createI18n();
+    getOrCreateI18n();
     expect(
       i18nTime('2024-10-10 13:30:28.005', {
         now: '2024-10-10 13:30:30',
@@ -148,7 +148,7 @@ describe('humanTime', () => {
   });
 
   test('relative - accuracy', async () => {
-    createI18n();
+    getOrCreateI18n();
     expect(
       i18nTime('2024-10-10 13:30:28.005', {
         now: '2024-10-10 13:30:30',
@@ -224,7 +224,7 @@ describe('humanTime', () => {
   });
 
   test('relative - disable yesterdayAndTomorrow', async () => {
-    createI18n();
+    getOrCreateI18n();
     expect(
       i18nTime('2024-10-9 13:30:30', {
         now: '2024-10-10 13:30:30',
@@ -244,7 +244,7 @@ describe('humanTime', () => {
   });
 
   test('relative - weekday', async () => {
-    createI18n();
+    getOrCreateI18n();
     expect(
       i18nTime('2024-10-9 13:30:30', {
         now: '2024-10-10 13:30:30',
@@ -302,7 +302,7 @@ describe('humanTime', () => {
   });
 
   test('mix relative and absolute', async () => {
-    createI18n();
+    getOrCreateI18n();
     expect(
       i18nTime('2024-10-9 14:30:30', {
         now: '2024-10-10 13:30:30',
@@ -348,9 +348,9 @@ describe('humanTime', () => {
     ).toBe('Oct 8, 2024');
   });
 
-  test('chinese', () => {
-    createI18n();
-    I18n.changeLanguage('zh-Hans');
+  test('chinese', async () => {
+    getOrCreateI18n();
+    await I18n.changeLanguage('zh-Hans');
     expect(i18nTime('2024-10-10 13:30:28.005')).toBe('2024年10月10日 13:30:28');
     expect(
       i18nTime('2024-10-10 13:30:28.005', {
@@ -398,7 +398,7 @@ describe('humanTime', () => {
   });
 
   test('invalid time', () => {
-    createI18n();
+    getOrCreateI18n();
     expect(i18nTime('foobar')).toBe('');
   });
 });

packages/frontend/i18n/src/utils/time.ts

@@ -1,6 +1,6 @@
 import dayjs from 'dayjs';
 
-import { I18n } from '../i18n';
+import { I18n } from '../i18next';
 
 export type TimeUnit =
   | 'second'