Merge pull request #379 from toeverything/feat/i18n

Feat/i18n

libs/components/layout/package.json

@@ -11,9 +11,7 @@
         "@mui/icons-material": "^5.8.4",
         "clsx": "^1.2.1",
         "date-fns": "^2.29.2",
-        "i18next": "^21.9.1",
         "jotai": "^1.8.1",
-        "react-i18next": "^11.18.4",
         "tinycolor2": "^1.4.2",
         "turndown": "7.1.1"
     },

libs/components/layout/src/header/FileSystem.tsx

@@ -3,8 +3,9 @@ import { useCallback, useMemo, useState } from 'react';
 import { CloseIcon } from '@toeverything/components/common';
 import { IconButton, MuiSnackbar, styled } from '@toeverything/components/ui';
 import { services } from '@toeverything/datasource/db-service';
+import { useTranslation } from '@toeverything/datasource/i18n';
 import { useLocalTrigger } from '@toeverything/datasource/state';
-import { useTranslation } from 'react-i18next';
+
 const cleanupWorkspace = (workspace: string) =>
     new Promise((resolve, reject) => {
         const req = indexedDB.deleteDatabase(workspace);

libs/components/layout/src/header/LayoutHeader.tsx

@@ -5,13 +5,13 @@ import {
     SideBarViewIcon,
 } from '@toeverything/components/icons';
 import { IconButton, styled } from '@toeverything/components/ui';
+import { useTranslation } from '@toeverything/datasource/i18n';
 import {
     useCurrentEditors,
     useLocalTrigger,
     useShowSettingsSidebar,
 } from '@toeverything/datasource/state';
 import { useCallback, useMemo } from 'react';
-import { useTranslation } from 'react-i18next';
 import { EditorBoardSwitcher } from './EditorBoardSwitcher';
 import { fsApiSupported } from './FileSystem';
 import { CurrentPageTitle } from './Title';

libs/components/layout/src/i18n/index.ts

@@ -1,40 +0,0 @@
-import i18next from 'i18next';
-import { initReactI18next } from 'react-i18next';
-import en_US from './resources/en.json';
-import zh_CN from './resources/zh.json';
-
-// See https://react.i18next.com/latest/typescript
-declare module 'react-i18next' {
-    // and extend them!
-    interface CustomTypeOptions {
-        // custom namespace type if you changed it
-        defaultNS: 'ns1';
-        // custom resources type
-        resources: {
-            en: typeof en_US.translation;
-            zh: typeof zh_CN.translation;
-        };
-    }
-}
-
-const resources = {
-    en: en_US,
-    zh: zh_CN,
-} as const;
-
-i18next.use(initReactI18next).init({
-    lng: 'en',
-    fallbackLng: 'en',
-
-    resources,
-    interpolation: {
-        escapeValue: false, // not needed for react as it escapes by default
-    },
-});
-
-export const options = [
-    { value: 'en', text: 'English' },
-    { value: 'zh', text: '简体中文' },
-] as const;
-
-export { i18next };

libs/components/layout/src/i18n/resources/en.json

@@ -1,26 +0,0 @@
-{
-    "translation": {
-        "Sync to Disk": "Sync to Disk",
-        "Share": "Share",
-        "WarningTips": {
-            "IsNotfsApiSupported": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC DATA TO DISK with the latest version of Chromium based browser like Chrome/Edge",
-            "IsNotLocalWorkspace": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC TO DISK.",
-            "DoNotStore": "AFFiNE is under active development and the current version is UNSTABLE. Please DO NOT store information or data"
-        },
-        "Layout": "Layout",
-        "Comment": "Comment",
-        "Settings": "Settings",
-        "ComingSoon": "Layout Settings Coming Soon...",
-        "Duplicate Page": "Duplicate Page",
-        "Copy Page Link": "Copy Page Link",
-        "Language": "Language",
-        "Clear Workspace": "Clear Workspace",
-        "Export As Markdown": "Export As Markdown",
-        "Export As HTML": "Export As HTML",
-        "Export As PDF (Unsupported)": "Export As PDF (Unsupported)",
-        "Import Workspace": "Import Workspace",
-        "Export Workspace": "Export Workspace",
-        "Last edited by": "Last edited by {{name}}",
-        "Logout": "Logout"
-    }
-}

libs/components/layout/src/i18n/resources/zh.json

@@ -1,26 +0,0 @@
-{
-    "translation": {
-        "Sync to Disk": "同步到磁盘",
-        "Share": "分享",
-        "WarningTips": {
-            "IsNotfsApiSupported": "欢迎来到AFFiNE 的演示界面。您可以使用最新版本的基于Chrome的浏览器（如Chrome/Edge）将数据同步到磁盘来进行保存",
-            "IsNotLocalWorkspace": "欢迎来到AFFiNE 的演示界面，您可以同步到磁盘来进行保存操作。",
-            "DoNotStore": "AFFINE 正在积极开发中，当前版本不稳定。请不要存储信息或数据。"
-        },
-        "ComingSoon": "布局设置即将到来",
-        "Layout": "布局",
-        "Comment": "评论",
-        "Settings": "设置",
-        "Duplicate Page": "复制页面",
-        "Copy Page Link": "复制页面链接",
-        "Language": "当前语言",
-        "Clear Workspace": "清空工作区域",
-        "Export As Markdown": "导出 markdown",
-        "Export As HTML": "导出 HTML",
-        "Export As PDF (Unsupported)": "导出 PDF (暂不支持)",
-        "Import Workspace": "导入 Workspace",
-        "Export Workspace": "导出 Workspace",
-        "Last edited by": "最后编辑者为 {{name}}",
-        "Logout": "退出登录"
-    }
-}

libs/components/layout/src/index.ts

@@ -1,4 +1,3 @@
 export * from './header';
-export * from './i18n';
 export * from './settings-sidebar';
 export * from './workspace-sidebar';

libs/components/layout/src/settings-sidebar/ContainerTabs/ContainerTabs.tsx

@@ -4,8 +4,8 @@ import {
     SettingsIcon,
 } from '@toeverything/components/icons';
 import { styled } from '@toeverything/components/ui';
+import { useTranslation } from '@toeverything/datasource/i18n';
 import { cloneElement, useCallback, useMemo, type ReactElement } from 'react';
-import { useTranslation } from 'react-i18next';
 import { Comments } from '../Comments';
 import { useActiveComment } from '../Comments/use-comments';
 import { LayoutSettings } from '../Layout';

libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx

@@ -6,8 +6,7 @@ import {
     styled,
     Switch,
 } from '@toeverything/components/ui';
-import { useTranslation } from 'react-i18next';
+import { LOCALES, useTranslation } from '@toeverything/datasource/i18n';
-import { options } from '../../i18n';
 import { useSettings } from './use-settings';
 
 export const SettingsList = () => {
@@ -52,7 +51,7 @@ export const SettingsList = () => {
                                     defaultValue={i18n.language}
                                     onChange={changeLanguage}
                                 >
-                                    {options.map(option => (
+                                    {LOCALES.map(option => (
                                         <Option
                                             key={option.value}
                                             value={option.value}

libs/components/layout/src/settings-sidebar/Settings/footer/LastModified.tsx

@@ -1,7 +1,7 @@
 import { styled, Typography } from '@toeverything/components/ui';
+import { useTranslation } from '@toeverything/datasource/i18n';
 import { useUserAndSpaces } from '@toeverything/datasource/state';
 import format from 'date-fns/format';
-import { useTranslation } from 'react-i18next';
 import { usePageLastUpdated, useWorkspaceAndPageId } from '../util';
 
 export const LastModified = () => {

libs/components/layout/src/settings-sidebar/Settings/footer/Logout.tsx

@@ -1,8 +1,8 @@
 import { MoveToIcon } from '@toeverything/components/icons';
 import { ListItem, styled, Typography } from '@toeverything/components/ui';
+import { useTranslation } from '@toeverything/datasource/i18n';
 import { LOGOUT_COOKIES, LOGOUT_LOCAL_STORAGE } from '@toeverything/utils';
 import { getAuth, signOut } from 'firebase/auth';
-import { useTranslation } from 'react-i18next';
 
 const logout = () => {
     LOGOUT_LOCAL_STORAGE.forEach(name => localStorage.removeItem(name));

libs/components/layout/src/settings-sidebar/Settings/use-settings.ts

@@ -1,6 +1,6 @@
 import { message } from '@toeverything/components/ui';
+import { useTranslation } from '@toeverything/datasource/i18n';
 import { copyToClipboard } from '@toeverything/utils';
-import { useTranslation } from 'react-i18next';
 import { useNavigate } from 'react-router-dom';
 import { useSettingFlags, type SettingFlags } from './use-setting-flags';
 

libs/datasource/i18n/.babelrc

@@ -0,0 +1,12 @@
+{
+    "presets": [
+        [
+            "@nrwl/react/babel",
+            {
+                "runtime": "automatic",
+                "useBuiltIns": "usage"
+            }
+        ]
+    ],
+    "plugins": []
+}

libs/datasource/i18n/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+    "extends": ["plugin:@nrwl/nx/react", "../../../.eslintrc.json"],
+    "ignorePatterns": ["!**/*"],
+    "overrides": [
+        {
+            "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.ts", "*.tsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.js", "*.jsx"],
+            "rules": {}
+        }
+    ]
+}

libs/datasource/i18n/README.md

@@ -0,0 +1,50 @@
+# i18n
+
+## Usages
+
+```tsx
+import { useTranslation } from '@toeverything/datasource/i18n';
+
+// base.json
+// {
+//     'Text': 'some text',
+//     'Switch to language': 'Switch to {language}',
+// };
+
+const App = () => {
+    const { t } = useTranslation();
+
+    const changeLanguage = (language: string) => {
+        i18n.changeLanguage(language);
+    };
+
+    return (
+        <div>
+            <div>{t('Text')}</div>
+
+            <button onClick={() => changeLanguage('en')}>
+                {t('Switch to language', { language: 'en' })}
+            </button>
+            <button onClick={() => changeLanguage('zh-Hans')}>
+                {t('Switch to language', { language: 'zh-Hans' })}
+            </button>
+        </div>
+    );
+};
+```
+
+## How to sync translations
+
+-   Set token as environment variable
+
+```shell
+export TOLGEE_API_KEY=tgpak_XXXXXXX
+```
+
+-   Run the `sync` script from package script
+
+## References
+
+-   [i18next](https://www.i18next.com/)
+-   [react-i18next](https://react.i18next.com/)
+-   [Tolgee](https://tolgee.io/docs/)

libs/datasource/i18n/jest.config.ts

@@ -0,0 +1,10 @@
+/* eslint-disable */
+export default {
+    displayName: 'datasource-i18n',
+    preset: '../../../jest.preset.js',
+    transform: {
+        '^.+\\.[tj]sx?$': 'babel-jest',
+    },
+    moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
+    coverageDirectory: '../../../coverage/libs/datasource/i18n',
+};

libs/datasource/i18n/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "@toeverything/datasource/i18n",
+    "version": "0.0.1",
+    "scripts": {
+        "sync": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/sync.ts"
+    },
+    "dependencies": {
+        "i18next": "^21.9.1",
+        "jotai": "^1.8.1",
+        "react-i18next": "^11.18.4"
+    }
+}

libs/datasource/i18n/project.json

@@ -0,0 +1,45 @@
+{
+    "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "libs/datasource/i18n/src",
+    "projectType": "library",
+    "tags": ["datasource"],
+    "targets": {
+        "build": {
+            "executor": "@nrwl/web:rollup",
+            "outputs": ["{options.outputPath}"],
+            "options": {
+                "outputPath": "dist/libs/datasource/i18n",
+                "tsConfig": "libs/datasource/i18n/tsconfig.lib.json",
+                "project": "libs/datasource/i18n/package.json",
+                "entryFile": "libs/datasource/i18n/src/index.ts",
+                "external": ["react/jsx-runtime"],
+                "rollupConfig": "@nrwl/react/plugins/bundle-rollup",
+                "compiler": "babel",
+                "assets": [
+                    {
+                        "glob": "libs/datasource/i18n/README.md",
+                        "input": ".",
+                        "output": "."
+                    }
+                ]
+            }
+        },
+        "lint": {
+            "executor": "@nrwl/linter:eslint",
+            "outputs": ["{options.outputFile}"],
+            "options": {
+                "lintFilePatterns": [
+                    "libs/datasource/i18n/**/*.{ts,tsx,js,jsx}"
+                ]
+            }
+        },
+        "test": {
+            "executor": "@nrwl/jest:jest",
+            "outputs": ["coverage/libs/datasource/i18n"],
+            "options": {
+                "jestConfig": "libs/datasource/i18n/jest.config.ts",
+                "passWithNoTests": true
+            }
+        }
+    }
+}

libs/datasource/i18n/src/base.json

@@ -0,0 +1,24 @@
+{
+    "Sync to Disk": "Sync to Disk",
+    "Share": "Share",
+    "WarningTips": {
+        "IsNotfsApiSupported": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC DATA TO DISK with the latest version of Chromium based browser like Chrome/Edge",
+        "IsNotLocalWorkspace": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC TO DISK.",
+        "DoNotStore": "AFFiNE is under active development and the current version is UNSTABLE. Please DO NOT store information or data"
+    },
+    "Layout": "Layout",
+    "Comment": "Comment",
+    "Settings": "Settings",
+    "ComingSoon": "Layout Settings Coming Soon...",
+    "Duplicate Page": "Duplicate Page",
+    "Copy Page Link": "Copy Page Link",
+    "Language": "Language",
+    "Clear Workspace": "Clear Workspace",
+    "Export As Markdown": "Export As Markdown",
+    "Export As HTML": "Export As HTML",
+    "Export As PDF (Unsupported)": "Export As PDF (Unsupported)",
+    "Import Workspace": "Import Workspace",
+    "Export Workspace": "Export Workspace",
+    "Last edited by": "Last edited by {{name}}",
+    "Logout": "Logout"
+}

libs/datasource/i18n/src/index.ts

@@ -0,0 +1,47 @@
+import i18next, { Resource } from 'i18next';
+import {
+    I18nextProvider,
+    initReactI18next,
+    useTranslation,
+} from 'react-i18next';
+import en_US from './resources/en.json';
+import zh_CN from './resources/zh.json';
+
+// See https://react.i18next.com/latest/typescript
+declare module 'react-i18next' {
+    interface CustomTypeOptions {
+        // custom namespace type if you changed it
+        // defaultNS: 'ns1';
+        // custom resources type
+        resources: {
+            en: typeof en_US;
+        };
+    }
+}
+
+const LOCALES = [
+    { value: 'en', text: 'English', res: en_US },
+    { value: 'zh', text: '简体中文', res: zh_CN },
+] as const;
+
+const resources = LOCALES.reduce<Resource>(
+    (acc, { value, res }) => ({ ...acc, [value]: { translation: res } }),
+    {}
+);
+
+const i18n = i18next.createInstance();
+i18n.use(initReactI18next).init({
+    // TODO auto detect
+    lng: LOCALES[0].value,
+    fallbackLng: LOCALES[0].value,
+    debug: process.env['NODE_ENV'] === 'development',
+
+    resources,
+    interpolation: {
+        escapeValue: false, // not needed for react as it escapes by default
+    },
+});
+
+const I18nProvider = I18nextProvider;
+
+export { i18n, useTranslation, I18nProvider, LOCALES };

libs/datasource/i18n/src/resources/en.json

@@ -0,0 +1,24 @@
+{
+    "Sync to Disk": "Sync to Disk",
+    "Share": "Share",
+    "WarningTips": {
+        "IsNotfsApiSupported": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC DATA TO DISK with the latest version of Chromium based browser like Chrome/Edge",
+        "IsNotLocalWorkspace": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC TO DISK.",
+        "DoNotStore": "AFFiNE is under active development and the current version is UNSTABLE. Please DO NOT store information or data"
+    },
+    "Layout": "Layout",
+    "Comment": "Comment",
+    "Settings": "Settings",
+    "ComingSoon": "Layout Settings Coming Soon...",
+    "Duplicate Page": "Duplicate Page",
+    "Copy Page Link": "Copy Page Link",
+    "Language": "Language",
+    "Clear Workspace": "Clear Workspace",
+    "Export As Markdown": "Export As Markdown",
+    "Export As HTML": "Export As HTML",
+    "Export As PDF (Unsupported)": "Export As PDF (Unsupported)",
+    "Import Workspace": "Import Workspace",
+    "Export Workspace": "Export Workspace",
+    "Last edited by": "Last edited by {{name}}",
+    "Logout": "Logout"
+}

libs/datasource/i18n/src/resources/zh.json

@@ -0,0 +1,24 @@
+{
+    "Sync to Disk": "同步到磁盘",
+    "Share": "分享",
+    "WarningTips": {
+        "IsNotfsApiSupported": "欢迎来到AFFiNE 的演示界面。您可以使用最新版本的基于Chrome的浏览器（如Chrome/Edge）将数据同步到磁盘来进行保存",
+        "IsNotLocalWorkspace": "欢迎来到AFFiNE 的演示界面，您可以同步到磁盘来进行保存操作。",
+        "DoNotStore": "AFFINE 正在积极开发中，当前版本不稳定。请不要存储信息或数据。"
+    },
+    "ComingSoon": "布局设置即将到来",
+    "Layout": "布局",
+    "Comment": "评论",
+    "Settings": "设置",
+    "Duplicate Page": "复制页面",
+    "Copy Page Link": "复制页面链接",
+    "Language": "当前语言",
+    "Clear Workspace": "清空工作区域",
+    "Export As Markdown": "导出 markdown",
+    "Export As HTML": "导出 HTML",
+    "Export As PDF (Unsupported)": "导出 PDF (暂不支持)",
+    "Import Workspace": "导入 Workspace",
+    "Export Workspace": "导出 Workspace",
+    "Last edited by": "最后编辑者为 {{name}}",
+    "Logout": "退出登录"
+}

libs/datasource/i18n/src/scripts/api.ts

@@ -0,0 +1,111 @@
+// cSpell:ignore Tolgee
+import { fetchTolgee } from './request';
+
+/**
+ * Returns all project languages
+ *
+ * See https://tolgee.io/api#operation/getAll_6
+ */
+export const getAllProjectLanguages = async () => {
+    const url = '/languages?size=1000';
+    const resp = await fetchTolgee(url);
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json = await resp.json();
+    return json;
+};
+
+/**
+ * Returns translations in project
+ *
+ * See https://tolgee.io/api#operation/getTranslations_
+ */
+export const getTranslations = async () => {
+    const url = '/translations';
+    const resp = await fetchTolgee(url);
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json = await resp.json();
+    return json;
+};
+
+/**
+ * Returns all translations for specified languages
+ *
+ * See https://tolgee.io/api#operation/getAllTranslations_1
+ */
+export const getLanguagesTranslations = async <T extends string>(
+    languages: T
+) => {
+    const url = `/translations/${languages}`;
+    const resp = await fetchTolgee(url);
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json: { [key in T]?: Record<string, string> } = await resp.json();
+    return json;
+};
+
+/**
+ * Creates new key
+ *
+ * See https://tolgee.io/api#operation/create_2
+ */
+export const createsNewKey = async (
+    key: string,
+    translations: Record<string, string>
+) => {
+    const url = '/translations/keys/create';
+    const resp = await fetchTolgee(url, {
+        method: 'POST',
+        body: JSON.stringify({ name: key, translations }),
+    });
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json = await resp.json();
+    return json;
+};
+
+/**
+ * Tags a key with tag. If tag with provided name doesn't exist, it is created
+ *
+ * See https://tolgee.io/api#operation/tagKey_1
+ */
+export const addTag = async (keyId: string, tagName: string) => {
+    const url = `/keys/${keyId}/tags`;
+    const resp = await fetchTolgee(url, {
+        method: 'PUT',
+        body: JSON.stringify({ name: tagName }),
+    });
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json = await resp.json();
+    return json;
+};
+
+/**
+ * Tags a key with tag. If tag with provided name doesn't exist, it is created
+ *
+ * See https://tolgee.io/api#operation/tagKey_1
+ */
+export const removeTag = async (keyId: string, tagId: number) => {
+    const url = `/keys/${keyId}/tags/${tagId}`;
+    const resp = await fetchTolgee(url, {
+        method: 'DELETE',
+    });
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json = await resp.json();
+    return json;
+};
+
+export const addTagByKey = async (key: string, tag: string) => {
+    // TODO get key id by key name
+    // const keyId =
+    // addTag(keyId, tag);
+};

libs/datasource/i18n/src/scripts/request.ts

@@ -0,0 +1,53 @@
+// cSpell:ignore Tolgee
+const TOLGEE_API_KEY = process.env['TOLGEE_API_KEY'];
+const TOLGEE_API_URL = 'https://i18n.affine.pro';
+
+if (!TOLGEE_API_KEY) {
+    throw new Error(`Please set "TOLGEE_API_KEY" as environment variable!`);
+}
+
+const withTolgee = (
+    fetch: typeof globalThis.fetch
+): typeof globalThis.fetch => {
+    const headers = new Headers({
+        'X-API-Key': TOLGEE_API_KEY,
+        'Content-Type': 'application/json',
+    });
+
+    const isRequest = (input: RequestInfo | URL): input is Request => {
+        return typeof input === 'object' && !('href' in input);
+    };
+
+    return new Proxy(fetch, {
+        apply(
+            target,
+            thisArg: unknown,
+            argArray: Parameters<typeof globalThis.fetch>
+        ) {
+            if (isRequest(argArray[0])) {
+                // Request
+                if (!argArray[0].headers) {
+                    argArray[0] = {
+                        ...argArray[0],
+                        url: `${TOLGEE_API_URL}/v2/projects${argArray[0].url}`,
+                        headers,
+                    };
+                }
+            } else {
+                // URL or URLLike + ?RequestInit
+                if (typeof argArray[0] === 'string') {
+                    argArray[0] = TOLGEE_API_URL + argArray[0];
+                }
+                if (!argArray[1]) {
+                    argArray[1] = {};
+                }
+                if (!argArray[1].headers) {
+                    argArray[1].headers = headers;
+                }
+            }
+            return target.apply(thisArg, argArray);
+        },
+    });
+};
+
+export const fetchTolgee = withTolgee(globalThis.fetch);

libs/datasource/i18n/src/scripts/sync.ts

@@ -0,0 +1,142 @@
+// cSpell:ignore Tolgee
+import { readFile } from 'fs/promises';
+import path from 'path';
+import { addTagByKey, createsNewKey, getLanguagesTranslations } from './api';
+
+const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json');
+const BASE_LANGUAGES = 'en' as const;
+
+const DEPRECATED_TAG_NAME = 'unused' as const;
+
+interface TranslationRes {
+    [x: string]: string | TranslationRes;
+}
+
+const getRemoteTranslations = async (languages: string) => {
+    const translations = await getLanguagesTranslations(languages);
+    if (!(languages in translations)) {
+        console.log(translations);
+        throw new Error(
+            'Failed to get base languages translation! base languages: ' +
+                languages
+        );
+    }
+    // The assert is safe because we checked above
+    return translations[languages]!;
+};
+
+/**
+ *
+ * @example
+ * ```ts
+ * flatRes({ a: { b: 'c' } }); // { 'a.b': 'c' }
+ * ```
+ */
+const flatRes = (obj: TranslationRes) => {
+    const getEntries = (o: TranslationRes, prefix = ''): [string, string][] =>
+        Object.entries(o).flatMap<[string, string]>(([k, v]) =>
+            typeof v !== 'string'
+                ? getEntries(v, `${prefix}${k}.`)
+                : [[`${prefix}${k}`, v]]
+        );
+    return Object.fromEntries(getEntries(obj));
+};
+
+const differenceObject = (
+    newObj: Record<string, string>,
+    oldObj: Record<string, string>
+) => {
+    const add: string[] = [];
+    const remove: string[] = [];
+    const modify: string[] = [];
+    const both: string[] = [];
+
+    Object.keys(newObj).forEach(key => {
+        if (!(key in oldObj)) {
+            add.push(key);
+        } else {
+            both.push(key);
+        }
+    });
+
+    Object.keys(oldObj).forEach(key => {
+        if (!(key in newObj)) {
+            remove.push(key);
+        }
+    });
+
+    both.forEach(key => {
+        if (!(key in newObj) || !(key in oldObj)) {
+            throw new Error('Unreachable');
+        }
+        const newVal = newObj[key];
+        const oldVal = oldObj[key];
+        if (newVal !== oldVal) {
+            modify.push(key);
+        }
+    });
+    return { add, remove, modify };
+};
+
+const main = async () => {
+    console.log('Loading local base translations...');
+    const baseLocalTranslations = JSON.parse(
+        await readFile(BASE_JSON_PATH, {
+            encoding: 'utf8',
+        })
+    );
+    const flatLocalTranslations = flatRes(baseLocalTranslations);
+    console.log(
+        `Loading local base translations success! Total ${
+            Object.keys(flatLocalTranslations).length
+        } keys`
+    );
+
+    console.log('Fetch remote base translations...');
+    const baseRemoteTranslations = await getRemoteTranslations(BASE_LANGUAGES);
+    const flatRemoteTranslations = flatRes(baseRemoteTranslations);
+    console.log(
+        `Fetch remote base translations success! Total ${
+            Object.keys(flatRemoteTranslations).length
+        } keys`
+    );
+
+    const diff = differenceObject(
+        flatLocalTranslations,
+        flatRemoteTranslations
+    );
+
+    console.log(''); // new line
+    diff.add.length && console.log('New keys found:', diff.add.join(', '));
+    diff.remove.length &&
+        console.warn('[WARN]', 'Unused keys found:', diff.remove.join(', '));
+    diff.modify.length &&
+        console.warn(
+            '[WARN]',
+            'Inconsistent keys found:',
+            diff.modify.join(', ')
+        );
+    console.log(''); // new line
+
+    diff.add.forEach(async key => {
+        const val = flatLocalTranslations[key];
+        console.log(`Creating new key: ${key} -> ${val}`);
+        await createsNewKey(key, { [BASE_LANGUAGES]: val });
+    });
+
+    // TODO remove unused tags from used keys
+
+    diff.remove.forEach(key => {
+        // TODO set unused tag
+        // console.log(`Add ${DEPRECATED_TAG_NAME} to ${key}`);
+        addTagByKey(key, DEPRECATED_TAG_NAME);
+    });
+
+    diff.modify.forEach(key => {
+        // TODO warn different between local and remote base translations
+    });
+
+    // TODO send notification
+};
+
+main();

libs/datasource/i18n/tsconfig.json

@@ -0,0 +1,33 @@
+{
+    "extends": "../../../tsconfig.base.json",
+    "compilerOptions": {
+        "jsx": "react-jsx",
+        "esModuleInterop": true,
+        "allowSyntheticDefaultImports": true,
+        "forceConsistentCasingInFileNames": true,
+        "strict": true,
+        "strictNullChecks": true,
+        "importsNotUsedAsValues": "error",
+        "noImplicitOverride": true,
+        "noPropertyAccessFromIndexSignature": true,
+        "noImplicitReturns": true,
+        "noFallthroughCasesInSwitch": true,
+        "resolveJsonModule": true
+    },
+    "files": [],
+    "include": [],
+    "references": [
+        {
+            "path": "./tsconfig.lib.json"
+        },
+        {
+            "path": "./tsconfig.spec.json"
+        }
+    ],
+    "ts-node": {
+        "compilerOptions": {
+            "module": "NodeNext"
+        },
+        "esm": true
+    }
+}

libs/datasource/i18n/tsconfig.lib.json

@@ -0,0 +1,23 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../../dist/out-tsc",
+        "types": ["node"]
+    },
+    "files": [
+        "../../../node_modules/@nrwl/react/typings/cssmodule.d.ts",
+        "../../../node_modules/@nrwl/react/typings/image.d.ts"
+    ],
+    "exclude": [
+        "jest.config.ts",
+        "**/*.spec.ts",
+        "**/*.test.ts",
+        "**/*.spec.tsx",
+        "**/*.test.tsx",
+        "**/*.spec.js",
+        "**/*.test.js",
+        "**/*.spec.jsx",
+        "**/*.test.jsx"
+    ],
+    "include": ["**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"]
+}

libs/datasource/i18n/tsconfig.spec.json

@@ -0,0 +1,20 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../../dist/out-tsc",
+        "module": "commonjs",
+        "types": ["jest", "node"]
+    },
+    "include": [
+        "jest.config.ts",
+        "**/*.test.ts",
+        "**/*.spec.ts",
+        "**/*.test.tsx",
+        "**/*.spec.tsx",
+        "**/*.test.js",
+        "**/*.spec.js",
+        "**/*.test.jsx",
+        "**/*.spec.jsx",
+        "**/*.d.ts"
+    ]
+}

tsconfig.base.json

@@ -74,6 +74,9 @@
             "@toeverything/datasource/feature-flags": [
                 "libs/datasource/feature-flags/src/index.ts"
             ],
+            "@toeverything/datasource/i18n": [
+                "libs/datasource/i18n/src/index.ts"
+            ],
             "@toeverything/datasource/jwt": [
                 "libs/datasource/jwt/src/index.ts"
             ],

workspace.json

@@ -21,6 +21,7 @@
         "datasource-commands": "libs/datasource/commands",
         "datasource-db-service": "libs/datasource/db-service",
         "datasource-feature-flags": "libs/datasource/feature-flags",
+        "datasource-i18n": "libs/datasource/i18n",
         "datasource-jwt": "libs/datasource/jwt",
         "datasource-jwt-rpc": "libs/datasource/jwt-rpc",
         "datasource-remote-kv": "libs/datasource/remote-kv",