diff --git a/.github/workflows/languages-download.yml b/.github/workflows/languages-download.yml
new file mode 100644
index 0000000000..4e180bccce
--- /dev/null
+++ b/.github/workflows/languages-download.yml
@@ -0,0 +1,84 @@
+name: Download Languages Resources
+
+on:
+ schedule:
+ - cron: "0 0 * * 5" # At 00:00(UTC) on Friday.
+ workflow_dispatch:
+
+# Cancels all previous workflow runs for pull requests that have not completed.
+# See https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+ # The concurrency group contains the workflow name and the branch name for
+ # pull requests or the commit hash for any other events.
+ group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+ cancel-in-progress: true
+
+jobs:
+ main:
+ strategy:
+ matrix:
+ node-version: [18]
+ os: [ubuntu-latest]
+ runs-on: ${{ matrix.os }}
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - name: Use pnpm
+ uses: pnpm/action-setup@v2
+ with:
+ version: 7
+
+ - name: Use Node.js ${{ matrix.node-version }}
+ # https://github.com/actions/setup-node
+ uses: actions/setup-node@v3
+ with:
+ node-version: ${{ matrix.node-version }}
+ cache: 'pnpm'
+
+ - name: Install node modules
+ run: pnpm install
+
+ - name: Sync Languages
+ if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master'
+ working-directory: ./libs/datasource/i18n
+ run: pnpm run download-resources
+ env:
+ TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}
+
+ - name: Push Branch
+ id: push
+ run: |
+ git add libs/datasource/i18n
+ # Do not proceed if there are no file differences
+ COMMIT=$(git rev-parse --verify origin/$TARGET_BRANCH || echo HEAD)
+ FILES_CHANGED=$(git diff-index --name-only --cached $COMMIT | wc -l)
+ if [[ "$FILES_CHANGED" = "0" ]]; then
+ echo "No file changes detected."
+ echo "::set-output name=skipPR::true"
+ exit 0
+ fi
+ git config user.name 'github-actions[bot]'
+ git config user.email 'github-actions[bot]@users.noreply.github.com'
+ git commit --message 'feat(i18n): new translations'
+ git remote set-url origin "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY"
+ git push --force origin HEAD:$TARGET_BRANCH
+ env:
+ GITHUB_TOKEN: ${{ secrets.github_token }}
+ TARGET_BRANCH: bot/new-translations
+
+ - name: Get current date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
+ # see https://github.com/repo-sync/pull-request
+ - name: Create Pull Request
+ if: steps.push.outputs.skipPR != 'true'
+ uses: repo-sync/pull-request@v2
+ with:
+ source_branch: 'bot/new-translations' # If blank, default: triggered branch
+ destination_branch: "develop"
+ pr_title: Update i18n (${{ steps.date.outputs.date }}) # Title of pull request
+ pr_label: 'data,bot' # Comma-separated list (no spaces)
+ github_token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx b/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx
index a308614d13..8ebaf90a82 100644
--- a/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx
+++ b/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx
@@ -53,10 +53,10 @@ export const SettingsList = () => {
>
{LOCALES.map(option => (
))}
diff --git a/libs/datasource/i18n/package.json b/libs/datasource/i18n/package.json
index eb1657865e..25e7a82139 100644
--- a/libs/datasource/i18n/package.json
+++ b/libs/datasource/i18n/package.json
@@ -3,7 +3,8 @@
"version": "0.0.1",
"scripts": {
"sync-languages": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/sync.ts",
- "sync-languages:check": "pnpm run sync-languages --check"
+ "sync-languages:check": "pnpm run sync-languages --check",
+ "download-resources": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/download.ts"
},
"dependencies": {
"i18next": "^21.9.1",
diff --git a/libs/datasource/i18n/src/base.json b/libs/datasource/i18n/src/base.json
deleted file mode 100644
index e243d71db1..0000000000
--- a/libs/datasource/i18n/src/base.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "Sync to Disk": "Sync to Disk",
- "Share": "Share",
- "WarningTips": {
- "IsNotfsApiSupported": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC DATA TO DISK with the latest version of Chromium based browser like Chrome/Edge",
- "IsNotLocalWorkspace": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC TO DISK.",
- "DoNotStore": "AFFiNE is under active development and the current version is UNSTABLE. Please DO NOT store information or data"
- },
- "Layout": "Layout",
- "Comment": "Comment",
- "Settings": "Settings",
- "ComingSoon": "Layout Settings Coming Soon...",
- "Duplicate Page": "Duplicate Page",
- "Copy Page Link": "Copy Page Link",
- "Language": "Language",
- "Clear Workspace": "Clear Workspace",
- "Export As Markdown": "Export As Markdown",
- "Export As HTML": "Export As HTML",
- "Export As PDF (Unsupported)": "Export As PDF (Unsupported)",
- "Import Workspace": "Import Workspace",
- "Export Workspace": "Export Workspace",
- "Last edited by": "Last edited by {{name}}",
- "Logout": "Logout"
-}
diff --git a/libs/datasource/i18n/src/index.ts b/libs/datasource/i18n/src/index.ts
index a421494bf8..f4428a9b09 100644
--- a/libs/datasource/i18n/src/index.ts
+++ b/libs/datasource/i18n/src/index.ts
@@ -4,8 +4,8 @@ import {
initReactI18next,
useTranslation,
} from 'react-i18next';
-import en_US from './resources/en.json';
-import zh_CN from './resources/zh.json';
+import { LOCALES } from './resources';
+import type en_US from './resources/en.json';
// See https://react.i18next.com/latest/typescript
declare module 'react-i18next' {
@@ -21,22 +21,19 @@ declare module 'react-i18next' {
const STORAGE_KEY = 'i18n_lng';
-const LOCALES = [
- { value: 'en', text: 'English', res: en_US },
- { value: 'zh', text: '简体中文', res: zh_CN },
-] as const;
+export { i18n, useTranslation, I18nProvider, LOCALES };
const resources = LOCALES.reduce(
- (acc, { value, res }) => ({ ...acc, [value]: { translation: res } }),
+ (acc, { tag, res }) => ({ ...acc, [tag]: { translation: res } }),
{}
);
-const fallbackLng = LOCALES[0].value;
+const fallbackLng = LOCALES[0].tag;
const standardizeLocale = (language: string) => {
- if (LOCALES.find(locale => locale.value === language)) return language;
+ if (LOCALES.find(locale => locale.tag === language)) return language;
if (
LOCALES.find(
- locale => locale.value === language.slice(0, 2).toLowerCase()
+ locale => locale.tag === language.slice(0, 2).toLowerCase()
)
)
return language;
@@ -64,5 +61,3 @@ i18n.on('languageChanged', lng => {
});
const I18nProvider = I18nextProvider;
-
-export { i18n, useTranslation, I18nProvider, LOCALES };
diff --git a/libs/datasource/i18n/src/resources/index.ts b/libs/datasource/i18n/src/resources/index.ts
new file mode 100644
index 0000000000..1d3e875c60
--- /dev/null
+++ b/libs/datasource/i18n/src/resources/index.ts
@@ -0,0 +1,28 @@
+// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+// Run `pnpm run download-resources` to regenerate.
+// To overwrite this, please overwrite download.ts
+import en from './en.json';
+import zh_Hans from './zh-Hans.json';
+
+export const LOCALES = [
+ {
+ id: 1000016008,
+ name: 'English',
+ tag: 'en',
+ originalName: 'English',
+ flagEmoji: '🇬🇧',
+ base: true,
+ completeRate: 1,
+ res: en,
+ },
+ {
+ id: 1000016009,
+ name: 'Simplified Chinese',
+ tag: 'zh-Hans',
+ originalName: '简体中文',
+ flagEmoji: '🇨🇳',
+ base: false,
+ completeRate: 1,
+ res: zh_Hans,
+ },
+] as const;
diff --git a/libs/datasource/i18n/src/resources/zh.json b/libs/datasource/i18n/src/resources/zh-Hans.json
similarity index 93%
rename from libs/datasource/i18n/src/resources/zh.json
rename to libs/datasource/i18n/src/resources/zh-Hans.json
index fe72ae542d..5122e4b881 100644
--- a/libs/datasource/i18n/src/resources/zh.json
+++ b/libs/datasource/i18n/src/resources/zh-Hans.json
@@ -1,4 +1,5 @@
{
+ "// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.": "",
"Sync to Disk": "同步到磁盘",
"Share": "分享",
"WarningTips": {
diff --git a/libs/datasource/i18n/src/scripts/api.ts b/libs/datasource/i18n/src/scripts/api.ts
index 52011e6042..b38ad5875b 100644
--- a/libs/datasource/i18n/src/scripts/api.ts
+++ b/libs/datasource/i18n/src/scripts/api.ts
@@ -5,15 +5,65 @@ import { fetchTolgee } from './request';
* Returns all project languages
*
* See https://tolgee.io/api#operation/getAll_6
+ * @example
+ * ```ts
+ * const languages = [
+ * {
+ * id: 1000016008,
+ * name: 'English',
+ * tag: 'en',
+ * originalName: 'English',
+ * flagEmoji: '🇬🇧',
+ * base: true
+ * },
+ * {
+ * id: 1000016013,
+ * name: 'Spanish',
+ * tag: 'es',
+ * originalName: 'español',
+ * flagEmoji: '🇪🇸',
+ * base: false
+ * },
+ * {
+ * id: 1000016009,
+ * name: 'Simplified Chinese',
+ * tag: 'zh-Hans',
+ * originalName: '简体中文',
+ * flagEmoji: '🇨🇳',
+ * base: false
+ * },
+ * {
+ * id: 1000016012,
+ * name: 'Traditional Chinese',
+ * tag: 'zh-Hant',
+ * originalName: '繁體中文',
+ * flagEmoji: '🇭🇰',
+ * base: false
+ * }
+ * ]
+ * ```
*/
-export const getAllProjectLanguages = async () => {
- const url = '/languages?size=1000';
+export const getAllProjectLanguages = async (size = 1000) => {
+ const url = `/languages?size=${size}`;
const resp = await fetchTolgee(url);
if (resp.status < 200 || resp.status >= 300) {
throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
}
- const json = await resp.json();
- return json;
+ const json: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ _embedded: {
+ languages: {
+ id: number;
+ name: string;
+ tag: string;
+ originalName: string;
+ flagEmoji: string;
+ base: boolean;
+ }[];
+ };
+ page: unknown;
+ } = await resp.json();
+ return json._embedded.languages;
};
/**
@@ -48,6 +98,16 @@ export const getLanguagesTranslations = async (
return json;
};
+export const getRemoteTranslations = async (languages: string) => {
+ const translations = await getLanguagesTranslations(languages);
+ if (!(languages in translations)) {
+ return {};
+ }
+ // The assert is safe because we checked above
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+ return translations[languages]!;
+};
+
/**
* Creates new key
*
@@ -109,3 +169,18 @@ export const addTagByKey = async (key: string, tag: string) => {
// const keyId =
// addTag(keyId, tag);
};
+
+/**
+ * Exports data
+ *
+ * See https://tolgee.io/api#operation/export_1
+ */
+export const exportResources = async () => {
+ const url = `/export`;
+ const resp = await fetchTolgee(url);
+
+ if (resp.status < 200 || resp.status >= 300) {
+ throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+ }
+ return resp;
+};
diff --git a/libs/datasource/i18n/src/scripts/download.ts b/libs/datasource/i18n/src/scripts/download.ts
new file mode 100644
index 0000000000..f45261c258
--- /dev/null
+++ b/libs/datasource/i18n/src/scripts/download.ts
@@ -0,0 +1,134 @@
+/* eslint-disable no-console */
+// cSpell:ignore Tolgee
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { format } from 'prettier';
+import { getAllProjectLanguages, getRemoteTranslations } from './api';
+import type { TranslationRes } from './utils';
+
+const RES_DIR = path.resolve(process.cwd(), 'src', 'resources');
+
+const countKeys = (obj: TranslationRes) => {
+ let count = 0;
+ Object.entries(obj).forEach(([key, value]) => {
+ if (typeof value === 'string') {
+ count++;
+ } else {
+ count += countKeys(value);
+ }
+ });
+ return count;
+};
+
+const getBaseTranslations = async (baseLanguage: { tag: string }) => {
+ try {
+ const baseTranslationsStr = await fs.readFile(
+ path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
+ { encoding: 'utf8' }
+ );
+ const baseTranslations = JSON.parse(baseTranslationsStr);
+ return baseTranslations;
+ } catch (e) {
+ console.error('base language:', JSON.stringify(baseLanguage));
+ console.error('Failed to read base language', e);
+ const translations = await getRemoteTranslations(baseLanguage.tag);
+ await fs.writeFile(
+ path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
+ JSON.stringify(translations, null, 4)
+ );
+ }
+};
+
+const main = async () => {
+ console.log('Loading project languages...');
+ const languages = await getAllProjectLanguages();
+ const baseLanguage = languages.find(language => language.base);
+ if (!baseLanguage) {
+ console.error(JSON.stringify(languages));
+ throw new Error('Could not find base language');
+ }
+ console.log(
+ `Loading ${baseLanguage.tag} languages translations as base...`
+ );
+
+ const baseTranslations = await getBaseTranslations(baseLanguage);
+ const baseKeyNum = countKeys(baseTranslations);
+ const languagesWithTranslations = await Promise.all(
+ languages.map(async language => {
+ console.log(`Loading ${language.tag} translations...`);
+ const translations = await getRemoteTranslations(language.tag);
+ const keyNum = countKeys(translations);
+
+ return {
+ ...language,
+ translations,
+ completeRate: keyNum / baseKeyNum,
+ };
+ })
+ );
+
+ const availableLanguages = languagesWithTranslations.filter(
+ language => language.completeRate > 0
+ );
+
+ availableLanguages
+ // skip base language
+ .filter(i => !i.base)
+ .forEach(async language => {
+ await fs.writeFile(
+ path.resolve(RES_DIR, `${language.tag}.json`),
+ JSON.stringify(
+ {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ '// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.':
+ '',
+ ...language.translations,
+ },
+ null,
+ 4
+ ) + '\n'
+ );
+ });
+
+ console.log('Generating meta data...');
+ const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+ // Run \`pnpm run download-resources\` to regenerate.
+ // To overwrite this, please overwrite ${path.basename(__filename)}
+ ${availableLanguages
+ .map(
+ language =>
+ `import ${language.tag.replaceAll('-', '_')} from './${
+ language.tag
+ }.json'`
+ )
+ .join('\n')}
+
+ export const LOCALES = [
+ ${availableLanguages
+ .map(({ translations, ...language }) =>
+ JSON.stringify({
+ ...language,
+ res: '__RES_PLACEHOLDER',
+ }).replace(
+ '"__RES_PLACEHOLDER"',
+ language.tag.replaceAll('-', '_')
+ )
+ )
+ .join(',\n')}
+ ] as const;
+ `;
+
+ await fs.writeFile(
+ path.resolve(RES_DIR, 'index.ts'),
+ format(code, {
+ parser: 'typescript',
+ singleQuote: true,
+ trailingComma: 'es5',
+ tabWidth: 4,
+ arrowParens: 'avoid',
+ })
+ );
+ console.log('Done');
+};
+
+main();
diff --git a/libs/datasource/i18n/src/scripts/request.ts b/libs/datasource/i18n/src/scripts/request.ts
index c016909a24..b599c71957 100644
--- a/libs/datasource/i18n/src/scripts/request.ts
+++ b/libs/datasource/i18n/src/scripts/request.ts
@@ -46,6 +46,7 @@ const withTolgee = (
argArray[1].headers = headers;
}
}
+ // console.log('fetch', argArray);
return target.apply(thisArg, argArray);
},
});
diff --git a/libs/datasource/i18n/src/scripts/sync.ts b/libs/datasource/i18n/src/scripts/sync.ts
index a71a0de61f..c2ba692dc5 100644
--- a/libs/datasource/i18n/src/scripts/sync.ts
+++ b/libs/datasource/i18n/src/scripts/sync.ts
@@ -1,30 +1,20 @@
+/* eslint-disable no-console */
// cSpell:ignore Tolgee
import { readFile } from 'fs/promises';
import path from 'path';
-import { addTagByKey, createsNewKey, getLanguagesTranslations } from './api';
+import { addTagByKey, createsNewKey, getRemoteTranslations } from './api';
+import type { TranslationRes } from './utils';
-const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json');
+const BASE_JSON_PATH = path.resolve(
+ process.cwd(),
+ 'src',
+ 'resources',
+ 'en.json'
+);
const BASE_LANGUAGES = 'en' as const;
const DEPRECATED_TAG_NAME = 'unused' as const;
-interface TranslationRes {
- [x: string]: string | TranslationRes;
-}
-
-const getRemoteTranslations = async (languages: string) => {
- const translations = await getLanguagesTranslations(languages);
- if (!(languages in translations)) {
- console.log(translations);
- throw new Error(
- 'Failed to get base languages translation! base languages: ' +
- languages
- );
- }
- // The assert is safe because we checked above
- return translations[languages]!;
-};
-
/**
*
* @example
diff --git a/libs/datasource/i18n/src/scripts/utils.ts b/libs/datasource/i18n/src/scripts/utils.ts
new file mode 100644
index 0000000000..129a8febc3
--- /dev/null
+++ b/libs/datasource/i18n/src/scripts/utils.ts
@@ -0,0 +1,3 @@
+export interface TranslationRes {
+ [x: string]: string | TranslationRes;
+}