Jayden/AFFiNE-Mirror
1
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-02-12 04:18:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: sync i18n with crowdin (#8293)

Browse Source
This commit is contained in:
EYHN
2024-09-19 05:30:17 +00:00
parent 6921c3073c
commit 9a01da76e1
11 changed files with 51 additions and 657 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
.github/workflows/languages-sync.yml vendored
View File

@@ -1,35 +0,0 @@
name: Languages Sync
on:
push:
branches: ['canary']
paths:
- 'packages/frontend/i18n/**'
- '.github/workflows/languages-sync.yml'
- '!.github/actions/setup-node/action.yml'
pull_request_target:
branches: ['canary']
paths:
- 'packages/frontend/i18n/**'
- '.github/workflows/languages-sync.yml'
- '!.github/actions/setup-node/action.yml'
workflow_dispatch:
jobs:
main:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Check Language Key
if: github.ref != 'refs/heads/canary'
run: yarn workspace @affine/i18n run sync-languages:check
env:
TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}
- name: Sync Languages
if: github.ref == 'refs/heads/canary'
run: yarn workspace @affine/i18n run sync-languages
env:
TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}

38
.github/workflows/sync-i18n.yml vendored Normal file
View File

@@ -0,0 +1,38 @@
name: Sync I18n with Crowdin
on:
push:
workflow_dispatch:
jobs:
synchronize-with-crowdin:
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: crowdin action
uses: crowdin/github-action@v2
with:
upload_sources: true
upload_translations: true
download_translations: true
auto_approve_imported: true
import_eq_suggestions: true
export_only_approved: true
skip_untranslated_strings: true
localization_branch_name: l10n_crowdin_translations
create_pull_request: true
pull_request_title: 'New Crowdin Translations'
pull_request_body: 'New Crowdin translations by [Crowdin GH Action](https://github.com/crowdin/github-action)'
pull_request_base_branch_name: 'canary'
config: packages/frontend/i18n/crowdin.yml
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}

12
packages/frontend/i18n/crowdin.yml Normal file
View File

@@ -0,0 +1,12 @@
'base_path': '.'
'base_url': 'https://api.crowdin.com'
'preserve_hierarchy': true
'files':
[
{
'source': '/src/resources/en.json',
'translation': '/src/resources/%locale%.json',
},
]

5
packages/frontend/i18n/package.json
View File

@@ -8,10 +8,7 @@
}, },
"scripts": { "scripts": {
"build": "node build.mjs", "build": "node build.mjs",
"dev": "node dev.mjs", "dev": "node dev.mjs"
"sync-languages": "node --loader ts-node/esm/transpile-only src/scripts/sync.ts",
"sync-languages:check": "yarn run sync-languages --check",
"download-resources": "node --loader ts-node/esm/transpile-only src/scripts/download.ts"
}, },
"keywords": [], "keywords": [],
"repository": { "repository": {

212
packages/frontend/i18n/src/scripts/api.ts
View File

@@ -1,212 +0,0 @@
import type { Response } from 'undici-types';
// cSpell:ignore Tolgee
import { fetchTolgee } from './request.js';
/**
* Returns all project languages
*
* See https://tolgee.io/api#operation/getAll_6
* @example
* ```ts
* const languages = [
* {
* id: 1000016008,
* name: 'English',
* tag: 'en',
* originalName: 'English',
* flagEmoji: '🇬🇧',
* base: true
* },
* {
* id: 1000016013,
* name: 'Spanish',
* tag: 'es',
* originalName: 'español',
* flagEmoji: '🇪🇸',
* base: false
* },
* {
* id: 1000016009,
* name: 'Simplified Chinese',
* tag: 'zh-Hans',
* originalName: '简体中文',
* flagEmoji: '🇨🇳',
* base: false
* },
* {
* id: 1000016012,
* name: 'Traditional Chinese',
* tag: 'zh-Hant',
* originalName: '繁體中文',
* flagEmoji: '🇭🇰',
* base: false
* }
* ]
* ```
*/
export const getAllProjectLanguages = async (
size = 1000
): Promise<
{
id: number;
name: string;
tag: string;
originalName: string;
flagEmoji: string;
base: boolean;
}[]
> => {
const url = `/languages?size=${size}`;
const resp = await fetchTolgee(url);
if (resp.status < 200 || resp.status >= 300) {
throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
}
const json: {
_embedded: {
languages: {
id: number;
name: string;
tag: string;
originalName: string;
flagEmoji: string;
base: boolean;
}[];
};
page: unknown;
} = (await resp.json()) as any;
return json._embedded.languages;
};
/**
* Returns translations in project
*
* See https://tolgee.io/api#operation/getTranslations_
*/
export const getTranslations = async (): Promise<unknown> => {
const url = '/translations';
const resp = await fetchTolgee(url);
if (resp.status < 200 || resp.status >= 300) {
throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
}
const json = await resp.json();
return json;
};
/**
* Returns all translations for specified languages
*
* See https://tolgee.io/api#operation/getAllTranslations_1
*/
export const getLanguagesTranslations = async <T extends string>(
languages: T
): Promise<{ [key in T]?: Record<string, string> }> => {
const url = `/translations/${languages}`;
const resp = await fetchTolgee(url);
if (resp.status < 200 || resp.status >= 300) {
throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
}
const json = await resp.json();
return json as { [key in T]?: Record<string, string> };
};
export const getRemoteTranslations = async (
languages: string
): Promise<Record<string, string>> => {
const translations = await getLanguagesTranslations(languages);
if (!(languages in translations)) {
return {};
}
// The assert is safe because we checked above
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
return translations[languages]!;
};
/**
* Creates new key
*
* See https://tolgee.io/api#operation/create_2
*/
export const createsNewKey = async (
key: string,
translations: Record<string, string>
): Promise<unknown> => {
const url = '/translations/keys/create';
const resp = await fetchTolgee(url, {
method: 'POST',
body: JSON.stringify({ name: key, translations }),
});
if (resp.status < 200 || resp.status >= 300) {
/**
* There are some problems in the i18n backend,
* which is used to temporarily solve the ci error.
*/
console.warn(url + ' ' + resp.status + '\n' + (await resp.text()));
return;
}
const json = await resp.json();
return json;
};
/**
* Tags a key with tag. If tag with provided name doesn't exist, it is created
*
* See https://tolgee.io/api#operation/tagKey_1
*/
export const addTag = async (
keyId: string,
tagName: string
): Promise<unknown> => {
const url = `/keys/${keyId}/tags`;
const resp = await fetchTolgee(url, {
method: 'PUT',
body: JSON.stringify({ name: tagName }),
});
if (resp.status < 200 || resp.status >= 300) {
throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
}
const json = await resp.json();
return json;
};
/**
* Tags a key with tag. If tag with provided name doesn't exist, it is created
*
* See https://tolgee.io/api#operation/tagKey_1
*/
export const removeTag = async (
keyId: string,
tagId: number
): Promise<unknown> => {
const url = `/keys/${keyId}/tags/${tagId}`;
const resp = await fetchTolgee(url, {
method: 'DELETE',
});
if (resp.status < 200 || resp.status >= 300) {
throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
}
const json = await resp.json();
return json;
};
// export const addTagByKey = async (key: string, tag: string) => {
// // TODO get key id by key name
// // const keyId =
// // addTag(keyId, tag);
// };
/**
* Exports data
*
* See https://tolgee.io/api#operation/export_1
*/
export const exportResources = async (): Promise<Response> => {
const url = `/export`;
const resp = await fetchTolgee(url);
if (resp.status < 200 || resp.status >= 300) {
throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
}
return resp;
};

155
packages/frontend/i18n/src/scripts/download.ts
View File

@@ -1,155 +0,0 @@
// cSpell:ignore Tolgee
import * as fs from 'node:fs/promises';
import * as path from 'node:path';
import { format } from 'prettier';
import { getAllProjectLanguages, getRemoteTranslations } from './api.js';
import type { TranslationRes } from './utils.js';
import { flattenTranslation } from './utils.js';
const INDENT = 2;
const RES_DIR = path.resolve(process.cwd(), 'src', 'resources');
const countKeys = (obj: TranslationRes | null) => {
if (!obj) {
return 0;
}
let count = 0;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
Object.entries(obj).forEach(([_, value]) => {
if (typeof value === 'string') {
count++;
} else {
count += countKeys(value);
}
});
return count;
};
const getBaseTranslations = async (baseLanguage: { tag: string }) => {
try {
const baseTranslationsStr = await fs.readFile(
path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
{ encoding: 'utf8' }
);
const baseTranslations = JSON.parse(baseTranslationsStr);
return baseTranslations;
} catch (e) {
console.error('base language:', JSON.stringify(baseLanguage));
console.error('Failed to read base language', e);
const translations = await getRemoteTranslations(baseLanguage.tag);
await fs.writeFile(
path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
JSON.stringify(translations, null, 4)
);
}
};
const main = async () => {
try {
await fs.access(RES_DIR);
} catch {
fs.mkdir(RES_DIR).catch(console.error);
console.log('Create directory', RES_DIR);
}
console.log('Loading project languages...');
const languages = await getAllProjectLanguages();
const baseLanguage = languages.find(language => language.base);
if (!baseLanguage) {
console.error(JSON.stringify(languages));
throw new Error('Could not find base language');
}
console.log(`Loading ${baseLanguage.tag} languages translations as base...`);
const baseTranslations = await getBaseTranslations(baseLanguage);
const baseKeyNum = countKeys(baseTranslations);
const languagesWithTranslations = await Promise.all(
languages.map(async language => {
console.log(`Loading ${language.tag} translations...`);
const translations = await getRemoteTranslations(language.tag);
const keyNum = countKeys(translations);
const completeRate = Number((keyNum / baseKeyNum).toFixed(3));
console.log(
`Load ${language.name} ${
completeRate * 100
}, %(${keyNum}/${baseKeyNum}) complete`
);
return {
...language,
translations,
completeRate,
};
})
);
const availableLanguages = languagesWithTranslations.filter(
language => language.completeRate > 0.2
);
for (const language of availableLanguages
// skip base language
.filter(i => !i.base)) {
await fs.writeFile(
path.resolve(RES_DIR, `${language.tag}.json`),
JSON.stringify(
{
'// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.':
'',
...flattenTranslation(language.translations),
},
null,
INDENT
) + '\n'
);
}
console.log('Generating meta data...');
const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
// Run \`yarn run download-resources\` to regenerate.
// If you need to update the code, please edit \`i18n/src/scripts/download.ts\` inside your project.
${availableLanguages
.map(
language =>
`import ${language.tag.replaceAll('-', '_')} from './${
language.tag
}.json'`
)
.sort()
.join('\n')}
export const LOCALES = [
${availableLanguages
// eslint-disable-next-line @typescript-eslint/no-unused-vars -- omit key
.map(({ translations, ...language }) =>
JSON.stringify({
...language,
// a trick to generate a string without quotation marks
res: '__RES_PLACEHOLDER',
}).replace(
'"__RES_PLACEHOLDER"',
language.tag.replaceAll('-', '_')
)
)
.join(',\n')}
] as const;
`;
await fs.writeFile(
path.resolve(RES_DIR, 'index.ts'),
await format(code, {
parser: 'typescript',
singleQuote: true,
trailingComma: 'es5',
tabWidth: INDENT,
arrowParens: 'avoid',
})
);
console.log('Done');
};
main().catch(e => {
console.error(e);
process.exit(1);
});

52
packages/frontend/i18n/src/scripts/request.ts
View File

@@ -1,52 +0,0 @@
import { Headers } from 'undici';
import type { fetch, Request, RequestInfo } from 'undici-types';
// cSpell:ignore Tolgee
const TOLGEE_API_KEY = process.env['TOLGEE_API_KEY'];
const TOLGEE_API_URL = 'https://i18n.affine.pro';
if (!TOLGEE_API_KEY) {
throw new Error(`Please set "TOLGEE_API_KEY" as environment variable!`);
}
const withTolgee = (f: typeof fetch): typeof fetch => {
const baseUrl = `${TOLGEE_API_URL}/v2/projects`;
const headers = new Headers({
'X-API-Key': TOLGEE_API_KEY,
'Content-Type': 'application/json',
});
const isRequest = (input: RequestInfo): input is Request => {
return typeof input === 'object' && !('href' in input);
};
return new Proxy(f, {
apply(target, thisArg: unknown, argArray: Parameters<typeof fetch>) {
if (isRequest(argArray[0])) {
// Request
if (!argArray[0].headers) {
argArray[0] = {
...argArray[0],
url: `${baseUrl}${argArray[0].url}`,
headers,
};
}
} else {
// URL or URLLike + ?RequestInit
if (typeof argArray[0] === 'string') {
argArray[0] = `${baseUrl}${argArray[0]}`;
}
if (!argArray[1]) {
argArray[1] = {};
}
if (!argArray[1].headers) {
argArray[1].headers = headers;
}
}
// console.log('fetch', argArray);
return target.apply(thisArg, argArray);
},
});
};
export const fetchTolgee = withTolgee(globalThis.fetch as typeof fetch);

153
packages/frontend/i18n/src/scripts/sync.ts
View File

@@ -1,153 +0,0 @@
// cSpell:ignore Tolgee
import { readFile } from 'node:fs/promises';
import { resolve } from 'node:path';
import { createsNewKey, getRemoteTranslations } from './api.js';
import type { TranslationRes } from './utils.js';
const BASE_JSON_PATH = resolve(process.cwd(), 'src', 'resources', 'en.json');
const BASE_LANGUAGES = 'en' as const;
/**
*
* @example
* ```ts
* flatRes({ a: { b: 'c' } }); // { 'a.b': 'c' }
* ```
*/
const flatRes = (obj: TranslationRes) => {
const getEntries = (o: TranslationRes, prefix = ''): [string, string][] =>
Object.entries(o).flatMap<[string, string]>(([k, v]) =>
typeof v !== 'string'
? getEntries(v, `${prefix}${k}.`)
: [[`${prefix}${k}`, v]]
);
return Object.fromEntries(getEntries(obj));
};
const differenceObject = (
newObj: Record<string, string>,
oldObj: Record<string, string>
) => {
const add: string[] = [];
const remove: string[] = [];
const modify: string[] = [];
const both: string[] = [];
Object.keys(newObj).forEach(key => {
if (!(key in oldObj)) {
add.push(key);
} else {
both.push(key);
}
});
Object.keys(oldObj).forEach(key => {
if (!(key in newObj)) {
remove.push(key);
}
});
both.forEach(key => {
if (!(key in newObj) || !(key in oldObj)) {
throw new Error('Unreachable');
}
const newVal = newObj[key];
const oldVal = oldObj[key];
if (newVal !== oldVal) {
modify.push(key);
}
});
return { add, remove, modify };
};
function warnDiff(diff: { add: string[]; remove: string[]; modify: string[] }) {
if (diff.add.length) {
console.log('New keys found:', diff.add.join(', '));
//See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-notice-message
process.env['CI'] &&
console.log(
`::notice file=${BASE_JSON_PATH},line=1,title=New keys::${diff.add.join(
', '
)}`
);
}
if (diff.remove.length) {
console.warn('[WARN]', 'Unused keys found:', diff.remove.join(', '));
process.env['CI'] &&
console.warn(
`::notice file=${BASE_JSON_PATH},line=1,title=Unused keys::${diff.remove.join(
', '
)}`
);
}
if (diff.modify.length) {
console.warn('[WARN]', 'Inconsistent keys found:', diff.modify.join(', '));
process.env['CI'] &&
console.warn(
`::warning file=${BASE_JSON_PATH},line=1,title=Inconsistent keys::${diff.modify.join(
', '
)}`
);
}
}
const main = async () => {
console.log('Loading local base translations...');
const baseLocalTranslations = JSON.parse(
await readFile(BASE_JSON_PATH, {
encoding: 'utf8',
})
);
const flatLocalTranslations = flatRes(baseLocalTranslations);
console.log(
`Loading local base translations success! Total ${
Object.keys(flatLocalTranslations).length
} keys`
);
console.log('Fetch remote base translations...');
const baseRemoteTranslations = await getRemoteTranslations(BASE_LANGUAGES);
const flatRemoteTranslations = flatRes(baseRemoteTranslations);
console.log(
`Fetch remote base translations success! Total ${
Object.keys(flatRemoteTranslations).length
} keys`
);
const diff = differenceObject(flatLocalTranslations, flatRemoteTranslations);
console.log(''); // new line
warnDiff(diff);
console.log(''); // new line
if (process.argv.slice(2).includes('--check')) {
// check mode
return;
}
for (const key of diff.add) {
const val = flatLocalTranslations[key];
console.log(`Creating new key: ${key} -> ${val}`);
await createsNewKey(key, { [BASE_LANGUAGES]: val });
}
// TODO remove unused tags from used keys
// diff.remove.forEach(key => {
// // TODO set unused tag
// // console.log(`Add ${DEPRECATED_TAG_NAME} to ${key}`);
// addTagByKey(key, DEPRECATED_TAG_NAME);
// });
// diff.modify.forEach(key => {
// // TODO warn different between local and remote base translations
// });
// TODO send notification
};
main().catch(e => {
console.error(e);
process.exit(1);
});

35
packages/frontend/i18n/src/scripts/utils.ts
View File

@@ -1,35 +0,0 @@
export interface TranslationRes {
[x: string]: string | TranslationRes;
}
/**
* Recursively flattens a JSON object using dot notation.
*
* NOTE: input must be an object as described by JSON spec. Arbitrary
* JS objects (e.g. {a: () => 42}) may result in unexpected output.
* MOREOVER, it removes keys with empty objects/arrays as value (see
* examples bellow).
*
* @example
* flattenTranslation({a: 1, b: [{c: 2, d: {e: 3}}, 4]})
* // {a: 1, b.0.c: 2, b.0.d.e: 3, b.1: 4}
* flattenTranslation({a: 1, b: [{c: 2, d: {e: [true, false, {f: 1}]}}]})
* // {a: 1, b.0.c: 2, b.0.d.e.0: true, b.0.d.e.1: false, b.0.d.e.2.f: 1}
* flattenTranslation({a: 1, b: [], c: {}})
* // {a: 1}
*
* @param obj item to be flattened
*/
export const flattenTranslation = (
obj: string | TranslationRes,
path?: string
): TranslationRes => {
if (!(obj instanceof Object)) return { [path ?? '']: obj };
return Object.keys(obj).reduce((output, key) => {
return Object.assign(
output,
flattenTranslation(obj[key], path ? path + '.' + key : key)
);
}, {});
};

3
packages/frontend/i18n/tsconfig.json
View File

@@ -10,9 +10,6 @@
"references": [ "references": [
{ {
"path": "./tsconfig.resources.json" "path": "./tsconfig.resources.json"
},
{
"path": "./tsconfig.node.json"
} }
] ]
} }

8
packages/frontend/i18n/tsconfig.node.json
View File

@@ -1,8 +0,0 @@
{
"extends": "../../../tsconfig.json",
"compilerOptions": {
"types": ["node"],
"outDir": "./lib/scripts"
},
"include": ["./src/scripts"]
}