From 4e4cb3536207f79475ed11a233be0f9aea018fd1 Mon Sep 17 00:00:00 2001 From: tzhangchi Date: Sun, 18 Dec 2022 07:53:01 +0800 Subject: [PATCH] feat: sync master branch i18n words scripts --- packages/app/src/libs/i18n/scripts/api.ts | 186 ++++++++++++++++++ .../app/src/libs/i18n/scripts/download.ts | 134 +++++++++++++ packages/app/src/libs/i18n/scripts/request.ts | 55 ++++++ packages/app/src/libs/i18n/scripts/sync.ts | 164 +++++++++++++++ packages/app/src/libs/i18n/scripts/utils.ts | 3 + 5 files changed, 542 insertions(+) create mode 100644 packages/app/src/libs/i18n/scripts/api.ts create mode 100644 packages/app/src/libs/i18n/scripts/download.ts create mode 100644 packages/app/src/libs/i18n/scripts/request.ts create mode 100644 packages/app/src/libs/i18n/scripts/sync.ts create mode 100644 packages/app/src/libs/i18n/scripts/utils.ts diff --git a/packages/app/src/libs/i18n/scripts/api.ts b/packages/app/src/libs/i18n/scripts/api.ts new file mode 100644 index 0000000000..b38ad5875b --- /dev/null +++ b/packages/app/src/libs/i18n/scripts/api.ts @@ -0,0 +1,186 @@ +// cSpell:ignore Tolgee +import { fetchTolgee } from './request'; + +/** + * Returns all project languages + * + * See https://tolgee.io/api#operation/getAll_6 + * @example + * ```ts + * const languages = [ + * { + * id: 1000016008, + * name: 'English', + * tag: 'en', + * originalName: 'English', + * flagEmoji: '🇬🇧', + * base: true + * }, + * { + * id: 1000016013, + * name: 'Spanish', + * tag: 'es', + * originalName: 'español', + * flagEmoji: '🇪🇸', + * base: false + * }, + * { + * id: 1000016009, + * name: 'Simplified Chinese', + * tag: 'zh-Hans', + * originalName: '简体中文', + * flagEmoji: '🇨🇳', + * base: false + * }, + * { + * id: 1000016012, + * name: 'Traditional Chinese', + * tag: 'zh-Hant', + * originalName: '繁體中文', + * flagEmoji: '🇭🇰', + * base: false + * } + * ] + * ``` + */ +export const getAllProjectLanguages = async (size = 1000) => { + const url = `/languages?size=${size}`; + const resp = await fetchTolgee(url); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json: { + // eslint-disable-next-line @typescript-eslint/naming-convention + _embedded: { + languages: { + id: number; + name: string; + tag: string; + originalName: string; + flagEmoji: string; + base: boolean; + }[]; + }; + page: unknown; + } = await resp.json(); + return json._embedded.languages; +}; + +/** + * Returns translations in project + * + * See https://tolgee.io/api#operation/getTranslations_ + */ +export const getTranslations = async () => { + const url = '/translations'; + const resp = await fetchTolgee(url); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json = await resp.json(); + return json; +}; + +/** + * Returns all translations for specified languages + * + * See https://tolgee.io/api#operation/getAllTranslations_1 + */ +export const getLanguagesTranslations = async ( + languages: T +) => { + const url = `/translations/${languages}`; + const resp = await fetchTolgee(url); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json: { [key in T]?: Record } = await resp.json(); + return json; +}; + +export const getRemoteTranslations = async (languages: string) => { + const translations = await getLanguagesTranslations(languages); + if (!(languages in translations)) { + return {}; + } + // The assert is safe because we checked above + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return translations[languages]!; +}; + +/** + * Creates new key + * + * See https://tolgee.io/api#operation/create_2 + */ +export const createsNewKey = async ( + key: string, + translations: Record +) => { + const url = '/translations/keys/create'; + const resp = await fetchTolgee(url, { + method: 'POST', + body: JSON.stringify({ name: key, translations }), + }); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json = await resp.json(); + return json; +}; + +/** + * Tags a key with tag. If tag with provided name doesn't exist, it is created + * + * See https://tolgee.io/api#operation/tagKey_1 + */ +export const addTag = async (keyId: string, tagName: string) => { + const url = `/keys/${keyId}/tags`; + const resp = await fetchTolgee(url, { + method: 'PUT', + body: JSON.stringify({ name: tagName }), + }); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json = await resp.json(); + return json; +}; + +/** + * Tags a key with tag. If tag with provided name doesn't exist, it is created + * + * See https://tolgee.io/api#operation/tagKey_1 + */ +export const removeTag = async (keyId: string, tagId: number) => { + const url = `/keys/${keyId}/tags/${tagId}`; + const resp = await fetchTolgee(url, { + method: 'DELETE', + }); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json = await resp.json(); + return json; +}; + +export const addTagByKey = async (key: string, tag: string) => { + // TODO get key id by key name + // const keyId = + // addTag(keyId, tag); +}; + +/** + * Exports data + * + * See https://tolgee.io/api#operation/export_1 + */ +export const exportResources = async () => { + const url = `/export`; + const resp = await fetchTolgee(url); + + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + return resp; +}; diff --git a/packages/app/src/libs/i18n/scripts/download.ts b/packages/app/src/libs/i18n/scripts/download.ts new file mode 100644 index 0000000000..f45261c258 --- /dev/null +++ b/packages/app/src/libs/i18n/scripts/download.ts @@ -0,0 +1,134 @@ +/* eslint-disable no-console */ +// cSpell:ignore Tolgee +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { format } from 'prettier'; +import { getAllProjectLanguages, getRemoteTranslations } from './api'; +import type { TranslationRes } from './utils'; + +const RES_DIR = path.resolve(process.cwd(), 'src', 'resources'); + +const countKeys = (obj: TranslationRes) => { + let count = 0; + Object.entries(obj).forEach(([key, value]) => { + if (typeof value === 'string') { + count++; + } else { + count += countKeys(value); + } + }); + return count; +}; + +const getBaseTranslations = async (baseLanguage: { tag: string }) => { + try { + const baseTranslationsStr = await fs.readFile( + path.resolve(RES_DIR, `${baseLanguage.tag}.json`), + { encoding: 'utf8' } + ); + const baseTranslations = JSON.parse(baseTranslationsStr); + return baseTranslations; + } catch (e) { + console.error('base language:', JSON.stringify(baseLanguage)); + console.error('Failed to read base language', e); + const translations = await getRemoteTranslations(baseLanguage.tag); + await fs.writeFile( + path.resolve(RES_DIR, `${baseLanguage.tag}.json`), + JSON.stringify(translations, null, 4) + ); + } +}; + +const main = async () => { + console.log('Loading project languages...'); + const languages = await getAllProjectLanguages(); + const baseLanguage = languages.find(language => language.base); + if (!baseLanguage) { + console.error(JSON.stringify(languages)); + throw new Error('Could not find base language'); + } + console.log( + `Loading ${baseLanguage.tag} languages translations as base...` + ); + + const baseTranslations = await getBaseTranslations(baseLanguage); + const baseKeyNum = countKeys(baseTranslations); + const languagesWithTranslations = await Promise.all( + languages.map(async language => { + console.log(`Loading ${language.tag} translations...`); + const translations = await getRemoteTranslations(language.tag); + const keyNum = countKeys(translations); + + return { + ...language, + translations, + completeRate: keyNum / baseKeyNum, + }; + }) + ); + + const availableLanguages = languagesWithTranslations.filter( + language => language.completeRate > 0 + ); + + availableLanguages + // skip base language + .filter(i => !i.base) + .forEach(async language => { + await fs.writeFile( + path.resolve(RES_DIR, `${language.tag}.json`), + JSON.stringify( + { + // eslint-disable-next-line @typescript-eslint/naming-convention + '// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.': + '', + ...language.translations, + }, + null, + 4 + ) + '\n' + ); + }); + + console.log('Generating meta data...'); + const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. + // Run \`pnpm run download-resources\` to regenerate. + // To overwrite this, please overwrite ${path.basename(__filename)} + ${availableLanguages + .map( + language => + `import ${language.tag.replaceAll('-', '_')} from './${ + language.tag + }.json'` + ) + .join('\n')} + + export const LOCALES = [ + ${availableLanguages + .map(({ translations, ...language }) => + JSON.stringify({ + ...language, + res: '__RES_PLACEHOLDER', + }).replace( + '"__RES_PLACEHOLDER"', + language.tag.replaceAll('-', '_') + ) + ) + .join(',\n')} + ] as const; + `; + + await fs.writeFile( + path.resolve(RES_DIR, 'index.ts'), + format(code, { + parser: 'typescript', + singleQuote: true, + trailingComma: 'es5', + tabWidth: 4, + arrowParens: 'avoid', + }) + ); + console.log('Done'); +}; + +main(); diff --git a/packages/app/src/libs/i18n/scripts/request.ts b/packages/app/src/libs/i18n/scripts/request.ts new file mode 100644 index 0000000000..b599c71957 --- /dev/null +++ b/packages/app/src/libs/i18n/scripts/request.ts @@ -0,0 +1,55 @@ +// cSpell:ignore Tolgee +const TOLGEE_API_KEY = process.env['TOLGEE_API_KEY']; +const TOLGEE_API_URL = 'https://i18n.affine.pro'; + +if (!TOLGEE_API_KEY) { + throw new Error(`Please set "TOLGEE_API_KEY" as environment variable!`); +} + +const withTolgee = ( + fetch: typeof globalThis.fetch +): typeof globalThis.fetch => { + const baseUrl = `${TOLGEE_API_URL}/v2/projects`; + const headers = new Headers({ + 'X-API-Key': TOLGEE_API_KEY, + 'Content-Type': 'application/json', + }); + + const isRequest = (input: RequestInfo | URL): input is Request => { + return typeof input === 'object' && !('href' in input); + }; + + return new Proxy(fetch, { + apply( + target, + thisArg: unknown, + argArray: Parameters + ) { + if (isRequest(argArray[0])) { + // Request + if (!argArray[0].headers) { + argArray[0] = { + ...argArray[0], + url: `${baseUrl}${argArray[0].url}`, + headers, + }; + } + } else { + // URL or URLLike + ?RequestInit + if (typeof argArray[0] === 'string') { + argArray[0] = `${baseUrl}${argArray[0]}`; + } + if (!argArray[1]) { + argArray[1] = {}; + } + if (!argArray[1].headers) { + argArray[1].headers = headers; + } + } + // console.log('fetch', argArray); + return target.apply(thisArg, argArray); + }, + }); +}; + +export const fetchTolgee = withTolgee(globalThis.fetch); diff --git a/packages/app/src/libs/i18n/scripts/sync.ts b/packages/app/src/libs/i18n/scripts/sync.ts new file mode 100644 index 0000000000..c2ba692dc5 --- /dev/null +++ b/packages/app/src/libs/i18n/scripts/sync.ts @@ -0,0 +1,164 @@ +/* eslint-disable no-console */ +// cSpell:ignore Tolgee +import { readFile } from 'fs/promises'; +import path from 'path'; +import { addTagByKey, createsNewKey, getRemoteTranslations } from './api'; +import type { TranslationRes } from './utils'; + +const BASE_JSON_PATH = path.resolve( + process.cwd(), + 'src', + 'resources', + 'en.json' +); +const BASE_LANGUAGES = 'en' as const; + +const DEPRECATED_TAG_NAME = 'unused' as const; + +/** + * + * @example + * ```ts + * flatRes({ a: { b: 'c' } }); // { 'a.b': 'c' } + * ``` + */ +const flatRes = (obj: TranslationRes) => { + const getEntries = (o: TranslationRes, prefix = ''): [string, string][] => + Object.entries(o).flatMap<[string, string]>(([k, v]) => + typeof v !== 'string' + ? getEntries(v, `${prefix}${k}.`) + : [[`${prefix}${k}`, v]] + ); + return Object.fromEntries(getEntries(obj)); +}; + +const differenceObject = ( + newObj: Record, + oldObj: Record +) => { + const add: string[] = []; + const remove: string[] = []; + const modify: string[] = []; + const both: string[] = []; + + Object.keys(newObj).forEach(key => { + if (!(key in oldObj)) { + add.push(key); + } else { + both.push(key); + } + }); + + Object.keys(oldObj).forEach(key => { + if (!(key in newObj)) { + remove.push(key); + } + }); + + both.forEach(key => { + if (!(key in newObj) || !(key in oldObj)) { + throw new Error('Unreachable'); + } + const newVal = newObj[key]; + const oldVal = oldObj[key]; + if (newVal !== oldVal) { + modify.push(key); + } + }); + return { add, remove, modify }; +}; + +function warnDiff(diff: { add: string[]; remove: string[]; modify: string[] }) { + if (diff.add.length) { + console.log('New keys found:', diff.add.join(', ')); + //See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-notice-message + process.env['CI'] && + console.log( + `::notice file=${BASE_JSON_PATH},line=1,title=New keys::${diff.add.join( + ', ' + )}` + ); + } + if (diff.remove.length) { + console.warn('[WARN]', 'Unused keys found:', diff.remove.join(', ')); + process.env['CI'] && + console.warn( + `::notice file=${BASE_JSON_PATH},line=1,title=Unused keys::${diff.remove.join( + ', ' + )}` + ); + } + if (diff.modify.length) { + console.warn( + '[WARN]', + 'Inconsistent keys found:', + diff.modify.join(', ') + ); + process.env['CI'] && + console.warn( + `::warning file=${BASE_JSON_PATH},line=1,title=Inconsistent keys::${diff.modify.join( + ', ' + )}` + ); + } +} + +const main = async () => { + console.log('Loading local base translations...'); + const baseLocalTranslations = JSON.parse( + await readFile(BASE_JSON_PATH, { + encoding: 'utf8', + }) + ); + const flatLocalTranslations = flatRes(baseLocalTranslations); + console.log( + `Loading local base translations success! Total ${ + Object.keys(flatLocalTranslations).length + } keys` + ); + + console.log('Fetch remote base translations...'); + const baseRemoteTranslations = await getRemoteTranslations(BASE_LANGUAGES); + const flatRemoteTranslations = flatRes(baseRemoteTranslations); + console.log( + `Fetch remote base translations success! Total ${ + Object.keys(flatRemoteTranslations).length + } keys` + ); + + const diff = differenceObject( + flatLocalTranslations, + flatRemoteTranslations + ); + + console.log(''); // new line + warnDiff(diff); + console.log(''); // new line + + if (process.argv.slice(2).includes('--check')) { + // check mode + return; + } + + diff.add.forEach(async key => { + const val = flatLocalTranslations[key]; + console.log(`Creating new key: ${key} -> ${val}`); + await createsNewKey(key, { [BASE_LANGUAGES]: val }); + }); + + // TODO remove unused tags from used keys + + diff.remove.forEach(key => { + // TODO set unused tag + // console.log(`Add ${DEPRECATED_TAG_NAME} to ${key}`); + addTagByKey(key, DEPRECATED_TAG_NAME); + }); + + diff.modify.forEach(key => { + // TODO warn different between local and remote base translations + }); + + // TODO send notification +}; + +main(); diff --git a/packages/app/src/libs/i18n/scripts/utils.ts b/packages/app/src/libs/i18n/scripts/utils.ts new file mode 100644 index 0000000000..129a8febc3 --- /dev/null +++ b/packages/app/src/libs/i18n/scripts/utils.ts @@ -0,0 +1,3 @@ +export interface TranslationRes { + [x: string]: string | TranslationRes; +}