From e6d05fefaa8b57cbe0af298b57c264bd534672c7 Mon Sep 17 00:00:00 2001 From: lawvs <18554747+lawvs@users.noreply.github.com> Date: Thu, 8 Sep 2022 18:45:56 +0800 Subject: [PATCH] feat: add translation sync scripts --- libs/datasource/i18n/README.md | 11 +- libs/datasource/i18n/package.json | 3 + libs/datasource/i18n/src/base.json | 24 ++++ libs/datasource/i18n/src/scripts/api.ts | 111 +++++++++++++++ libs/datasource/i18n/src/scripts/request.ts | 53 ++++++++ libs/datasource/i18n/src/scripts/sync.ts | 142 ++++++++++++++++++++ libs/datasource/i18n/tsconfig.json | 11 +- 7 files changed, 350 insertions(+), 5 deletions(-) create mode 100644 libs/datasource/i18n/src/base.json create mode 100644 libs/datasource/i18n/src/scripts/api.ts create mode 100644 libs/datasource/i18n/src/scripts/request.ts create mode 100644 libs/datasource/i18n/src/scripts/sync.ts diff --git a/libs/datasource/i18n/README.md b/libs/datasource/i18n/README.md index e31ddf7292..ff902b333c 100644 --- a/libs/datasource/i18n/README.md +++ b/libs/datasource/i18n/README.md @@ -33,10 +33,15 @@ const App = () => { }; ``` -## TODO +## How to sync translations -- [ ] language detection -- [ ] storage +- Set token as environment variable + +```shell +export TOLGEE_API_KEY=tgpak_XXXXXXX +``` + +- Run the `sync` script from package script ## References diff --git a/libs/datasource/i18n/package.json b/libs/datasource/i18n/package.json index 170fd2691e..8f6afe9b85 100644 --- a/libs/datasource/i18n/package.json +++ b/libs/datasource/i18n/package.json @@ -1,6 +1,9 @@ { "name": "@toeverything/datasource/i18n", "version": "0.0.1", + "scripts": { + "sync": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/sync.ts" + }, "dependencies": { "i18next": "^21.9.1", "jotai": "^1.8.1", diff --git a/libs/datasource/i18n/src/base.json b/libs/datasource/i18n/src/base.json new file mode 100644 index 0000000000..e243d71db1 --- /dev/null +++ b/libs/datasource/i18n/src/base.json @@ -0,0 +1,24 @@ +{ + "Sync to Disk": "Sync to Disk", + "Share": "Share", + "WarningTips": { + "IsNotfsApiSupported": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC DATA TO DISK with the latest version of Chromium based browser like Chrome/Edge", + "IsNotLocalWorkspace": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC TO DISK.", + "DoNotStore": "AFFiNE is under active development and the current version is UNSTABLE. Please DO NOT store information or data" + }, + "Layout": "Layout", + "Comment": "Comment", + "Settings": "Settings", + "ComingSoon": "Layout Settings Coming Soon...", + "Duplicate Page": "Duplicate Page", + "Copy Page Link": "Copy Page Link", + "Language": "Language", + "Clear Workspace": "Clear Workspace", + "Export As Markdown": "Export As Markdown", + "Export As HTML": "Export As HTML", + "Export As PDF (Unsupported)": "Export As PDF (Unsupported)", + "Import Workspace": "Import Workspace", + "Export Workspace": "Export Workspace", + "Last edited by": "Last edited by {{name}}", + "Logout": "Logout" +} diff --git a/libs/datasource/i18n/src/scripts/api.ts b/libs/datasource/i18n/src/scripts/api.ts new file mode 100644 index 0000000000..52011e6042 --- /dev/null +++ b/libs/datasource/i18n/src/scripts/api.ts @@ -0,0 +1,111 @@ +// cSpell:ignore Tolgee +import { fetchTolgee } from './request'; + +/** + * Returns all project languages + * + * See https://tolgee.io/api#operation/getAll_6 + */ +export const getAllProjectLanguages = async () => { + const url = '/languages?size=1000'; + const resp = await fetchTolgee(url); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json = await resp.json(); + return json; +}; + +/** + * Returns translations in project + * + * See https://tolgee.io/api#operation/getTranslations_ + */ +export const getTranslations = async () => { + const url = '/translations'; + const resp = await fetchTolgee(url); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json = await resp.json(); + return json; +}; + +/** + * Returns all translations for specified languages + * + * See https://tolgee.io/api#operation/getAllTranslations_1 + */ +export const getLanguagesTranslations = async ( + languages: T +) => { + const url = `/translations/${languages}`; + const resp = await fetchTolgee(url); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json: { [key in T]?: Record } = await resp.json(); + return json; +}; + +/** + * Creates new key + * + * See https://tolgee.io/api#operation/create_2 + */ +export const createsNewKey = async ( + key: string, + translations: Record +) => { + const url = '/translations/keys/create'; + const resp = await fetchTolgee(url, { + method: 'POST', + body: JSON.stringify({ name: key, translations }), + }); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json = await resp.json(); + return json; +}; + +/** + * Tags a key with tag. If tag with provided name doesn't exist, it is created + * + * See https://tolgee.io/api#operation/tagKey_1 + */ +export const addTag = async (keyId: string, tagName: string) => { + const url = `/keys/${keyId}/tags`; + const resp = await fetchTolgee(url, { + method: 'PUT', + body: JSON.stringify({ name: tagName }), + }); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json = await resp.json(); + return json; +}; + +/** + * Tags a key with tag. If tag with provided name doesn't exist, it is created + * + * See https://tolgee.io/api#operation/tagKey_1 + */ +export const removeTag = async (keyId: string, tagId: number) => { + const url = `/keys/${keyId}/tags/${tagId}`; + const resp = await fetchTolgee(url, { + method: 'DELETE', + }); + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + const json = await resp.json(); + return json; +}; + +export const addTagByKey = async (key: string, tag: string) => { + // TODO get key id by key name + // const keyId = + // addTag(keyId, tag); +}; diff --git a/libs/datasource/i18n/src/scripts/request.ts b/libs/datasource/i18n/src/scripts/request.ts new file mode 100644 index 0000000000..64e18bd9bd --- /dev/null +++ b/libs/datasource/i18n/src/scripts/request.ts @@ -0,0 +1,53 @@ +// cSpell:ignore Tolgee +const TOLGEE_API_KEY = process.env['TOLGEE_API_KEY']; +const TOLGEE_API_URL = 'https://i18n.affine.pro'; + +if (!TOLGEE_API_KEY) { + throw new Error(`Please set "TOLGEE_API_KEY" as environment variable!`); +} + +const withTolgee = ( + fetch: typeof globalThis.fetch +): typeof globalThis.fetch => { + const headers = new Headers({ + 'X-API-Key': TOLGEE_API_KEY, + 'Content-Type': 'application/json', + }); + + const isRequest = (input: RequestInfo | URL): input is Request => { + return typeof input === 'object' && !('href' in input); + }; + + return new Proxy(fetch, { + apply( + target, + thisArg: unknown, + argArray: Parameters + ) { + if (isRequest(argArray[0])) { + // Request + if (!argArray[0].headers) { + argArray[0] = { + ...argArray[0], + url: `${TOLGEE_API_URL}/v2/projects${argArray[0].url}`, + headers, + }; + } + } else { + // URL or URLLike + ?RequestInit + if (typeof argArray[0] === 'string') { + argArray[0] = TOLGEE_API_URL + argArray[0]; + } + if (!argArray[1]) { + argArray[1] = {}; + } + if (!argArray[1].headers) { + argArray[1].headers = headers; + } + } + return target.apply(thisArg, argArray); + }, + }); +}; + +export const fetchTolgee = withTolgee(globalThis.fetch); diff --git a/libs/datasource/i18n/src/scripts/sync.ts b/libs/datasource/i18n/src/scripts/sync.ts new file mode 100644 index 0000000000..5b264bf329 --- /dev/null +++ b/libs/datasource/i18n/src/scripts/sync.ts @@ -0,0 +1,142 @@ +// cSpell:ignore Tolgee +import { readFile } from 'fs/promises'; +import path from 'path'; +import { addTagByKey, createsNewKey, getLanguagesTranslations } from './api'; + +const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json'); +const BASE_LANGUAGES = 'en' as const; + +const DEPRECATED_TAG_NAME = 'unused' as const; + +interface TranslationRes { + [x: string]: string | TranslationRes; +} + +const getRemoteTranslations = async (languages: string) => { + const translations = await getLanguagesTranslations(languages); + if (!(languages in translations)) { + console.log(translations); + throw new Error( + 'Failed to get base languages translation! base languages: ' + + languages + ); + } + // The assert is safe because we checked above + return translations[languages]!; +}; + +/** + * + * @example + * ```ts + * flatRes({ a: { b: 'c' } }); // { 'a.b': 'c' } + * ``` + */ +const flatRes = (obj: TranslationRes) => { + const getEntries = (o: TranslationRes, prefix = ''): [string, string][] => + Object.entries(o).flatMap<[string, string]>(([k, v]) => + typeof v !== 'string' + ? getEntries(v, `${prefix}${k}.`) + : [[`${prefix}${k}`, v]] + ); + return Object.fromEntries(getEntries(obj)); +}; + +const differenceObject = ( + newObj: Record, + oldObj: Record +) => { + const add: string[] = []; + const remove: string[] = []; + const modify: string[] = []; + const both: string[] = []; + + Object.keys(newObj).forEach(key => { + if (!(key in oldObj)) { + add.push(key); + } else { + both.push(key); + } + }); + + Object.keys(oldObj).forEach(key => { + if (!(key in newObj)) { + remove.push(key); + } + }); + + both.forEach(key => { + if (!(key in newObj) || !(key in oldObj)) { + throw new Error('Unreachable'); + } + const newVal = newObj[key]; + const oldVal = oldObj[key]; + if (newVal !== oldVal) { + modify.push(key); + } + }); + return { add, remove, modify }; +}; + +const main = async () => { + console.log('Loading local base translations...'); + const baseLocalTranslations = JSON.parse( + await readFile(BASE_JSON_PATH, { + encoding: 'utf8', + }) + ); + const flatLocalTranslations = flatRes(baseLocalTranslations); + console.log( + `Loading local base translations success! Total ${ + Object.keys(flatLocalTranslations).length + } keys` + ); + + console.log('Fetch remote base translations...'); + const baseRemoteTranslations = await getRemoteTranslations(BASE_LANGUAGES); + const flatRemoteTranslations = flatRes(baseRemoteTranslations); + console.log( + `Fetch remote base translations success! Total ${ + Object.keys(flatRemoteTranslations).length + } keys` + ); + + const diff = differenceObject( + flatLocalTranslations, + flatRemoteTranslations + ); + + console.log(''); // new line + diff.add.length && console.log('New keys found:', diff.add.join(', ')); + diff.remove.length && + console.warn('[WARN]', 'Unused keys found:', diff.remove.join(', ')); + diff.modify.length && + console.warn( + '[WARN]', + 'Inconsistent keys found:', + diff.modify.join(', ') + ); + console.log(''); // new line + + diff.add.forEach(async key => { + const val = flatLocalTranslations[key]; + console.log(`Creating new key: ${key} -> ${val}`); + await createsNewKey(key, { [BASE_LANGUAGES]: val }); + }); + + // TODO remove unused tags from used keys + + diff.remove.forEach(key => { + // TODO set unused tag + // console.log(`Add ${DEPRECATED_TAG_NAME} to ${key}`); + addTagByKey(key, DEPRECATED_TAG_NAME); + }); + + diff.modify.forEach(key => { + // TODO warn different between local and remote base translations + }); + + // TODO send notification +}; + +main(); diff --git a/libs/datasource/i18n/tsconfig.json b/libs/datasource/i18n/tsconfig.json index 7e4c7a4ad6..c78be8fd2e 100644 --- a/libs/datasource/i18n/tsconfig.json +++ b/libs/datasource/i18n/tsconfig.json @@ -11,7 +11,8 @@ "noImplicitOverride": true, "noPropertyAccessFromIndexSignature": true, "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true + "noFallthroughCasesInSwitch": true, + "resolveJsonModule": true }, "files": [], "include": [], @@ -22,5 +23,11 @@ { "path": "./tsconfig.spec.json" } - ] + ], + "ts-node": { + "compilerOptions": { + "module": "NodeNext" + }, + "esm": true + } }