feat: add translation sync scripts

libs/datasource/i18n/README.md

@@ -33,10 +33,15 @@ const App = () => {
 };
 ```
 
-## TODO
+## How to sync translations
 
--   [ ] language detection
--   [ ] storage
+-   Set token as environment variable
+
+```shell
+export TOLGEE_API_KEY=tgpak_XXXXXXX
+```
+
+-   Run the `sync` script from package script
 
 ## References
 

libs/datasource/i18n/package.json

@@ -1,6 +1,9 @@
 {
     "name": "@toeverything/datasource/i18n",
     "version": "0.0.1",
+    "scripts": {
+        "sync": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/sync.ts"
+    },
     "dependencies": {
         "i18next": "^21.9.1",
         "jotai": "^1.8.1",

libs/datasource/i18n/src/base.json

@@ -0,0 +1,24 @@
+{
+    "Sync to Disk": "Sync to Disk",
+    "Share": "Share",
+    "WarningTips": {
+        "IsNotfsApiSupported": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC DATA TO DISK with the latest version of Chromium based browser like Chrome/Edge",
+        "IsNotLocalWorkspace": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC TO DISK.",
+        "DoNotStore": "AFFiNE is under active development and the current version is UNSTABLE. Please DO NOT store information or data"
+    },
+    "Layout": "Layout",
+    "Comment": "Comment",
+    "Settings": "Settings",
+    "ComingSoon": "Layout Settings Coming Soon...",
+    "Duplicate Page": "Duplicate Page",
+    "Copy Page Link": "Copy Page Link",
+    "Language": "Language",
+    "Clear Workspace": "Clear Workspace",
+    "Export As Markdown": "Export As Markdown",
+    "Export As HTML": "Export As HTML",
+    "Export As PDF (Unsupported)": "Export As PDF (Unsupported)",
+    "Import Workspace": "Import Workspace",
+    "Export Workspace": "Export Workspace",
+    "Last edited by": "Last edited by {{name}}",
+    "Logout": "Logout"
+}

libs/datasource/i18n/src/scripts/api.ts

@@ -0,0 +1,111 @@
+// cSpell:ignore Tolgee
+import { fetchTolgee } from './request';
+
+/**
+ * Returns all project languages
+ *
+ * See https://tolgee.io/api#operation/getAll_6
+ */
+export const getAllProjectLanguages = async () => {
+    const url = '/languages?size=1000';
+    const resp = await fetchTolgee(url);
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json = await resp.json();
+    return json;
+};
+
+/**
+ * Returns translations in project
+ *
+ * See https://tolgee.io/api#operation/getTranslations_
+ */
+export const getTranslations = async () => {
+    const url = '/translations';
+    const resp = await fetchTolgee(url);
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json = await resp.json();
+    return json;
+};
+
+/**
+ * Returns all translations for specified languages
+ *
+ * See https://tolgee.io/api#operation/getAllTranslations_1
+ */
+export const getLanguagesTranslations = async <T extends string>(
+    languages: T
+) => {
+    const url = `/translations/${languages}`;
+    const resp = await fetchTolgee(url);
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json: { [key in T]?: Record<string, string> } = await resp.json();
+    return json;
+};
+
+/**
+ * Creates new key
+ *
+ * See https://tolgee.io/api#operation/create_2
+ */
+export const createsNewKey = async (
+    key: string,
+    translations: Record<string, string>
+) => {
+    const url = '/translations/keys/create';
+    const resp = await fetchTolgee(url, {
+        method: 'POST',
+        body: JSON.stringify({ name: key, translations }),
+    });
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json = await resp.json();
+    return json;
+};
+
+/**
+ * Tags a key with tag. If tag with provided name doesn't exist, it is created
+ *
+ * See https://tolgee.io/api#operation/tagKey_1
+ */
+export const addTag = async (keyId: string, tagName: string) => {
+    const url = `/keys/${keyId}/tags`;
+    const resp = await fetchTolgee(url, {
+        method: 'PUT',
+        body: JSON.stringify({ name: tagName }),
+    });
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json = await resp.json();
+    return json;
+};
+
+/**
+ * Tags a key with tag. If tag with provided name doesn't exist, it is created
+ *
+ * See https://tolgee.io/api#operation/tagKey_1
+ */
+export const removeTag = async (keyId: string, tagId: number) => {
+    const url = `/keys/${keyId}/tags/${tagId}`;
+    const resp = await fetchTolgee(url, {
+        method: 'DELETE',
+    });
+    if (resp.status < 200 || resp.status >= 300) {
+        throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
+    }
+    const json = await resp.json();
+    return json;
+};
+
+export const addTagByKey = async (key: string, tag: string) => {
+    // TODO get key id by key name
+    // const keyId =
+    // addTag(keyId, tag);
+};

libs/datasource/i18n/src/scripts/request.ts

@@ -0,0 +1,53 @@
+// cSpell:ignore Tolgee
+const TOLGEE_API_KEY = process.env['TOLGEE_API_KEY'];
+const TOLGEE_API_URL = 'https://i18n.affine.pro';
+
+if (!TOLGEE_API_KEY) {
+    throw new Error(`Please set "TOLGEE_API_KEY" as environment variable!`);
+}
+
+const withTolgee = (
+    fetch: typeof globalThis.fetch
+): typeof globalThis.fetch => {
+    const headers = new Headers({
+        'X-API-Key': TOLGEE_API_KEY,
+        'Content-Type': 'application/json',
+    });
+
+    const isRequest = (input: RequestInfo | URL): input is Request => {
+        return typeof input === 'object' && !('href' in input);
+    };
+
+    return new Proxy(fetch, {
+        apply(
+            target,
+            thisArg: unknown,
+            argArray: Parameters<typeof globalThis.fetch>
+        ) {
+            if (isRequest(argArray[0])) {
+                // Request
+                if (!argArray[0].headers) {
+                    argArray[0] = {
+                        ...argArray[0],
+                        url: `${TOLGEE_API_URL}/v2/projects${argArray[0].url}`,
+                        headers,
+                    };
+                }
+            } else {
+                // URL or URLLike + ?RequestInit
+                if (typeof argArray[0] === 'string') {
+                    argArray[0] = TOLGEE_API_URL + argArray[0];
+                }
+                if (!argArray[1]) {
+                    argArray[1] = {};
+                }
+                if (!argArray[1].headers) {
+                    argArray[1].headers = headers;
+                }
+            }
+            return target.apply(thisArg, argArray);
+        },
+    });
+};
+
+export const fetchTolgee = withTolgee(globalThis.fetch);

libs/datasource/i18n/src/scripts/sync.ts

@@ -0,0 +1,142 @@
+// cSpell:ignore Tolgee
+import { readFile } from 'fs/promises';
+import path from 'path';
+import { addTagByKey, createsNewKey, getLanguagesTranslations } from './api';
+
+const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json');
+const BASE_LANGUAGES = 'en' as const;
+
+const DEPRECATED_TAG_NAME = 'unused' as const;
+
+interface TranslationRes {
+    [x: string]: string | TranslationRes;
+}
+
+const getRemoteTranslations = async (languages: string) => {
+    const translations = await getLanguagesTranslations(languages);
+    if (!(languages in translations)) {
+        console.log(translations);
+        throw new Error(
+            'Failed to get base languages translation! base languages: ' +
+                languages
+        );
+    }
+    // The assert is safe because we checked above
+    return translations[languages]!;
+};
+
+/**
+ *
+ * @example
+ * ```ts
+ * flatRes({ a: { b: 'c' } }); // { 'a.b': 'c' }
+ * ```
+ */
+const flatRes = (obj: TranslationRes) => {
+    const getEntries = (o: TranslationRes, prefix = ''): [string, string][] =>
+        Object.entries(o).flatMap<[string, string]>(([k, v]) =>
+            typeof v !== 'string'
+                ? getEntries(v, `${prefix}${k}.`)
+                : [[`${prefix}${k}`, v]]
+        );
+    return Object.fromEntries(getEntries(obj));
+};
+
+const differenceObject = (
+    newObj: Record<string, string>,
+    oldObj: Record<string, string>
+) => {
+    const add: string[] = [];
+    const remove: string[] = [];
+    const modify: string[] = [];
+    const both: string[] = [];
+
+    Object.keys(newObj).forEach(key => {
+        if (!(key in oldObj)) {
+            add.push(key);
+        } else {
+            both.push(key);
+        }
+    });
+
+    Object.keys(oldObj).forEach(key => {
+        if (!(key in newObj)) {
+            remove.push(key);
+        }
+    });
+
+    both.forEach(key => {
+        if (!(key in newObj) || !(key in oldObj)) {
+            throw new Error('Unreachable');
+        }
+        const newVal = newObj[key];
+        const oldVal = oldObj[key];
+        if (newVal !== oldVal) {
+            modify.push(key);
+        }
+    });
+    return { add, remove, modify };
+};
+
+const main = async () => {
+    console.log('Loading local base translations...');
+    const baseLocalTranslations = JSON.parse(
+        await readFile(BASE_JSON_PATH, {
+            encoding: 'utf8',
+        })
+    );
+    const flatLocalTranslations = flatRes(baseLocalTranslations);
+    console.log(
+        `Loading local base translations success! Total ${
+            Object.keys(flatLocalTranslations).length
+        } keys`
+    );
+
+    console.log('Fetch remote base translations...');
+    const baseRemoteTranslations = await getRemoteTranslations(BASE_LANGUAGES);
+    const flatRemoteTranslations = flatRes(baseRemoteTranslations);
+    console.log(
+        `Fetch remote base translations success! Total ${
+            Object.keys(flatRemoteTranslations).length
+        } keys`
+    );
+
+    const diff = differenceObject(
+        flatLocalTranslations,
+        flatRemoteTranslations
+    );
+
+    console.log(''); // new line
+    diff.add.length && console.log('New keys found:', diff.add.join(', '));
+    diff.remove.length &&
+        console.warn('[WARN]', 'Unused keys found:', diff.remove.join(', '));
+    diff.modify.length &&
+        console.warn(
+            '[WARN]',
+            'Inconsistent keys found:',
+            diff.modify.join(', ')
+        );
+    console.log(''); // new line
+
+    diff.add.forEach(async key => {
+        const val = flatLocalTranslations[key];
+        console.log(`Creating new key: ${key} -> ${val}`);
+        await createsNewKey(key, { [BASE_LANGUAGES]: val });
+    });
+
+    // TODO remove unused tags from used keys
+
+    diff.remove.forEach(key => {
+        // TODO set unused tag
+        // console.log(`Add ${DEPRECATED_TAG_NAME} to ${key}`);
+        addTagByKey(key, DEPRECATED_TAG_NAME);
+    });
+
+    diff.modify.forEach(key => {
+        // TODO warn different between local and remote base translations
+    });
+
+    // TODO send notification
+};
+
+main();

libs/datasource/i18n/tsconfig.json

@@ -11,7 +11,8 @@
         "noImplicitOverride": true,
         "noPropertyAccessFromIndexSignature": true,
         "noImplicitReturns": true,
-        "noFallthroughCasesInSwitch": true
+        "noFallthroughCasesInSwitch": true,
+        "resolveJsonModule": true
     },
     "files": [],
     "include": [],
@@ -22,5 +23,11 @@
         {
             "path": "./tsconfig.spec.json"
         }
-    ]
+    ],
+    "ts-node": {
+        "compilerOptions": {
+            "module": "NodeNext"
+        },
+        "esm": true
+    }
 }