feat: sync i18n with crowdin (#8293)

.github/workflows/languages-sync.yml

@@ -1,35 +0,0 @@
-name: Languages Sync
-
-on:
-  push:
-    branches: ['canary']
-    paths:
-      - 'packages/frontend/i18n/**'
-      - '.github/workflows/languages-sync.yml'
-      - '!.github/actions/setup-node/action.yml'
-  pull_request_target:
-    branches: ['canary']
-    paths:
-      - 'packages/frontend/i18n/**'
-      - '.github/workflows/languages-sync.yml'
-      - '!.github/actions/setup-node/action.yml'
-  workflow_dispatch:
-
-jobs:
-  main:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: ./.github/actions/setup-node
-      - name: Check Language Key
-        if: github.ref != 'refs/heads/canary'
-        run: yarn workspace @affine/i18n run sync-languages:check
-        env:
-          TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}
-
-      - name: Sync Languages
-        if: github.ref == 'refs/heads/canary'
-        run: yarn workspace @affine/i18n run sync-languages
-        env:
-          TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }}

.github/workflows/sync-i18n.yml

@@ -0,0 +1,38 @@
+name: Sync I18n with Crowdin
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  synchronize-with-crowdin:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: crowdin action
+        uses: crowdin/github-action@v2
+        with:
+          upload_sources: true
+          upload_translations: true
+          download_translations: true
+          auto_approve_imported: true
+          import_eq_suggestions: true
+          export_only_approved: true
+          skip_untranslated_strings: true
+          localization_branch_name: l10n_crowdin_translations
+          create_pull_request: true
+          pull_request_title: 'New Crowdin Translations'
+          pull_request_body: 'New Crowdin translations by [Crowdin GH Action](https://github.com/crowdin/github-action)'
+          pull_request_base_branch_name: 'canary'
+          config: packages/frontend/i18n/crowdin.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
+          CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}

packages/frontend/i18n/crowdin.yml

@@ -0,0 +1,12 @@
+'base_path': '.'
+'base_url': 'https://api.crowdin.com'
+
+'preserve_hierarchy': true
+
+'files':
+  [
+    {
+      'source': '/src/resources/en.json',
+      'translation': '/src/resources/%locale%.json',
+    },
+  ]

packages/frontend/i18n/package.json

@@ -8,10 +8,7 @@
   },
   "scripts": {
     "build": "node build.mjs",
-    "dev": "node dev.mjs",
-    "sync-languages": "node --loader ts-node/esm/transpile-only src/scripts/sync.ts",
-    "sync-languages:check": "yarn run sync-languages --check",
-    "download-resources": "node --loader ts-node/esm/transpile-only src/scripts/download.ts"
+    "dev": "node dev.mjs"
   },
   "keywords": [],
   "repository": {

packages/frontend/i18n/src/scripts/api.ts

@@ -1,212 +0,0 @@
-import type { Response } from 'undici-types';
-
-// cSpell:ignore Tolgee
-import { fetchTolgee } from './request.js';
-
-/**
- * Returns all project languages
- *
- * See https://tolgee.io/api#operation/getAll_6
- * @example
- * ```ts
- * const languages = [
- *   {
- *     id: 1000016008,
- *     name: 'English',
- *     tag: 'en',
- *     originalName: 'English',
- *     flagEmoji: '🇬🇧',
- *     base: true
- *   },
- *   {
- *     id: 1000016013,
- *     name: 'Spanish',
- *     tag: 'es',
- *     originalName: 'español',
- *     flagEmoji: '🇪🇸',
- *     base: false
- *   },
- *   {
- *     id: 1000016009,
- *     name: 'Simplified Chinese',
- *     tag: 'zh-Hans',
- *     originalName: '简体中文',
- *     flagEmoji: '🇨🇳',
- *     base: false
- *   },
- *   {
- *     id: 1000016012,
- *     name: 'Traditional Chinese',
- *     tag: 'zh-Hant',
- *     originalName: '繁體中文',
- *     flagEmoji: '🇭🇰',
- *     base: false
- *   }
- * ]
- * ```
- */
-
-export const getAllProjectLanguages = async (
-  size = 1000
-): Promise<
-  {
-    id: number;
-    name: string;
-    tag: string;
-    originalName: string;
-    flagEmoji: string;
-    base: boolean;
-  }[]
-> => {
-  const url = `/languages?size=${size}`;
-  const resp = await fetchTolgee(url);
-  if (resp.status < 200 || resp.status >= 300) {
-    throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
-  }
-  const json: {
-    _embedded: {
-      languages: {
-        id: number;
-        name: string;
-        tag: string;
-        originalName: string;
-        flagEmoji: string;
-        base: boolean;
-      }[];
-    };
-    page: unknown;
-  } = (await resp.json()) as any;
-  return json._embedded.languages;
-};
-
-/**
- * Returns translations in project
- *
- * See https://tolgee.io/api#operation/getTranslations_
- */
-export const getTranslations = async (): Promise<unknown> => {
-  const url = '/translations';
-  const resp = await fetchTolgee(url);
-  if (resp.status < 200 || resp.status >= 300) {
-    throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
-  }
-  const json = await resp.json();
-  return json;
-};
-
-/**
- * Returns all translations for specified languages
- *
- * See https://tolgee.io/api#operation/getAllTranslations_1
- */
-export const getLanguagesTranslations = async <T extends string>(
-  languages: T
-): Promise<{ [key in T]?: Record<string, string> }> => {
-  const url = `/translations/${languages}`;
-  const resp = await fetchTolgee(url);
-  if (resp.status < 200 || resp.status >= 300) {
-    throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
-  }
-  const json = await resp.json();
-  return json as { [key in T]?: Record<string, string> };
-};
-
-export const getRemoteTranslations = async (
-  languages: string
-): Promise<Record<string, string>> => {
-  const translations = await getLanguagesTranslations(languages);
-  if (!(languages in translations)) {
-    return {};
-  }
-  // The assert is safe because we checked above
-  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-  return translations[languages]!;
-};
-
-/**
- * Creates new key
- *
- * See https://tolgee.io/api#operation/create_2
- */
-export const createsNewKey = async (
-  key: string,
-  translations: Record<string, string>
-): Promise<unknown> => {
-  const url = '/translations/keys/create';
-  const resp = await fetchTolgee(url, {
-    method: 'POST',
-    body: JSON.stringify({ name: key, translations }),
-  });
-  if (resp.status < 200 || resp.status >= 300) {
-    /**
-     * There are some problems in the i18n backend,
-     * which is used to temporarily solve the ci error.
-     */
-    console.warn(url + ' ' + resp.status + '\n' + (await resp.text()));
-    return;
-  }
-  const json = await resp.json();
-  return json;
-};
-
-/**
- * Tags a key with tag. If tag with provided name doesn't exist, it is created
- *
- * See https://tolgee.io/api#operation/tagKey_1
- */
-export const addTag = async (
-  keyId: string,
-  tagName: string
-): Promise<unknown> => {
-  const url = `/keys/${keyId}/tags`;
-  const resp = await fetchTolgee(url, {
-    method: 'PUT',
-    body: JSON.stringify({ name: tagName }),
-  });
-  if (resp.status < 200 || resp.status >= 300) {
-    throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
-  }
-  const json = await resp.json();
-  return json;
-};
-
-/**
- * Tags a key with tag. If tag with provided name doesn't exist, it is created
- *
- * See https://tolgee.io/api#operation/tagKey_1
- */
-export const removeTag = async (
-  keyId: string,
-  tagId: number
-): Promise<unknown> => {
-  const url = `/keys/${keyId}/tags/${tagId}`;
-  const resp = await fetchTolgee(url, {
-    method: 'DELETE',
-  });
-  if (resp.status < 200 || resp.status >= 300) {
-    throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
-  }
-  const json = await resp.json();
-  return json;
-};
-
-// export const addTagByKey = async (key: string, tag: string) => {
-//   // TODO get key id by key name
-//   // const keyId =
-//   // addTag(keyId, tag);
-// };
-
-/**
- * Exports data
- *
- * See https://tolgee.io/api#operation/export_1
- */
-export const exportResources = async (): Promise<Response> => {
-  const url = `/export`;
-  const resp = await fetchTolgee(url);
-
-  if (resp.status < 200 || resp.status >= 300) {
-    throw new Error(url + ' ' + resp.status + '\n' + (await resp.text()));
-  }
-  return resp;
-};

packages/frontend/i18n/src/scripts/download.ts

@@ -1,155 +0,0 @@
-// cSpell:ignore Tolgee
-import * as fs from 'node:fs/promises';
-import * as path from 'node:path';
-
-import { format } from 'prettier';
-
-import { getAllProjectLanguages, getRemoteTranslations } from './api.js';
-import type { TranslationRes } from './utils.js';
-import { flattenTranslation } from './utils.js';
-
-const INDENT = 2;
-const RES_DIR = path.resolve(process.cwd(), 'src', 'resources');
-
-const countKeys = (obj: TranslationRes | null) => {
-  if (!obj) {
-    return 0;
-  }
-  let count = 0;
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  Object.entries(obj).forEach(([_, value]) => {
-    if (typeof value === 'string') {
-      count++;
-    } else {
-      count += countKeys(value);
-    }
-  });
-  return count;
-};
-
-const getBaseTranslations = async (baseLanguage: { tag: string }) => {
-  try {
-    const baseTranslationsStr = await fs.readFile(
-      path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
-      { encoding: 'utf8' }
-    );
-    const baseTranslations = JSON.parse(baseTranslationsStr);
-    return baseTranslations;
-  } catch (e) {
-    console.error('base language:', JSON.stringify(baseLanguage));
-    console.error('Failed to read base language', e);
-    const translations = await getRemoteTranslations(baseLanguage.tag);
-    await fs.writeFile(
-      path.resolve(RES_DIR, `${baseLanguage.tag}.json`),
-      JSON.stringify(translations, null, 4)
-    );
-  }
-};
-
-const main = async () => {
-  try {
-    await fs.access(RES_DIR);
-  } catch {
-    fs.mkdir(RES_DIR).catch(console.error);
-    console.log('Create directory', RES_DIR);
-  }
-  console.log('Loading project languages...');
-  const languages = await getAllProjectLanguages();
-  const baseLanguage = languages.find(language => language.base);
-  if (!baseLanguage) {
-    console.error(JSON.stringify(languages));
-    throw new Error('Could not find base language');
-  }
-  console.log(`Loading ${baseLanguage.tag} languages translations as base...`);
-
-  const baseTranslations = await getBaseTranslations(baseLanguage);
-  const baseKeyNum = countKeys(baseTranslations);
-  const languagesWithTranslations = await Promise.all(
-    languages.map(async language => {
-      console.log(`Loading ${language.tag} translations...`);
-      const translations = await getRemoteTranslations(language.tag);
-      const keyNum = countKeys(translations);
-      const completeRate = Number((keyNum / baseKeyNum).toFixed(3));
-      console.log(
-        `Load ${language.name} ${
-          completeRate * 100
-        }, %(${keyNum}/${baseKeyNum}) complete`
-      );
-
-      return {
-        ...language,
-        translations,
-        completeRate,
-      };
-    })
-  );
-
-  const availableLanguages = languagesWithTranslations.filter(
-    language => language.completeRate > 0.2
-  );
-
-  for (const language of availableLanguages
-    // skip base language
-    .filter(i => !i.base)) {
-    await fs.writeFile(
-      path.resolve(RES_DIR, `${language.tag}.json`),
-      JSON.stringify(
-        {
-          '// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.':
-            '',
-          ...flattenTranslation(language.translations),
-        },
-        null,
-        INDENT
-      ) + '\n'
-    );
-  }
-
-  console.log('Generating meta data...');
-  const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
-        // Run \`yarn run download-resources\` to regenerate.
-        // If you need to update the code, please edit \`i18n/src/scripts/download.ts\` inside your project.
-        ${availableLanguages
-          .map(
-            language =>
-              `import ${language.tag.replaceAll('-', '_')} from './${
-                language.tag
-              }.json'`
-          )
-          .sort()
-          .join('\n')}
-
-        export const LOCALES = [
-            ${availableLanguages
-              // eslint-disable-next-line @typescript-eslint/no-unused-vars -- omit key
-              .map(({ translations, ...language }) =>
-                JSON.stringify({
-                  ...language,
-                  // a trick to generate a string without quotation marks
-                  res: '__RES_PLACEHOLDER',
-                }).replace(
-                  '"__RES_PLACEHOLDER"',
-                  language.tag.replaceAll('-', '_')
-                )
-              )
-              .join(',\n')}
-        ] as const;
-        `;
-
-  await fs.writeFile(
-    path.resolve(RES_DIR, 'index.ts'),
-    await format(code, {
-      parser: 'typescript',
-      singleQuote: true,
-      trailingComma: 'es5',
-      tabWidth: INDENT,
-      arrowParens: 'avoid',
-    })
-  );
-  console.log('Done');
-};
-
-main().catch(e => {
-  console.error(e);
-  process.exit(1);
-});

packages/frontend/i18n/src/scripts/request.ts

@@ -1,52 +0,0 @@
-import { Headers } from 'undici';
-import type { fetch, Request, RequestInfo } from 'undici-types';
-
-// cSpell:ignore Tolgee
-const TOLGEE_API_KEY = process.env['TOLGEE_API_KEY'];
-const TOLGEE_API_URL = 'https://i18n.affine.pro';
-
-if (!TOLGEE_API_KEY) {
-  throw new Error(`Please set "TOLGEE_API_KEY" as environment variable!`);
-}
-
-const withTolgee = (f: typeof fetch): typeof fetch => {
-  const baseUrl = `${TOLGEE_API_URL}/v2/projects`;
-  const headers = new Headers({
-    'X-API-Key': TOLGEE_API_KEY,
-    'Content-Type': 'application/json',
-  });
-
-  const isRequest = (input: RequestInfo): input is Request => {
-    return typeof input === 'object' && !('href' in input);
-  };
-
-  return new Proxy(f, {
-    apply(target, thisArg: unknown, argArray: Parameters<typeof fetch>) {
-      if (isRequest(argArray[0])) {
-        // Request
-        if (!argArray[0].headers) {
-          argArray[0] = {
-            ...argArray[0],
-            url: `${baseUrl}${argArray[0].url}`,
-            headers,
-          };
-        }
-      } else {
-        // URL or URLLike + ?RequestInit
-        if (typeof argArray[0] === 'string') {
-          argArray[0] = `${baseUrl}${argArray[0]}`;
-        }
-        if (!argArray[1]) {
-          argArray[1] = {};
-        }
-        if (!argArray[1].headers) {
-          argArray[1].headers = headers;
-        }
-      }
-      // console.log('fetch', argArray);
-      return target.apply(thisArg, argArray);
-    },
-  });
-};
-
-export const fetchTolgee = withTolgee(globalThis.fetch as typeof fetch);

packages/frontend/i18n/src/scripts/sync.ts

@@ -1,153 +0,0 @@
-// cSpell:ignore Tolgee
-import { readFile } from 'node:fs/promises';
-import { resolve } from 'node:path';
-
-import { createsNewKey, getRemoteTranslations } from './api.js';
-import type { TranslationRes } from './utils.js';
-
-const BASE_JSON_PATH = resolve(process.cwd(), 'src', 'resources', 'en.json');
-const BASE_LANGUAGES = 'en' as const;
-
-/**
- *
- * @example
- * ```ts
- * flatRes({ a: { b: 'c' } }); // { 'a.b': 'c' }
- * ```
- */
-const flatRes = (obj: TranslationRes) => {
-  const getEntries = (o: TranslationRes, prefix = ''): [string, string][] =>
-    Object.entries(o).flatMap<[string, string]>(([k, v]) =>
-      typeof v !== 'string'
-        ? getEntries(v, `${prefix}${k}.`)
-        : [[`${prefix}${k}`, v]]
-    );
-  return Object.fromEntries(getEntries(obj));
-};
-
-const differenceObject = (
-  newObj: Record<string, string>,
-  oldObj: Record<string, string>
-) => {
-  const add: string[] = [];
-  const remove: string[] = [];
-  const modify: string[] = [];
-  const both: string[] = [];
-
-  Object.keys(newObj).forEach(key => {
-    if (!(key in oldObj)) {
-      add.push(key);
-    } else {
-      both.push(key);
-    }
-  });
-
-  Object.keys(oldObj).forEach(key => {
-    if (!(key in newObj)) {
-      remove.push(key);
-    }
-  });
-
-  both.forEach(key => {
-    if (!(key in newObj) || !(key in oldObj)) {
-      throw new Error('Unreachable');
-    }
-    const newVal = newObj[key];
-    const oldVal = oldObj[key];
-    if (newVal !== oldVal) {
-      modify.push(key);
-    }
-  });
-  return { add, remove, modify };
-};
-
-function warnDiff(diff: { add: string[]; remove: string[]; modify: string[] }) {
-  if (diff.add.length) {
-    console.log('New keys found:', diff.add.join(', '));
-    //See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-notice-message
-    process.env['CI'] &&
-      console.log(
-        `::notice file=${BASE_JSON_PATH},line=1,title=New keys::${diff.add.join(
-          ', '
-        )}`
-      );
-  }
-  if (diff.remove.length) {
-    console.warn('[WARN]', 'Unused keys found:', diff.remove.join(', '));
-    process.env['CI'] &&
-      console.warn(
-        `::notice file=${BASE_JSON_PATH},line=1,title=Unused keys::${diff.remove.join(
-          ', '
-        )}`
-      );
-  }
-  if (diff.modify.length) {
-    console.warn('[WARN]', 'Inconsistent keys found:', diff.modify.join(', '));
-    process.env['CI'] &&
-      console.warn(
-        `::warning file=${BASE_JSON_PATH},line=1,title=Inconsistent keys::${diff.modify.join(
-          ', '
-        )}`
-      );
-  }
-}
-
-const main = async () => {
-  console.log('Loading local base translations...');
-  const baseLocalTranslations = JSON.parse(
-    await readFile(BASE_JSON_PATH, {
-      encoding: 'utf8',
-    })
-  );
-  const flatLocalTranslations = flatRes(baseLocalTranslations);
-  console.log(
-    `Loading local base translations success! Total ${
-      Object.keys(flatLocalTranslations).length
-    } keys`
-  );
-
-  console.log('Fetch remote base translations...');
-  const baseRemoteTranslations = await getRemoteTranslations(BASE_LANGUAGES);
-  const flatRemoteTranslations = flatRes(baseRemoteTranslations);
-  console.log(
-    `Fetch remote base translations success! Total ${
-      Object.keys(flatRemoteTranslations).length
-    } keys`
-  );
-
-  const diff = differenceObject(flatLocalTranslations, flatRemoteTranslations);
-
-  console.log(''); // new line
-  warnDiff(diff);
-  console.log(''); // new line
-
-  if (process.argv.slice(2).includes('--check')) {
-    // check mode
-    return;
-  }
-
-  for (const key of diff.add) {
-    const val = flatLocalTranslations[key];
-    console.log(`Creating new key: ${key} -> ${val}`);
-    await createsNewKey(key, { [BASE_LANGUAGES]: val });
-  }
-
-  // TODO remove unused tags from used keys
-
-  // diff.remove.forEach(key => {
-  //   // TODO set unused tag
-  //   // console.log(`Add ${DEPRECATED_TAG_NAME} to ${key}`);
-  //   addTagByKey(key, DEPRECATED_TAG_NAME);
-  // });
-
-  // diff.modify.forEach(key => {
-  //   // TODO warn different between local and remote base translations
-  // });
-
-  // TODO send notification
-};
-
-main().catch(e => {
-  console.error(e);
-  process.exit(1);
-});

packages/frontend/i18n/src/scripts/utils.ts

@@ -1,35 +0,0 @@
-export interface TranslationRes {
-  [x: string]: string | TranslationRes;
-}
-
-/**
- * Recursively flattens a JSON object using dot notation.
- *
- * NOTE: input must be an object as described by JSON spec. Arbitrary
- * JS objects (e.g. {a: () => 42}) may result in unexpected output.
- * MOREOVER, it removes keys with empty objects/arrays as value (see
- * examples bellow).
- *
- * @example
- * flattenTranslation({a: 1, b: [{c: 2, d: {e: 3}}, 4]})
- * // {a: 1, b.0.c: 2, b.0.d.e: 3, b.1: 4}
- * flattenTranslation({a: 1, b: [{c: 2, d: {e: [true, false, {f: 1}]}}]})
- * // {a: 1, b.0.c: 2, b.0.d.e.0: true, b.0.d.e.1: false, b.0.d.e.2.f: 1}
- * flattenTranslation({a: 1, b: [], c: {}})
- * // {a: 1}
- *
- * @param obj item to be flattened
- */
-export const flattenTranslation = (
-  obj: string | TranslationRes,
-  path?: string
-): TranslationRes => {
-  if (!(obj instanceof Object)) return { [path ?? '']: obj };
-
-  return Object.keys(obj).reduce((output, key) => {
-    return Object.assign(
-      output,
-      flattenTranslation(obj[key], path ? path + '.' + key : key)
-    );
-  }, {});
-};

packages/frontend/i18n/tsconfig.json

@@ -10,9 +10,6 @@
   "references": [
     {
       "path": "./tsconfig.resources.json"
-    },
-    {
-      "path": "./tsconfig.node.json"
     }
   ]
 }

packages/frontend/i18n/tsconfig.node.json

@@ -1,8 +0,0 @@
-{
-  "extends": "../../../tsconfig.json",
-  "compilerOptions": {
-    "types": ["node"],
-    "outDir": "./lib/scripts"
-  },
-  "include": ["./src/scripts"]
-}