diff --git a/.github/deployment/Dockerfile-affine b/.github/deployment/Dockerfile-affine index 9d54eb0cf4..e0bb7e9edf 100644 --- a/.github/deployment/Dockerfile-affine +++ b/.github/deployment/Dockerfile-affine @@ -1,9 +1,9 @@ FROM node:16-alpine as builder WORKDIR /app +RUN apk add git && npm i -g pnpm@7 COPY . . # RUN apk add g++ make python3 git libpng-dev -RUN apk add git -RUN npm i -g pnpm@7 && pnpm i --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus" && pnpm run build:local --skip-nx-cache +RUN --mount=type=cache,target=/app/node_modules,rw,sharing=private pnpm i --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus" && pnpm run build:local --skip-nx-cache FROM node:16-alpine as relocate WORKDIR /app diff --git a/.github/deployment/Dockerfile-affine-keck b/.github/deployment/Dockerfile-affine-keck index ae76e1b01e..75d02dcac8 100644 --- a/.github/deployment/Dockerfile-affine-keck +++ b/.github/deployment/Dockerfile-affine-keck @@ -1,9 +1,9 @@ FROM node:16-alpine as builder WORKDIR /app +RUN apk add git && npm i -g pnpm@7 COPY . . # RUN apk add g++ make python3 git libpng-dev -RUN apk add git -RUN npm i -g pnpm@7 && pnpm i --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus" && pnpm run build:local-keck --skip-nx-cache +RUN --mount=type=cache,target=/app/node_modules,rw,sharing=private pnpm i --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus" && pnpm run build:local-keck --skip-nx-cache FROM node:16-alpine as relocate WORKDIR /app diff --git a/.github/deployment/Dockerfile-keck b/.github/deployment/Dockerfile-keck index 7918e49427..7b611c27b2 100644 --- a/.github/deployment/Dockerfile-keck +++ b/.github/deployment/Dockerfile-keck @@ -1,8 +1,8 @@ FROM node:16-alpine as builder WORKDIR /app +RUN apk add g++ make python3 git libpng-dev && npm i -g pnpm@7 COPY . . -RUN apk add g++ make python3 git libpng-dev -RUN npm i -g pnpm@7 && pnpm i --frozen-lockfile --store=node_modules/.pnpm-store && pnpm run build:keck +RUN --mount=type=cache,target=/app/node_modules,rw,sharing=private pnpm i --frozen-lockfile --store=node_modules/.pnpm-store && pnpm run build:keck FROM node:16-alpine as node_modules WORKDIR /app diff --git a/.github/deployment/Dockerfile-lisa b/.github/deployment/Dockerfile-lisa index f469a7fce8..4f80f6ebab 100644 --- a/.github/deployment/Dockerfile-lisa +++ b/.github/deployment/Dockerfile-lisa @@ -1,9 +1,9 @@ FROM node:16-alpine as builder WORKDIR /app +RUN apk add git && npm i -g pnpm@7 COPY . . # RUN apk add g++ make python3 git libpng-dev -RUN apk add git -RUN npm i -g pnpm@7 && pnpm i --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus" && pnpm run build +RUN --mount=type=cache,target=/app/node_modules,rw,sharing=private pnpm i --frozen-lockfile --store=node_modules/.pnpm-store --filter "!ligo-virgo-e2e" --filter "!keck" --filter "!venus" && pnpm run build FROM node:16-alpine as relocate WORKDIR /app diff --git a/.github/deployment/Dockerfile-venus b/.github/deployment/Dockerfile-venus index 585ade8a20..92a2bb8305 100644 --- a/.github/deployment/Dockerfile-venus +++ b/.github/deployment/Dockerfile-venus @@ -1,8 +1,8 @@ FROM node:16-alpine as builder WORKDIR /app +RUN apk add g++ make python3 git libpng-dev && npm i -g pnpm@7 COPY . . -RUN apk add g++ make python3 git libpng-dev -RUN npm i -g pnpm@7 && pnpm i --frozen-lockfile --store=node_modules/.pnpm-store && pnpm run build:venus +RUN --mount=type=cache,target=/app/node_modules,rw,sharing=private pnpm i --frozen-lockfile --store=node_modules/.pnpm-store && pnpm run build:venus FROM node:16-alpine as relocate WORKDIR /app diff --git a/.github/workflows/languages-download.yml b/.github/workflows/languages-download.yml new file mode 100644 index 0000000000..4e180bccce --- /dev/null +++ b/.github/workflows/languages-download.yml @@ -0,0 +1,84 @@ +name: Download Languages Resources + +on: + schedule: + - cron: "0 0 * * 5" # At 00:00(UTC) on Friday. + workflow_dispatch: + +# Cancels all previous workflow runs for pull requests that have not completed. +# See https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # The concurrency group contains the workflow name and the branch name for + # pull requests or the commit hash for any other events. + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +jobs: + main: + strategy: + matrix: + node-version: [18] + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Use pnpm + uses: pnpm/action-setup@v2 + with: + version: 7 + + - name: Use Node.js ${{ matrix.node-version }} + # https://github.com/actions/setup-node + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: 'pnpm' + + - name: Install node modules + run: pnpm install + + - name: Sync Languages + if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master' + working-directory: ./libs/datasource/i18n + run: pnpm run download-resources + env: + TOLGEE_API_KEY: ${{ secrets.TOLGEE_API_KEY }} + + - name: Push Branch + id: push + run: | + git add libs/datasource/i18n + # Do not proceed if there are no file differences + COMMIT=$(git rev-parse --verify origin/$TARGET_BRANCH || echo HEAD) + FILES_CHANGED=$(git diff-index --name-only --cached $COMMIT | wc -l) + if [[ "$FILES_CHANGED" = "0" ]]; then + echo "No file changes detected." + echo "::set-output name=skipPR::true" + exit 0 + fi + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + git commit --message 'feat(i18n): new translations' + git remote set-url origin "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY" + git push --force origin HEAD:$TARGET_BRANCH + env: + GITHUB_TOKEN: ${{ secrets.github_token }} + TARGET_BRANCH: bot/new-translations + + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + + # see https://github.com/repo-sync/pull-request + - name: Create Pull Request + if: steps.push.outputs.skipPR != 'true' + uses: repo-sync/pull-request@v2 + with: + source_branch: 'bot/new-translations' # If blank, default: triggered branch + destination_branch: "develop" + pr_title: Update i18n (${{ steps.date.outputs.date }}) # Title of pull request + pr_label: 'data,bot' # Comma-separated list (no spaces) + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index d4f92bc519..256da653f0 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,11 @@ We would also like to give thanks to open-source projects that make affine possi - [MUI](https://github.com/mui/material-ui) -- Our most used graphic UI component library. - Other [dependencies](https://github.com/toeverything/AFFiNE/network/dependencies) +We use the following open source projects to help us build a better development experience: + +- [nx](https://github.com/nrwl/nx) -- Awesome monorepo manager & build system +- [tolgee](https://github.com/tolgee/tolgee-platform) -- Elegant i18n collaborative editing platform + Thanks a lot to the community for providing such powerful and simple libraries, so that we can focus more on the implementation of the product logic, and we hope that in the future our projects will also provide a more easy-to-use knowledge base for everyone. # Contributors diff --git a/libs/components/editor-plugins/src/menu/command-menu/Menu.tsx b/libs/components/editor-plugins/src/menu/command-menu/Menu.tsx index 1bdda4e739..66b2679220 100644 --- a/libs/components/editor-plugins/src/menu/command-menu/Menu.tsx +++ b/libs/components/editor-plugins/src/menu/command-menu/Menu.tsx @@ -227,7 +227,7 @@ export const CommandMenu = ({ editor, hooks, style }: CommandMenuProps) => { const nextBlock = await block.nextSibling(); setTimeout(() => { editor.selectionManager.activeNodeByNodeId(nextBlock.id); - }); + }, 100); if (block.blockProvider.isEmpty()) { block.remove(); } diff --git a/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx b/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx index a308614d13..8ebaf90a82 100644 --- a/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx +++ b/libs/components/layout/src/settings-sidebar/Settings/SettingsList.tsx @@ -53,10 +53,10 @@ export const SettingsList = () => { > {LOCALES.map(option => ( ))} diff --git a/libs/datasource/i18n/package.json b/libs/datasource/i18n/package.json index eb1657865e..25e7a82139 100644 --- a/libs/datasource/i18n/package.json +++ b/libs/datasource/i18n/package.json @@ -3,7 +3,8 @@ "version": "0.0.1", "scripts": { "sync-languages": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/sync.ts", - "sync-languages:check": "pnpm run sync-languages --check" + "sync-languages:check": "pnpm run sync-languages --check", + "download-resources": "NODE_OPTIONS=--experimental-fetch ts-node src/scripts/download.ts" }, "dependencies": { "i18next": "^21.9.1", diff --git a/libs/datasource/i18n/src/base.json b/libs/datasource/i18n/src/base.json deleted file mode 100644 index e243d71db1..0000000000 --- a/libs/datasource/i18n/src/base.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "Sync to Disk": "Sync to Disk", - "Share": "Share", - "WarningTips": { - "IsNotfsApiSupported": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC DATA TO DISK with the latest version of Chromium based browser like Chrome/Edge", - "IsNotLocalWorkspace": "Welcome to the AFFiNE demo. To begin saving changes you can SYNC TO DISK.", - "DoNotStore": "AFFiNE is under active development and the current version is UNSTABLE. Please DO NOT store information or data" - }, - "Layout": "Layout", - "Comment": "Comment", - "Settings": "Settings", - "ComingSoon": "Layout Settings Coming Soon...", - "Duplicate Page": "Duplicate Page", - "Copy Page Link": "Copy Page Link", - "Language": "Language", - "Clear Workspace": "Clear Workspace", - "Export As Markdown": "Export As Markdown", - "Export As HTML": "Export As HTML", - "Export As PDF (Unsupported)": "Export As PDF (Unsupported)", - "Import Workspace": "Import Workspace", - "Export Workspace": "Export Workspace", - "Last edited by": "Last edited by {{name}}", - "Logout": "Logout" -} diff --git a/libs/datasource/i18n/src/index.ts b/libs/datasource/i18n/src/index.ts index a421494bf8..f4428a9b09 100644 --- a/libs/datasource/i18n/src/index.ts +++ b/libs/datasource/i18n/src/index.ts @@ -4,8 +4,8 @@ import { initReactI18next, useTranslation, } from 'react-i18next'; -import en_US from './resources/en.json'; -import zh_CN from './resources/zh.json'; +import { LOCALES } from './resources'; +import type en_US from './resources/en.json'; // See https://react.i18next.com/latest/typescript declare module 'react-i18next' { @@ -21,22 +21,19 @@ declare module 'react-i18next' { const STORAGE_KEY = 'i18n_lng'; -const LOCALES = [ - { value: 'en', text: 'English', res: en_US }, - { value: 'zh', text: '简体中文', res: zh_CN }, -] as const; +export { i18n, useTranslation, I18nProvider, LOCALES }; const resources = LOCALES.reduce( - (acc, { value, res }) => ({ ...acc, [value]: { translation: res } }), + (acc, { tag, res }) => ({ ...acc, [tag]: { translation: res } }), {} ); -const fallbackLng = LOCALES[0].value; +const fallbackLng = LOCALES[0].tag; const standardizeLocale = (language: string) => { - if (LOCALES.find(locale => locale.value === language)) return language; + if (LOCALES.find(locale => locale.tag === language)) return language; if ( LOCALES.find( - locale => locale.value === language.slice(0, 2).toLowerCase() + locale => locale.tag === language.slice(0, 2).toLowerCase() ) ) return language; @@ -64,5 +61,3 @@ i18n.on('languageChanged', lng => { }); const I18nProvider = I18nextProvider; - -export { i18n, useTranslation, I18nProvider, LOCALES }; diff --git a/libs/datasource/i18n/src/resources/index.ts b/libs/datasource/i18n/src/resources/index.ts new file mode 100644 index 0000000000..1d3e875c60 --- /dev/null +++ b/libs/datasource/i18n/src/resources/index.ts @@ -0,0 +1,28 @@ +// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +// Run `pnpm run download-resources` to regenerate. +// To overwrite this, please overwrite download.ts +import en from './en.json'; +import zh_Hans from './zh-Hans.json'; + +export const LOCALES = [ + { + id: 1000016008, + name: 'English', + tag: 'en', + originalName: 'English', + flagEmoji: '🇬🇧', + base: true, + completeRate: 1, + res: en, + }, + { + id: 1000016009, + name: 'Simplified Chinese', + tag: 'zh-Hans', + originalName: '简体中文', + flagEmoji: '🇨🇳', + base: false, + completeRate: 1, + res: zh_Hans, + }, +] as const; diff --git a/libs/datasource/i18n/src/resources/zh.json b/libs/datasource/i18n/src/resources/zh-Hans.json similarity index 93% rename from libs/datasource/i18n/src/resources/zh.json rename to libs/datasource/i18n/src/resources/zh-Hans.json index fe72ae542d..5122e4b881 100644 --- a/libs/datasource/i18n/src/resources/zh.json +++ b/libs/datasource/i18n/src/resources/zh-Hans.json @@ -1,4 +1,5 @@ { + "// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.": "", "Sync to Disk": "同步到磁盘", "Share": "分享", "WarningTips": { diff --git a/libs/datasource/i18n/src/scripts/api.ts b/libs/datasource/i18n/src/scripts/api.ts index 52011e6042..b38ad5875b 100644 --- a/libs/datasource/i18n/src/scripts/api.ts +++ b/libs/datasource/i18n/src/scripts/api.ts @@ -5,15 +5,65 @@ import { fetchTolgee } from './request'; * Returns all project languages * * See https://tolgee.io/api#operation/getAll_6 + * @example + * ```ts + * const languages = [ + * { + * id: 1000016008, + * name: 'English', + * tag: 'en', + * originalName: 'English', + * flagEmoji: '🇬🇧', + * base: true + * }, + * { + * id: 1000016013, + * name: 'Spanish', + * tag: 'es', + * originalName: 'español', + * flagEmoji: '🇪🇸', + * base: false + * }, + * { + * id: 1000016009, + * name: 'Simplified Chinese', + * tag: 'zh-Hans', + * originalName: '简体中文', + * flagEmoji: '🇨🇳', + * base: false + * }, + * { + * id: 1000016012, + * name: 'Traditional Chinese', + * tag: 'zh-Hant', + * originalName: '繁體中文', + * flagEmoji: '🇭🇰', + * base: false + * } + * ] + * ``` */ -export const getAllProjectLanguages = async () => { - const url = '/languages?size=1000'; +export const getAllProjectLanguages = async (size = 1000) => { + const url = `/languages?size=${size}`; const resp = await fetchTolgee(url); if (resp.status < 200 || resp.status >= 300) { throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); } - const json = await resp.json(); - return json; + const json: { + // eslint-disable-next-line @typescript-eslint/naming-convention + _embedded: { + languages: { + id: number; + name: string; + tag: string; + originalName: string; + flagEmoji: string; + base: boolean; + }[]; + }; + page: unknown; + } = await resp.json(); + return json._embedded.languages; }; /** @@ -48,6 +98,16 @@ export const getLanguagesTranslations = async ( return json; }; +export const getRemoteTranslations = async (languages: string) => { + const translations = await getLanguagesTranslations(languages); + if (!(languages in translations)) { + return {}; + } + // The assert is safe because we checked above + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return translations[languages]!; +}; + /** * Creates new key * @@ -109,3 +169,18 @@ export const addTagByKey = async (key: string, tag: string) => { // const keyId = // addTag(keyId, tag); }; + +/** + * Exports data + * + * See https://tolgee.io/api#operation/export_1 + */ +export const exportResources = async () => { + const url = `/export`; + const resp = await fetchTolgee(url); + + if (resp.status < 200 || resp.status >= 300) { + throw new Error(url + ' ' + resp.status + '\n' + (await resp.text())); + } + return resp; +}; diff --git a/libs/datasource/i18n/src/scripts/download.ts b/libs/datasource/i18n/src/scripts/download.ts new file mode 100644 index 0000000000..f45261c258 --- /dev/null +++ b/libs/datasource/i18n/src/scripts/download.ts @@ -0,0 +1,134 @@ +/* eslint-disable no-console */ +// cSpell:ignore Tolgee +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { format } from 'prettier'; +import { getAllProjectLanguages, getRemoteTranslations } from './api'; +import type { TranslationRes } from './utils'; + +const RES_DIR = path.resolve(process.cwd(), 'src', 'resources'); + +const countKeys = (obj: TranslationRes) => { + let count = 0; + Object.entries(obj).forEach(([key, value]) => { + if (typeof value === 'string') { + count++; + } else { + count += countKeys(value); + } + }); + return count; +}; + +const getBaseTranslations = async (baseLanguage: { tag: string }) => { + try { + const baseTranslationsStr = await fs.readFile( + path.resolve(RES_DIR, `${baseLanguage.tag}.json`), + { encoding: 'utf8' } + ); + const baseTranslations = JSON.parse(baseTranslationsStr); + return baseTranslations; + } catch (e) { + console.error('base language:', JSON.stringify(baseLanguage)); + console.error('Failed to read base language', e); + const translations = await getRemoteTranslations(baseLanguage.tag); + await fs.writeFile( + path.resolve(RES_DIR, `${baseLanguage.tag}.json`), + JSON.stringify(translations, null, 4) + ); + } +}; + +const main = async () => { + console.log('Loading project languages...'); + const languages = await getAllProjectLanguages(); + const baseLanguage = languages.find(language => language.base); + if (!baseLanguage) { + console.error(JSON.stringify(languages)); + throw new Error('Could not find base language'); + } + console.log( + `Loading ${baseLanguage.tag} languages translations as base...` + ); + + const baseTranslations = await getBaseTranslations(baseLanguage); + const baseKeyNum = countKeys(baseTranslations); + const languagesWithTranslations = await Promise.all( + languages.map(async language => { + console.log(`Loading ${language.tag} translations...`); + const translations = await getRemoteTranslations(language.tag); + const keyNum = countKeys(translations); + + return { + ...language, + translations, + completeRate: keyNum / baseKeyNum, + }; + }) + ); + + const availableLanguages = languagesWithTranslations.filter( + language => language.completeRate > 0 + ); + + availableLanguages + // skip base language + .filter(i => !i.base) + .forEach(async language => { + await fs.writeFile( + path.resolve(RES_DIR, `${language.tag}.json`), + JSON.stringify( + { + // eslint-disable-next-line @typescript-eslint/naming-convention + '// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.': + '', + ...language.translations, + }, + null, + 4 + ) + '\n' + ); + }); + + console.log('Generating meta data...'); + const code = `// THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. + // Run \`pnpm run download-resources\` to regenerate. + // To overwrite this, please overwrite ${path.basename(__filename)} + ${availableLanguages + .map( + language => + `import ${language.tag.replaceAll('-', '_')} from './${ + language.tag + }.json'` + ) + .join('\n')} + + export const LOCALES = [ + ${availableLanguages + .map(({ translations, ...language }) => + JSON.stringify({ + ...language, + res: '__RES_PLACEHOLDER', + }).replace( + '"__RES_PLACEHOLDER"', + language.tag.replaceAll('-', '_') + ) + ) + .join(',\n')} + ] as const; + `; + + await fs.writeFile( + path.resolve(RES_DIR, 'index.ts'), + format(code, { + parser: 'typescript', + singleQuote: true, + trailingComma: 'es5', + tabWidth: 4, + arrowParens: 'avoid', + }) + ); + console.log('Done'); +}; + +main(); diff --git a/libs/datasource/i18n/src/scripts/request.ts b/libs/datasource/i18n/src/scripts/request.ts index c016909a24..b599c71957 100644 --- a/libs/datasource/i18n/src/scripts/request.ts +++ b/libs/datasource/i18n/src/scripts/request.ts @@ -46,6 +46,7 @@ const withTolgee = ( argArray[1].headers = headers; } } + // console.log('fetch', argArray); return target.apply(thisArg, argArray); }, }); diff --git a/libs/datasource/i18n/src/scripts/sync.ts b/libs/datasource/i18n/src/scripts/sync.ts index a71a0de61f..c2ba692dc5 100644 --- a/libs/datasource/i18n/src/scripts/sync.ts +++ b/libs/datasource/i18n/src/scripts/sync.ts @@ -1,30 +1,20 @@ +/* eslint-disable no-console */ // cSpell:ignore Tolgee import { readFile } from 'fs/promises'; import path from 'path'; -import { addTagByKey, createsNewKey, getLanguagesTranslations } from './api'; +import { addTagByKey, createsNewKey, getRemoteTranslations } from './api'; +import type { TranslationRes } from './utils'; -const BASE_JSON_PATH = path.resolve(process.cwd(), 'src', 'base.json'); +const BASE_JSON_PATH = path.resolve( + process.cwd(), + 'src', + 'resources', + 'en.json' +); const BASE_LANGUAGES = 'en' as const; const DEPRECATED_TAG_NAME = 'unused' as const; -interface TranslationRes { - [x: string]: string | TranslationRes; -} - -const getRemoteTranslations = async (languages: string) => { - const translations = await getLanguagesTranslations(languages); - if (!(languages in translations)) { - console.log(translations); - throw new Error( - 'Failed to get base languages translation! base languages: ' + - languages - ); - } - // The assert is safe because we checked above - return translations[languages]!; -}; - /** * * @example diff --git a/libs/datasource/i18n/src/scripts/utils.ts b/libs/datasource/i18n/src/scripts/utils.ts new file mode 100644 index 0000000000..129a8febc3 --- /dev/null +++ b/libs/datasource/i18n/src/scripts/utils.ts @@ -0,0 +1,3 @@ +export interface TranslationRes { + [x: string]: string | TranslationRes; +}