From 591f874dad30887a80143a061a44bd3ca7ee3299 Mon Sep 17 00:00:00 2001 From: qiaoyanfei Date: Fri, 21 Aug 2026 13:55:14 +0800 Subject: [PATCH] fix(android): stabilize IME composition and deletion handling (#15370) ## Summary Fix Android editor IME corruption around composition, autocorrect replay, delete, Enter, and old WebView delete behavior. ## What changed - Add Android WebView InputConnection wrapper for editor IME handling. - Route Android delete events through BlockSuite editor input. - Guard against keyboard autocorrect/composition replay after delete or space. - Stabilize delete fallback on older Android/WebView versions. - Gate IME diagnostic logs behind debug builds. - Add Android IME fix notes and regression coverage. ## Validation - Manual Android testing passed. - `git diff --check upstream/canary...HEAD` ## Summary by CodeRabbit * **Bug Fixes** * Improved Android text editing for Backspace, Delete, Enter, composing text, and keyboard events. * Prevented input methods from targeting the wrong editor area. * Improved caret-based text handling, focus synchronization, and composing-session cleanup. * **Platform Improvements** * Added a dedicated Android input bridge for smoother IME interactions and fallback keyboard behavior. * Improved editor actions, input recovery, and trusted-page validation for Android communication. --------- Co-authored-by: DarkSky --- .../blocks/root/src/page/page-root-block.ts | 6 +- blocksuite/affine/rich-text/src/utils.ts | 18 +- .../std/src/__tests__/keymap.unit.spec.ts | 88 ++++- blocksuite/framework/std/src/event/keymap.ts | 38 +- .../std/src/inline/services/event.ts | 163 +++++++++ .../app/affine/pro/AffineEditorWebView.kt | 80 +++++ .../java/app/affine/pro/AffineImeBridge.kt | 64 ++++ .../app/affine/pro/AffineImeWebDispatcher.kt | 70 ++++ .../app/affine/pro/AffineInputConnection.kt | 332 ++++++++++++++++++ .../app/affine/pro/AffineWebViewClient.kt | 25 ++ .../java/app/affine/pro/AndroidImeState.kt | 27 ++ .../app/affine/pro/ImeReplayController.kt | 217 ++++++++++++ .../main/java/app/affine/pro/MainActivity.kt | 18 + .../layout/capacitor_bridge_layout_main.xml | 13 + 14 files changed, 1142 insertions(+), 17 deletions(-) create mode 100644 packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineEditorWebView.kt create mode 100644 packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineImeBridge.kt create mode 100644 packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineImeWebDispatcher.kt create mode 100644 packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineInputConnection.kt create mode 100644 packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineWebViewClient.kt create mode 100644 packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AndroidImeState.kt create mode 100644 packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/ImeReplayController.kt create mode 100644 packages/frontend/apps/android/App/app/src/main/res/layout/capacitor_bridge_layout_main.xml diff --git a/blocksuite/affine/blocks/root/src/page/page-root-block.ts b/blocksuite/affine/blocks/root/src/page/page-root-block.ts index 11b468dad4..4ddc75b6e9 100644 --- a/blocksuite/affine/blocks/root/src/page/page-root-block.ts +++ b/blocksuite/affine/blocks/root/src/page/page-root-block.ts @@ -19,6 +19,7 @@ import { getScrollContainer, matchModels, } from '@blocksuite/affine-shared/utils'; +import { IS_ANDROID } from '@blocksuite/global/env'; import { Point } from '@blocksuite/global/gfx'; import type { PointerEventState } from '@blocksuite/std'; import { BlockComponent, BlockSelection, TextSelection } from '@blocksuite/std'; @@ -413,7 +414,10 @@ export class PageRootBlockComponent extends BlockComponent { return !(isNote && displayOnEdgeless); }); - this.contentEditable = String(!this.store.readonly$.value); + // Android IMEs can target this outer editable root instead of a block's + // inline editor, leaving composition text in the DOM without committing it + // to the document model. Keep only the block editors editable on Android. + this.contentEditable = String(!this.store.readonly$.value && !IS_ANDROID); return html`
${children} ${widgets}
diff --git a/blocksuite/affine/rich-text/src/utils.ts b/blocksuite/affine/rich-text/src/utils.ts index 8317ba49a5..6dbbfb916d 100644 --- a/blocksuite/affine/rich-text/src/utils.ts +++ b/blocksuite/affine/rich-text/src/utils.ts @@ -57,12 +57,14 @@ export function getPrefixText(inlineEditor: InlineEditor) { const inlineRange = inlineEditor.getInlineRange(); if (!inlineRange || inlineRange.length > 0) return ''; - const nearestLineBreakIndex = inlineEditor.yTextString - .slice(0, inlineRange.index) - .lastIndexOf('\n'); - const prefixText = inlineEditor.yTextString.slice( - nearestLineBreakIndex + 1, - inlineRange.index - ); - return prefixText; + const maxMarkdownPrefixLength = 512; + const prefixStart = Math.max(0, inlineRange.index - maxMarkdownPrefixLength); + const yTextString = inlineEditor.yTextString; + const prefixWindow = yTextString.slice(prefixStart, inlineRange.index); + const nearestLineBreakIndex = prefixWindow.lastIndexOf('\n'); + if (nearestLineBreakIndex === -1 && prefixStart > 0) return ''; + + return nearestLineBreakIndex === -1 + ? prefixWindow + : prefixWindow.slice(nearestLineBreakIndex + 1); } diff --git a/blocksuite/framework/std/src/__tests__/keymap.unit.spec.ts b/blocksuite/framework/std/src/__tests__/keymap.unit.spec.ts index e9c0ba05b7..1f597ab0d9 100644 --- a/blocksuite/framework/std/src/__tests__/keymap.unit.spec.ts +++ b/blocksuite/framework/std/src/__tests__/keymap.unit.spec.ts @@ -1,6 +1,7 @@ -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; -import { bindKeymap } from '../event/keymap.js'; +import { UIEventState, UIEventStateContext } from '../event/base.js'; +import { androidBindKeymapPatch, bindKeymap } from '../event/keymap.js'; const createKeyboardEvent = (options: { key: string; @@ -117,3 +118,86 @@ describe('bindKeymap', () => { expect(handled).toBe(false); }); }); + +describe('androidBindKeymapPatch', () => { + const beforeInputCtx = (inputType: string) => { + const event = new InputEvent('beforeinput', { + inputType, + cancelable: true, + }); + return { ctx: UIEventStateContext.from(new UIEventState(event)), event }; + }; + + test('routes deleteContentBackward to the Backspace binding', () => { + const backspace = vi.fn(() => true); + const handler = androidBindKeymapPatch({ Backspace: backspace }); + const { ctx } = beforeInputCtx('deleteContentBackward'); + + expect(handler(ctx)).toBe(true); + expect(backspace).toHaveBeenCalledOnce(); + }); + + test('routes insertParagraph to the Enter binding', () => { + const enter = vi.fn((ctx: UIEventStateContext) => { + ctx.get('keyboardState').raw.preventDefault(); + return true; + }); + const handler = androidBindKeymapPatch({ Enter: enter }); + const { ctx, event } = beforeInputCtx('insertParagraph'); + const preventDefault = vi.spyOn(event, 'preventDefault'); + + expect(handler(ctx)).toBe(true); + expect(enter).toHaveBeenCalledOnce(); + expect(preventDefault).toHaveBeenCalledOnce(); + expect(ctx.get('keyboardState').raw.key).toBe('Enter'); + expect(ctx.get('keyboardState').composing).toBe(false); + }); + + test('propagates preventDefault when the binding returns false', () => { + const backspace = vi.fn((ctx: UIEventStateContext) => { + ctx.get('keyboardState').raw.preventDefault(); + return false; + }); + const handler = androidBindKeymapPatch({ Backspace: backspace }); + const { ctx, event } = beforeInputCtx('deleteContentBackward'); + + expect(handler(ctx)).toBe(false); + expect(event.defaultPrevented).toBe(true); + }); + + test('does nothing for insertParagraph without an Enter binding', () => { + const handler = androidBindKeymapPatch({ Backspace: vi.fn(() => true) }); + const { ctx } = beforeInputCtx('insertParagraph'); + + expect(handler(ctx)).toBe(false); + expect(ctx.has('keyboardState')).toBe(false); + }); + + test('ignores non-input events', () => { + const enter = vi.fn(() => true); + const backspace = vi.fn(() => true); + const ctx = UIEventStateContext.from( + new UIEventState(new KeyboardEvent('keydown', { key: 'Enter' })) + ); + + expect( + androidBindKeymapPatch({ Enter: enter, Backspace: backspace })(ctx) + ).toBeUndefined(); + expect(enter).not.toHaveBeenCalled(); + expect(backspace).not.toHaveBeenCalled(); + }); + + test('ignores unrelated input types', () => { + const enter = vi.fn(() => true); + const backspace = vi.fn(() => true); + const handler = androidBindKeymapPatch({ + Enter: enter, + Backspace: backspace, + }); + const { ctx } = beforeInputCtx('insertText'); + + expect(handler(ctx)).toBe(false); + expect(enter).not.toHaveBeenCalled(); + expect(backspace).not.toHaveBeenCalled(); + }); +}); diff --git a/blocksuite/framework/std/src/event/keymap.ts b/blocksuite/framework/std/src/event/keymap.ts index ebacbdeb20..1f2b07e028 100644 --- a/blocksuite/framework/std/src/event/keymap.ts +++ b/blocksuite/framework/std/src/event/keymap.ts @@ -3,6 +3,7 @@ import { BlockSuiteError, ErrorCode } from '@blocksuite/global/exceptions'; import { base, keyName } from 'w3c-keyname'; import type { UIEventHandler } from './base.js'; +import { KeyboardEventState } from './state/index.js'; function normalizeKeyName(name: string) { const parts = name.split(/-(?!$)/); @@ -127,13 +128,38 @@ export function androidBindKeymapPatch( const event = ctx.get('defaultState').event; if (!(event instanceof InputEvent)) return; - if ( - event.inputType === 'deleteContentBackward' && - 'Backspace' in bindings - ) { - return bindings['Backspace'](ctx); + const bindingName = + event.inputType === 'deleteContentBackward' + ? 'Backspace' + : event.inputType === 'deleteContentForward' + ? 'Delete' + : event.inputType === 'insertParagraph' + ? 'Enter' + : undefined; + if (!bindingName || !(bindingName in bindings)) return false; + + if (!ctx.has('keyboardState')) { + const keyboardEvent = new KeyboardEvent('keydown', { + key: bindingName, + code: bindingName, + cancelable: true, + }); + Object.defineProperty(keyboardEvent, 'isComposing', { + configurable: true, + value: event.isComposing, + }); + ctx.add( + new KeyboardEventState({ + event: keyboardEvent, + composing: event.isComposing, + }) + ); } - return false; + const handled = bindings[bindingName](ctx); + if (handled || ctx.get('keyboardState').raw.defaultPrevented) { + event.preventDefault(); + } + return handled; }; } diff --git a/blocksuite/framework/std/src/inline/services/event.ts b/blocksuite/framework/std/src/inline/services/event.ts index 5765b31358..4d8387ae0e 100644 --- a/blocksuite/framework/std/src/inline/services/event.ts +++ b/blocksuite/framework/std/src/inline/services/event.ts @@ -13,11 +13,58 @@ import { isMaybeInlineRangeEqual } from '../utils/inline-range.js'; import { transformInput } from '../utils/transform-input.js'; import type { BeforeinputHookCtx, CompositionEndHookCtx } from './hook.js'; +type AndroidIMEInputType = 'deleteContentBackward' | 'deleteContentForward'; + +type AndroidIMEInputDetail = { + inputType?: AndroidIMEInputType; + handled?: boolean; +}; + +type AndroidIMEBridge = { + getProtocolVersion?: () => number; + finishComposingSession?: () => void; + finishDeleteSession?: () => void; + setEditorFocused?: (focused: boolean) => void; +}; + +declare global { + interface HTMLElementEventMap { + 'affine-android-ime-input': CustomEvent; + } +} + export class EventService { private _compositionInlineRange: InlineRange | null = null; private _isComposing = false; + private readonly _androidIMEBridge = () => { + const bridge = ( + globalThis as typeof globalThis & { + AffineAndroidIME?: AndroidIMEBridge; + } + ).AffineAndroidIME; + return bridge?.getProtocolVersion?.() === 1 ? bridge : undefined; + }; + + private readonly _finishAndroidComposingSession = (isDelete: boolean) => { + if (!IS_ANDROID) return; + + window.setTimeout(() => { + const bridge = this._androidIMEBridge(); + if (isDelete) { + bridge?.finishDeleteSession?.(); + } else { + bridge?.finishComposingSession?.(); + } + }, 0); + }; + + private readonly _setAndroidEditorFocused = (focused: boolean) => { + if (!IS_ANDROID) return; + this._androidIMEBridge()?.setEditorFocused?.(focused); + }; + private readonly _getClosestInlineRoot = (node: Node): Element | null => { const el = node instanceof Element ? node : node.parentElement; return el?.closest(`[${INLINE_ROOT_ATTR}]`) ?? null; @@ -203,6 +250,97 @@ export class EventService { ); this.editor.slots.inputting.next(event.data ?? ''); + + if ( + IS_ANDROID && + (ctx.raw.inputType === 'deleteContentBackward' || + ctx.raw.inputType === 'deleteContentForward' || + ctx.raw.inputType === 'insertParagraph' || + ctx.raw.inputType === 'insertLineBreak' || + (ctx.raw.inputType === 'insertText' && + (ctx.data === ' ' || ctx.data === '\n'))) + ) { + this._finishAndroidComposingSession( + ctx.raw.inputType === 'deleteContentBackward' || + ctx.raw.inputType === 'deleteContentForward' + ); + } + }; + + private readonly _onAndroidIMEInput = async ( + event: CustomEvent + ) => { + if (!IS_ANDROID) return; + + const inputType = event.detail?.inputType; + if ( + inputType !== 'deleteContentBackward' && + inputType !== 'deleteContentForward' + ) { + return; + } + + const range = this.editor.rangeService.getNativeRange(); + if (!range || !this._isRangeCompletelyInRoot(range)) return; + + event.detail.handled = true; + event.preventDefault(); + event.stopPropagation(); + + if (this.editor.isReadonly) return; + + let inlineRange = this.editor.toInlineRange(range); + if (!inlineRange) { + this.editor.rerenderWholeEditor(); + await this.editor.waitForUpdate(); + const newRange = this.editor.rangeService.getNativeRange(); + inlineRange = newRange ? this.editor.toInlineRange(newRange) : null; + if (!inlineRange) return; + } + + if (inlineRange.length === 0) { + if (inputType === 'deleteContentBackward') { + if (inlineRange.index === 0) return; + inlineRange = { + index: inlineRange.index - 1, + length: 1, + }; + } else { + if (inlineRange.index >= this.editor.yTextLength) return; + inlineRange = { + index: inlineRange.index, + length: 1, + }; + } + } + + this._isComposing = false; + this._compositionInlineRange = null; + + const raw = new InputEvent('beforeinput', { + inputType, + bubbles: true, + cancelable: true, + composed: true, + }); + const ctx: BeforeinputHookCtx = { + inlineEditor: this.editor, + raw, + inlineRange, + data: null, + attributes: {} as TextAttributes, + }; + this.editor.hooks.beforeinput?.(ctx); + + transformInput( + ctx.raw.inputType, + ctx.data, + ctx.attributes, + ctx.inlineRange, + this.editor as never + ); + this.editor.slots.inputting.next(''); + this._finishAndroidComposingSession(true); }; private readonly _onClick = (event: MouseEvent) => { @@ -265,6 +403,7 @@ export class EventService { } this.editor.slots.inputting.next(event.data ?? ''); + this._finishAndroidComposingSession(false); }; private readonly _onCompositionStart = (event: CompositionEvent) => { @@ -432,6 +571,30 @@ export class EventService { this.editor.disposables.addFromEvent(eventSource, 'beforeinput', e => { this._onBeforeInput(e).catch(console.error); }); + this.editor.disposables.addFromEvent( + eventSource, + 'affine-android-ime-input', + e => { + this._onAndroidIMEInput(e).catch(console.error); + } + ); + this.editor.disposables.addFromEvent(eventSource, 'focusin', () => { + this._setAndroidEditorFocused(true); + }); + this.editor.disposables.addFromEvent( + eventSource, + 'focusout', + (event: FocusEvent) => { + const relatedTarget = event.relatedTarget; + if ( + relatedTarget instanceof Node && + this.editor.rootElement?.contains(relatedTarget) + ) { + return; + } + this._setAndroidEditorFocused(false); + } + ); this.editor.disposables.addFromEvent( eventSource, 'compositionstart', diff --git a/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineEditorWebView.kt b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineEditorWebView.kt new file mode 100644 index 0000000000..0aa74d5a3b --- /dev/null +++ b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineEditorWebView.kt @@ -0,0 +1,80 @@ +package app.affine.pro + +import android.content.Context +import android.util.AttributeSet +import android.view.inputmethod.EditorInfo +import android.view.inputmethod.InputConnection +import android.view.inputmethod.InputMethodManager +import com.getcapacitor.CapacitorWebView + +class AffineEditorWebView( + context: Context, + attrs: AttributeSet, +) : CapacitorWebView(context, attrs) { + private val imeState = AndroidImeState() + private var imeBridgeInstalled = false + @Volatile + private var isTrustedPage = false + + private val imeBridge = AffineImeBridge( + isTrustedPage = { this.isTrustedPage }, + clearComposingState = { imeState.nextClearRequestGeneration() }, + requestRestartInput = ::requestRestartInput, + onEditorFocusedChanged = { focused -> imeState.editorFocused = focused }, + ) + + fun updateAndroidIMEBridge(url: String?, expectedOrigin: String?) { + val shouldInstallBridge = isTrustedAffineOrigin(url, expectedOrigin) + if (shouldInstallBridge == imeBridgeInstalled) { + isTrustedPage = shouldInstallBridge + return + } + + if (shouldInstallBridge) { + addJavascriptInterface(imeBridge, AFFINE_IME_BRIDGE_NAME) + imeBridgeInstalled = true + isTrustedPage = true + } else { + isTrustedPage = false + imeBridgeInstalled = false + removeJavascriptInterface(AFFINE_IME_BRIDGE_NAME) + imeState.editorFocused = false + } + } + + override fun onCreateInputConnection(outAttrs: EditorInfo): InputConnection? { + val connection = super.onCreateInputConnection(outAttrs) ?: return null + return AffineInputConnection( + connection, + imeState, + dispatchDeleteBackward = { + dispatchAndroidEditorInput(this, AndroidImeInputType.BACKWARD_DELETE) + }, + dispatchDeleteForward = { + dispatchAndroidEditorInput(this, AndroidImeInputType.FORWARD_DELETE) + }, + ) + } + + private fun requestRestartInput(delayMs: Long) { + val restartGeneration = imeState.nextRestartGeneration() + if (delayMs <= 0L) { + post { restartInput() } + return + } + + postDelayed( + { + if (restartGeneration != imeState.restartInputGeneration) return@postDelayed + restartInput() + }, + delayMs, + ) + } + + private fun restartInput() { + val inputMethodManager = + context.getSystemService(Context.INPUT_METHOD_SERVICE) as? InputMethodManager + inputMethodManager?.restartInput(this) + } +} diff --git a/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineImeBridge.kt b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineImeBridge.kt new file mode 100644 index 0000000000..6dbc964796 --- /dev/null +++ b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineImeBridge.kt @@ -0,0 +1,64 @@ +package app.affine.pro + +import android.net.Uri +import android.webkit.JavascriptInterface + +internal const val AFFINE_IME_BRIDGE_NAME = "AffineAndroidIME" +internal const val AFFINE_IME_BRIDGE_PROTOCOL_VERSION = 1 +internal const val DELETE_RESTART_INPUT_DEBOUNCE_MS = 120L + +internal fun normalizeAffineOrigin(url: String?): String? { + val uri = Uri.parse(url ?: return null) + val scheme = uri.scheme?.lowercase() ?: return null + val host = uri.host?.lowercase() ?: return null + val isSupportedOrigin = + (scheme == "https" && host == "localhost") || + (BuildConfig.DEBUG && + scheme == "http" && + host in setOf("localhost", "127.0.0.1", "10.0.2.2")) + if (!isSupportedOrigin) return null + + val port = when { + uri.port == -1 -> "" + scheme == "https" && uri.port == 443 -> "" + scheme == "http" && uri.port == 80 -> "" + else -> ":${uri.port}" + } + return "$scheme://$host$port" +} + +internal fun isTrustedAffineOrigin(url: String?, expectedOrigin: String?): Boolean { + return expectedOrigin != null && normalizeAffineOrigin(url) == expectedOrigin +} + +internal class AffineImeBridge( + private val isTrustedPage: () -> Boolean, + private val clearComposingState: () -> Unit, + private val requestRestartInput: (Long) -> Unit, + private val onEditorFocusedChanged: (Boolean) -> Unit, +) { + @JavascriptInterface + fun getProtocolVersion(): Int { + return if (isTrustedPage()) AFFINE_IME_BRIDGE_PROTOCOL_VERSION else 0 + } + + @JavascriptInterface + fun finishComposingSession() { + if (!isTrustedPage()) return + clearComposingState() + requestRestartInput(0L) + } + + @JavascriptInterface + fun finishDeleteSession() { + if (!isTrustedPage()) return + requestRestartInput(DELETE_RESTART_INPUT_DEBOUNCE_MS) + } + + @JavascriptInterface + fun setEditorFocused(focused: Boolean) { + if (isTrustedPage()) { + onEditorFocusedChanged(focused) + } + } +} diff --git a/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineImeWebDispatcher.kt b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineImeWebDispatcher.kt new file mode 100644 index 0000000000..0c6fd8b4e2 --- /dev/null +++ b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineImeWebDispatcher.kt @@ -0,0 +1,70 @@ +package app.affine.pro + +import android.webkit.WebView + +internal enum class AndroidImeInputType( + val value: String, + val key: String, + val keyCode: Int, +) { + BACKWARD_DELETE("deleteContentBackward", "Backspace", 8), + FORWARD_DELETE("deleteContentForward", "Delete", 46), +} + +internal fun dispatchAndroidEditorInput( + webView: WebView, + inputType: AndroidImeInputType, +) { + webView.post { + webView.evaluateJavascript( + """ + (() => { + try { + const selection = document.getSelection(); + let target = selection?.anchorNode ?? document.activeElement ?? document.body; + if (target && target.nodeType === Node.TEXT_NODE) { + target = target.parentElement; + } + if (!(target instanceof EventTarget)) { + target = document.activeElement ?? document.body; + } + const detail = { + inputType: '${inputType.value}', + handled: false, + }; + const event = new CustomEvent('affine-android-ime-input', { + detail, + bubbles: true, + cancelable: true, + composed: true, + }); + const dispatched = target.dispatchEvent(event); + const handled = detail.handled || !dispatched; + let fallbackKey = null; + if (!handled) { + fallbackKey = '${inputType.key}'; + target.dispatchEvent(new KeyboardEvent('keydown', { + key: '${inputType.key}', + code: '${inputType.key}', + keyCode: ${inputType.keyCode}, + which: ${inputType.keyCode}, + bubbles: true, + cancelable: true, + composed: true, + })); + } + return { + inputType: '${inputType.value}', + handled, + fallbackKey, + }; + } catch (error) { + console.error('[AffineIME] dispatch editor input failed', error); + return { error: String(error) }; + } + })(); + """.trimIndent(), + null, + ) + } +} diff --git a/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineInputConnection.kt b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineInputConnection.kt new file mode 100644 index 0000000000..0f9024dc21 --- /dev/null +++ b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineInputConnection.kt @@ -0,0 +1,332 @@ +package app.affine.pro + +import android.view.KeyEvent +import android.view.inputmethod.ExtractedTextRequest +import android.view.inputmethod.InputConnection +import android.view.inputmethod.InputConnectionWrapper + +internal class AffineInputConnection( + target: InputConnection, + private val state: AndroidImeState, + private val dispatchDeleteBackward: () -> Unit, + private val dispatchDeleteForward: () -> Unit, +) : InputConnectionWrapper(target, true) { + private var handledClearRequestGeneration = state.clearRequestGeneration + private var composingText = "" + private var isComposingTextActive = false + private var isConsumingDeleteKeyEvent = false + + private val replay = ImeReplayController( + deleteBefore = { length -> super.deleteSurroundingText(length, 0) }, + recordDeleteIntent = { recordDeleteIntent() }, + hasRecentDeleteIntent = { currentTime -> hasRecentDeleteIntent(currentTime) }, + ) + + override fun setComposingRegion(start: Int, end: Int): Boolean { + consumeClearRequest() + val regionText = getTextForRegion(start, end) + + val nextComposingText = replay.updateComposingRegion( + regionText, + composingText, + isComposingTextActive, + ) + if (nextComposingText != null) { + composingText = nextComposingText + isComposingTextActive = nextComposingText.isNotEmpty() + } + + // Keep the native composing region untouched so IME autocorrect replay stays in the + // explicit replay state machine instead of being applied twice by the platform. + return true + } + + override fun setComposingText(text: CharSequence?, newCursorPosition: Int): Boolean { + consumeClearRequest() + val nextText = text?.toString() ?: "" + + if (replay.shouldAdoptExternalRegionForReplacement(nextText)) { + replay.clearDroppingReplay() + } + + if (replay.shouldDropExternalReplay(nextText)) { + replay.markDroppingReplay() + replay.deleteForShrinkingExternalReplay(nextText.length) + return true + } + + val adoptedText = replay.adoptExternalRegionAsComposingTextIfNeeded(nextText) + if (adoptedText != null) { + composingText = adoptedText + isComposingTextActive = adoptedText.isNotEmpty() + } + replay.clearExternalRegion() + return applyComposingText(nextText) + } + + override fun commitText(text: CharSequence?, newCursorPosition: Int): Boolean { + consumeClearRequest() + val committedText = text?.toString() ?: "" + + if ( + replay.shouldDeleteExternalRegionOnEmptyCommit( + committedText, + isComposingTextActive, + ) + ) { + replay.deleteRemainingExternalReplayText() + replay.clearExternalRegion() + return true + } + + if (replay.isDroppingReplay) { + if ( + committedText.isEmpty() || + committedText == replay.currentExternalRegionText + ) { + if (committedText.isEmpty()) { + replay.deleteRemainingExternalReplayTextAfterShrink() + } + return true + } + replay.clearExternalRegion() + } + + if (isComposingTextActive && committedText.isNotEmpty()) { + if (isWordBoundaryCommit(committedText)) { + resetComposingText() + replay.clearExternalRegion() + return super.commitText(text, newCursorPosition) + } + + val result = applyComposingText(committedText) + resetComposingText() + return result + } + + if (committedText.isNotEmpty()) { + replay.clearExternalRegion() + } + + return super.commitText(text, newCursorPosition) + } + + override fun finishComposingText(): Boolean { + consumeClearRequest() + resetComposingText() + replay.clearExternalRegion() + return super.finishComposingText() + } + + override fun deleteSurroundingText(beforeLength: Int, afterLength: Int): Boolean { + consumeClearRequest() + recordDeleteIntent(beforeLength, afterLength) + if (replay.shouldDropNativeDeleteAfterSyntheticExternalDelete(beforeLength, afterLength)) { + replay.clearExternalRegion() + return true + } + replay.clearExternalRegion() + if (isComposingTextActive && beforeLength > 0) { + trimComposingTail(beforeLength, codePoints = false) + } + return super.deleteSurroundingText(beforeLength, afterLength) + } + + override fun deleteSurroundingTextInCodePoints( + beforeLength: Int, + afterLength: Int, + ): Boolean { + consumeClearRequest() + recordDeleteIntent(beforeLength, afterLength) + if (replay.shouldDropNativeDeleteAfterSyntheticExternalDelete(beforeLength, afterLength)) { + replay.clearExternalRegion() + return true + } + replay.clearExternalRegion() + if (isComposingTextActive && beforeLength > 0) { + trimComposingTail(beforeLength, codePoints = true) + } + return super.deleteSurroundingTextInCodePoints(beforeLength, afterLength) + } + + override fun setSelection(start: Int, end: Int): Boolean { + consumeClearRequest() + replay.clearExternalRegion() + resetComposingText() + return super.setSelection(start, end) + } + + override fun performEditorAction(editorAction: Int): Boolean { + consumeClearRequest() + resetComposingText() + replay.clearExternalRegion() + return super.performEditorAction(editorAction) + } + + override fun sendKeyEvent(event: KeyEvent): Boolean { + val isDeleteActionDown = + event.action == KeyEvent.ACTION_DOWN && + (event.keyCode == KeyEvent.KEYCODE_DEL || + event.keyCode == KeyEvent.KEYCODE_FORWARD_DEL) + consumeClearRequest(skipNativeFinish = isDeleteActionDown) + + if (event.keyCode == KeyEvent.KEYCODE_DEL) { + if (!state.editorFocused && !isConsumingDeleteKeyEvent) { + return super.sendKeyEvent(event) + } + if (event.action == KeyEvent.ACTION_DOWN) { + recordDeleteIntent() + dispatchDeleteBackward() + isConsumingDeleteKeyEvent = true + return true + } + if (event.action == KeyEvent.ACTION_UP && isConsumingDeleteKeyEvent) { + isConsumingDeleteKeyEvent = false + return true + } + } + + if (event.keyCode == KeyEvent.KEYCODE_FORWARD_DEL) { + if (!state.editorFocused && !isConsumingDeleteKeyEvent) { + return super.sendKeyEvent(event) + } + if (event.action == KeyEvent.ACTION_DOWN) { + recordDeleteIntent() + dispatchDeleteForward() + isConsumingDeleteKeyEvent = true + return true + } + if (event.action == KeyEvent.ACTION_UP && isConsumingDeleteKeyEvent) { + isConsumingDeleteKeyEvent = false + return true + } + } + + if ( + event.action == KeyEvent.ACTION_DOWN && + (event.keyCode == KeyEvent.KEYCODE_SPACE || + event.keyCode == KeyEvent.KEYCODE_ENTER) + ) { + resetComposingText() + replay.clearExternalRegion() + } + return super.sendKeyEvent(event) + } + + private fun applyComposingText(nextText: String): Boolean { + val previousText = composingText + val prefixLength = commonPrefixLength(previousText, nextText) + val deleteCount = previousText.length - prefixLength + val insertText = nextText.substring(prefixLength) + + if (deleteCount > 0) { + super.deleteSurroundingText(deleteCount, 0) + } + if (insertText.isNotEmpty()) { + super.commitText(insertText, 1) + } + + composingText = nextText + isComposingTextActive = nextText.isNotEmpty() + return true + } + + private fun trimComposingTail(beforeLength: Int, codePoints: Boolean) { + val length = + if (codePoints) { + beforeLength.coerceAtMost(composingText.codePointCount(0, composingText.length)) + } else { + beforeLength.coerceAtMost(composingText.length) + } + composingText = + if (codePoints) { + val end = composingText.offsetByCodePoints(composingText.length, -length) + composingText.substring(0, end) + } else { + composingText.dropLast(length) + } + if (composingText.isEmpty()) { + resetComposingText() + } + } + + private fun resetComposingText() { + composingText = "" + isComposingTextActive = false + } + + private fun consumeClearRequest(skipNativeFinish: Boolean = false) { + val clearRequestGeneration = state.clearRequestGeneration + if ( + clearRequestGeneration == 0L || + clearRequestGeneration == handledClearRequestGeneration + ) { + return + } + + handledClearRequestGeneration = clearRequestGeneration + resetComposingText() + replay.clearExternalRegion() + if (skipNativeFinish) { + return + } + super.finishComposingText() + } + + private fun getTextForRegion(start: Int, end: Int): String { + if (start < 0 || end <= start) return "" + + val extractedText = + try { + getExtractedText(ExtractedTextRequest(), 0) + } catch (_: Exception) { + null + } ?: return "" + val text = extractedText.text?.toString().orEmpty() + val localStart = start - extractedText.startOffset + val localEnd = end - extractedText.startOffset + if (localStart < 0 || localEnd > text.length) return "" + + return text.substring(localStart, localEnd) + } + + private fun recordDeleteIntent(beforeLength: Int, afterLength: Int) { + if (beforeLength <= 0 || afterLength != 0) return + recordDeleteIntent() + } + + private fun recordDeleteIntent() { + state.lastDeleteIntentAtMs = android.os.SystemClock.uptimeMillis() + } + + private fun hasRecentDeleteIntent(currentTime: Long): Boolean { + return currentTime - state.lastDeleteIntentAtMs <= IME_REPLAY_DELETE_WINDOW_MS + } + + private fun isWordBoundaryCommit(text: String): Boolean { + return text == " " || text == "\n" + } + + private fun commonPrefixLength(left: String, right: String): Int { + val maxLength = minOf(left.length, right.length) + for (index in 0 until maxLength) { + if (left[index] != right[index]) { + return snapToCodePointBoundary(left, index) + } + } + return snapToCodePointBoundary(left, maxLength) + } + + private fun snapToCodePointBoundary(text: String, index: Int): Int { + return if ( + index > 0 && + index < text.length && + Character.isHighSurrogate(text[index - 1]) && + Character.isLowSurrogate(text[index]) + ) { + index - 1 + } else { + index + } + } +} diff --git a/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineWebViewClient.kt b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineWebViewClient.kt new file mode 100644 index 0000000000..f3f388588d --- /dev/null +++ b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AffineWebViewClient.kt @@ -0,0 +1,25 @@ +package app.affine.pro + +import android.webkit.WebResourceRequest +import android.webkit.WebView +import com.getcapacitor.Bridge +import com.getcapacitor.BridgeWebViewClient + +internal class AffineWebViewClient( + bridge: Bridge, + private val trustedOrigin: String?, +) : BridgeWebViewClient(bridge) { + override fun shouldOverrideUrlLoading( + view: WebView, + request: WebResourceRequest, + ): Boolean { + val shouldOverride = super.shouldOverrideUrlLoading(view, request) + if (!shouldOverride && request.isForMainFrame) { + (view as? AffineEditorWebView)?.updateAndroidIMEBridge( + request.url.toString(), + trustedOrigin, + ) + } + return shouldOverride + } +} diff --git a/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AndroidImeState.kt b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AndroidImeState.kt new file mode 100644 index 0000000000..9b86719e31 --- /dev/null +++ b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/AndroidImeState.kt @@ -0,0 +1,27 @@ +package app.affine.pro + +internal class AndroidImeState { + @Volatile + var clearRequestGeneration: Long = 0L + + @Volatile + var editorFocused: Boolean = false + + @Volatile + var lastDeleteIntentAtMs: Long = 0L + + @Volatile + var restartInputGeneration: Int = 0 + + @Synchronized + fun nextRestartGeneration(): Int { + restartInputGeneration += 1 + return restartInputGeneration + } + + @Synchronized + fun nextClearRequestGeneration(): Long { + clearRequestGeneration += 1 + return clearRequestGeneration + } +} diff --git a/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/ImeReplayController.kt b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/ImeReplayController.kt new file mode 100644 index 0000000000..aaace82506 --- /dev/null +++ b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/ImeReplayController.kt @@ -0,0 +1,217 @@ +package app.affine.pro + +import android.os.SystemClock + +internal const val IME_REPLAY_DELETE_WINDOW_MS = 500L + +internal class ImeReplayController( + private val deleteBefore: (Int) -> Unit, + private val recordDeleteIntent: () -> Unit, + private val hasRecentDeleteIntent: (Long) -> Boolean, + private val now: () -> Long = SystemClock::uptimeMillis, +) { + private var externalRegionText = "" + private var externalRegionAtMs = 0L + private var isDroppingExternalReplay = false + private var lastExternalReplayTextLength = -1 + private var lastExternalReplayTextAtMs = 0L + private var lastExternalReplayDeletedLength = 0 + private var syntheticExternalDeleteAtMs = 0L + + val currentExternalRegionText: String + get() = externalRegionText + + val isDroppingReplay: Boolean + get() = isDroppingExternalReplay + + fun updateComposingRegion( + regionText: String, + composingText: String, + isComposingTextActive: Boolean, + ): String? { + if (!isComposingTextActive) { + externalRegionText = regionText + externalRegionAtMs = now() + isDroppingExternalReplay = false + lastExternalReplayTextLength = regionText.length + lastExternalReplayTextAtMs = externalRegionAtMs + lastExternalReplayDeletedLength = 0 + return null + } + + if (regionText == composingText) return null + + clearExternalRegion() + return regionText + } + + fun shouldAdoptExternalRegionForReplacement(nextText: String): Boolean { + if (nextText.isEmpty() || externalRegionText.isEmpty()) return false + if (externalRegionText.startsWith(nextText)) return false + + val commonPrefixLength = commonPrefixLength(externalRegionText, nextText) + val minPrefixLength = minOf( + MIN_REPLACEMENT_COMMON_PREFIX_LENGTH, + externalRegionText.length, + nextText.length, + ) + + return commonPrefixLength >= minPrefixLength && + nextText.length >= externalRegionText.length + } + + fun shouldDropExternalReplay(nextText: String): Boolean { + if (nextText.isEmpty()) return false + if (isDroppingExternalReplay) { + return !shouldAdoptExternalRegionForReplacement(nextText) + } + if (externalRegionText.isEmpty()) return false + + val currentTime = now() + if ( + currentTime - externalRegionAtMs > EXTERNAL_REGION_REPLAY_WINDOW_MS && + !shouldAdoptExternalRegionForReplacement(nextText) + ) { + clearExternalRegion() + return false + } + + val isSameRegionReplay = + externalRegionText == nextText && hasRecentDeleteIntent(currentTime) + val isDeleteShrinkReplay = + externalRegionText.startsWith(nextText) && hasRecentDeleteIntent(currentTime) + val isLikelyPassiveReplay = nextText.length > 1 || externalRegionText.length > 1 + + return (isSameRegionReplay || isDeleteShrinkReplay) && isLikelyPassiveReplay + } + + fun markDroppingReplay() { + isDroppingExternalReplay = true + } + + fun clearDroppingReplay() { + isDroppingExternalReplay = false + } + + fun adoptExternalRegionAsComposingTextIfNeeded(nextText: String): String? { + if (externalRegionText.isEmpty()) return null + + val currentTime = now() + if ( + currentTime - externalRegionAtMs > EXTERNAL_REGION_REPLAY_WINDOW_MS && + !shouldAdoptExternalRegionForReplacement(nextText) + ) { + return null + } + + return externalRegionText + } + + fun clearExternalRegion() { + externalRegionText = "" + externalRegionAtMs = 0L + isDroppingExternalReplay = false + lastExternalReplayTextLength = -1 + lastExternalReplayTextAtMs = 0L + lastExternalReplayDeletedLength = 0 + } + + fun deleteForShrinkingExternalReplay(nextTextLength: Int) { + val currentTime = now() + val deleteCount = + if ( + lastExternalReplayTextLength > 0 && + nextTextLength < lastExternalReplayTextLength && + currentTime - lastExternalReplayTextAtMs <= IME_REPLAY_DELETE_WINDOW_MS + ) { + lastExternalReplayTextLength - nextTextLength + } else { + 0 + } + + if (deleteCount > 0) { + recordDeleteIntent() + deleteBefore(deleteCount) + lastExternalReplayDeletedLength += deleteCount + syntheticExternalDeleteAtMs = now() + } + + lastExternalReplayTextLength = nextTextLength + lastExternalReplayTextAtMs = currentTime + } + + fun deleteRemainingExternalReplayText() { + val currentTime = now() + if ( + lastExternalReplayTextLength <= 0 || + currentTime - lastExternalReplayTextAtMs > IME_REPLAY_DELETE_WINDOW_MS + ) { + return + } + + recordDeleteIntent() + deleteBefore(lastExternalReplayTextLength) + syntheticExternalDeleteAtMs = now() + lastExternalReplayTextLength = 0 + lastExternalReplayTextAtMs = currentTime + } + + fun deleteRemainingExternalReplayTextAfterShrink() { + if (lastExternalReplayDeletedLength <= 0) return + deleteRemainingExternalReplayText() + } + + fun shouldDropNativeDeleteAfterSyntheticExternalDelete( + beforeLength: Int, + afterLength: Int, + ): Boolean { + val currentTime = now() + return beforeLength > 0 && + afterLength == 0 && + currentTime - syntheticExternalDeleteAtMs <= + SYNTHETIC_EXTERNAL_DELETE_SUPPRESS_WINDOW_MS + } + + fun shouldDeleteExternalRegionOnEmptyCommit( + committedText: String, + isComposingTextActive: Boolean, + ): Boolean { + if (committedText.isNotEmpty()) return false + if (isComposingTextActive || externalRegionText.isEmpty()) return false + if (lastExternalReplayTextLength <= 0) return false + if (lastExternalReplayDeletedLength <= 0) return false + + val currentTime = now() + return currentTime - externalRegionAtMs <= IME_REPLAY_DELETE_WINDOW_MS && + hasRecentDeleteIntent(currentTime) + } + + private fun commonPrefixLength(left: String, right: String): Int { + val maxLength = minOf(left.length, right.length) + for (index in 0 until maxLength) { + if (left[index] != right[index]) { + return snapToCodePointBoundary(left, index) + } + } + return snapToCodePointBoundary(left, maxLength) + } + + private fun snapToCodePointBoundary(text: String, index: Int): Int { + return if ( + index > 0 && + index < text.length && + Character.isHighSurrogate(text[index - 1]) && + Character.isLowSurrogate(text[index]) + ) { + index - 1 + } else { + index + } + } + + private companion object { + const val EXTERNAL_REGION_REPLAY_WINDOW_MS = 500L + const val SYNTHETIC_EXTERNAL_DELETE_SUPPRESS_WINDOW_MS = 120L + const val MIN_REPLACEMENT_COMMON_PREFIX_LENGTH = 2 + } +} diff --git a/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/MainActivity.kt b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/MainActivity.kt index 35fd864a9d..fc7612d334 100644 --- a/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/MainActivity.kt +++ b/packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/MainActivity.kt @@ -6,6 +6,7 @@ import android.os.Bundle import android.view.Gravity import android.view.View import android.webkit.WebSettings +import android.webkit.WebView import androidx.activity.enableEdgeToEdge import androidx.coordinatorlayout.widget.CoordinatorLayout import androidx.core.content.ContextCompat @@ -30,6 +31,7 @@ import app.affine.pro.service.WebService import app.affine.pro.utils.px2dp import app.affine.pro.utils.dp2px import com.getcapacitor.BridgeActivity +import com.getcapacitor.WebViewListener import com.google.android.material.floatingactionbutton.FloatingActionButton import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.launch @@ -99,10 +101,26 @@ class MainActivity : BridgeActivity(), AIButtonPlugin.Callback, AFFiNEThemePlugi override fun load() { super.load() + configureAndroidIMEBridge() AuthInitializer.initialize(bridge) configureEditorWebView() } + private fun configureAndroidIMEBridge() { + val trustedOrigin = normalizeAffineOrigin(bridge.localUrl) + bridge.setWebViewClient(AffineWebViewClient(bridge, trustedOrigin)) + bridge.addWebViewListener(object : WebViewListener() { + override fun onPageCommitVisible(view: WebView?, url: String?) { + (view as? AffineEditorWebView)?.updateAndroidIMEBridge(url, trustedOrigin) + } + }) + + (bridge.webView as? AffineEditorWebView)?.updateAndroidIMEBridge( + bridge.webView.url ?: bridge.localUrl, + trustedOrigin, + ) + } + override fun onTrimMemory(level: Int) { super.onTrimMemory(level) if (level >= ComponentCallbacks2.TRIM_MEMORY_UI_HIDDEN) { diff --git a/packages/frontend/apps/android/App/app/src/main/res/layout/capacitor_bridge_layout_main.xml b/packages/frontend/apps/android/App/app/src/main/res/layout/capacitor_bridge_layout_main.xml new file mode 100644 index 0000000000..29e02335b0 --- /dev/null +++ b/packages/frontend/apps/android/App/app/src/main/res/layout/capacitor_bridge_layout_main.xml @@ -0,0 +1,13 @@ + + + + + +