diff --git a/packages/backend/native/index.d.ts b/packages/backend/native/index.d.ts index 024db8eaea..3e60c5c62c 100644 --- a/packages/backend/native/index.d.ts +++ b/packages/backend/native/index.d.ts @@ -59,7 +59,7 @@ export declare function parseDoc(filePath: string, doc: Buffer): Promise diff --git a/packages/backend/native/src/doc.rs b/packages/backend/native/src/doc.rs index c5da3c83b1..18fd743cc5 100644 --- a/packages/backend/native/src/doc.rs +++ b/packages/backend/native/src/doc.rs @@ -75,10 +75,15 @@ pub fn parse_doc_to_markdown( doc_bin: Buffer, doc_id: String, ai_editable: Option, + doc_url_prefix: Option, ) -> Result { - let result = - doc_parser::parse_doc_to_markdown(doc_bin.into(), doc_id, ai_editable.unwrap_or(false)) - .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?; + let result = doc_parser::parse_doc_to_markdown( + doc_bin.into(), + doc_id, + ai_editable.unwrap_or(false), + doc_url_prefix, + ) + .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?; Ok(result.into()) } diff --git a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md b/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md index 19b5633f8c..fc81da2a2b 100644 --- a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md +++ b/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md @@ -9,43 +9,66 @@ Generated by [AVA](https://avajs.dev). > Snapshot 1 { - markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ␊ + markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊ + ␊ + ␊ ␊ # You own your data, with no compromises␊ + ␊ ## Local-first & Real-time collaborative␊ + ␊ We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊ + ␊ AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊ ␊ + ␊ + ␊ ### Blocks that assemble your next docs, tasks kanban or whiteboard␊ - There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ␊ + ␊ + There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊ + ␊ We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊ + ␊ If you want to learn more about the product design of AFFiNE, here goes the concepts:␊ + ␊ To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊ + ␊ ## A true canvas for blocks in any form␊ - Many editor apps claimed to be a canvas for productivity. Since the Mother of All Demos, Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers. ␊ + ␊ + [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊ + ␊ + ␊ ␊ "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊ - - Quip & Notion with their great concept of "everything is a block"␊ - - Trello with their Kanban␊ - - Airtable & Miro with their no-code programable datasheets␊ - - Miro & Whimiscal with their edgeless visual whiteboard␊ - - Remnote & Capacities with their object-based tag system␊ - For more details, please refer to our RoadMap␊ + ␊ + * Quip & Notion with their great concept of "everything is a block"␊ + * Trello with their Kanban␊ + * Airtable & Miro with their no-code programable datasheets␊ + * Miro & Whimiscal with their edgeless visual whiteboard␊ + * Remnote & Capacities with their object-based tag system␊ + For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊ + ␊ ## Self Host␊ + ␊ Self host AFFiNE␊ ␊ + ␊ ### Learning From␊ ||Title|Tag|␊ |---|---|---|␊ - |Affine Development|Affine Development||␊ - |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc||␊ - |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"||␊ - |Trello with their Kanban|Trello with their Kanban||␊ - |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets||␊ - |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard||␊ + |Affine Development|Affine Development|AFFiNE|␊ + |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc|Developers|␊ + |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"|Reference|␊ + |Trello with their Kanban|Trello with their Kanban|Reference|␊ + |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets|Reference|␊ + |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|Reference|␊ |Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊ + ␊ ## Affine Development␊ - For developer or installation guides, please go to AFFiNE Development␊ + ␊ + For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊ + ␊ + ␊ ␊ `, title: 'Write, Draw, Plan all at Once.', diff --git a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap b/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap index 97e09a2abc..bc9899e48a 100644 Binary files a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap and b/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap differ diff --git a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.md b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.md index d5e57a9f14..9e86c8604e 100644 --- a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.md +++ b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.md @@ -9,43 +9,66 @@ Generated by [AVA](https://avajs.dev). > Snapshot 1 { - markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ␊ + markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊ + ␊ + ␊ ␊ # You own your data, with no compromises␊ + ␊ ## Local-first & Real-time collaborative␊ + ␊ We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊ + ␊ AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊ ␊ + ␊ + ␊ ### Blocks that assemble your next docs, tasks kanban or whiteboard␊ - There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ␊ + ␊ + There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊ + ␊ We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊ + ␊ If you want to learn more about the product design of AFFiNE, here goes the concepts:␊ + ␊ To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊ + ␊ ## A true canvas for blocks in any form␊ - Many editor apps claimed to be a canvas for productivity. Since the Mother of All Demos, Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers. ␊ + ␊ + [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊ + ␊ + ␊ ␊ "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊ - - Quip & Notion with their great concept of "everything is a block"␊ - - Trello with their Kanban␊ - - Airtable & Miro with their no-code programable datasheets␊ - - Miro & Whimiscal with their edgeless visual whiteboard␊ - - Remnote & Capacities with their object-based tag system␊ - For more details, please refer to our RoadMap␊ + ␊ + * Quip & Notion with their great concept of "everything is a block"␊ + * Trello with their Kanban␊ + * Airtable & Miro with their no-code programable datasheets␊ + * Miro & Whimiscal with their edgeless visual whiteboard␊ + * Remnote & Capacities with their object-based tag system␊ + For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊ + ␊ ## Self Host␊ + ␊ Self host AFFiNE␊ ␊ + ␊ ### Learning From␊ ||Title|Tag|␊ |---|---|---|␊ - |Affine Development|Affine Development||␊ - |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc||␊ - |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"||␊ - |Trello with their Kanban|Trello with their Kanban||␊ - |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets||␊ - |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard||␊ + |Affine Development|Affine Development|AFFiNE|␊ + |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc|Developers|␊ + |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"|Reference|␊ + |Trello with their Kanban|Trello with their Kanban|Reference|␊ + |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets|Reference|␊ + |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|Reference|␊ |Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊ + ␊ ## Affine Development␊ - For developer or installation guides, please go to AFFiNE Development␊ + ␊ + For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊ + ␊ + ␊ ␊ `, title: 'Write, Draw, Plan all at Once.', diff --git a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.snap b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.snap index e0bf7a8463..70dca3ed84 100644 Binary files a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.snap and b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.snap differ diff --git a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md index 3edbc40983..73ecc0f057 100644 --- a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md +++ b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md @@ -9,43 +9,66 @@ Generated by [AVA](https://avajs.dev). > Snapshot 1 { - markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ␊ + markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊ + ␊ + ␊ ␊ # You own your data, with no compromises␊ + ␊ ## Local-first & Real-time collaborative␊ + ␊ We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊ + ␊ AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊ ␊ + ␊ + ␊ ### Blocks that assemble your next docs, tasks kanban or whiteboard␊ - There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ␊ + ␊ + There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊ + ␊ We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊ + ␊ If you want to learn more about the product design of AFFiNE, here goes the concepts:␊ + ␊ To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊ + ␊ ## A true canvas for blocks in any form␊ - Many editor apps claimed to be a canvas for productivity. Since the Mother of All Demos, Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers. ␊ + ␊ + [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊ + ␊ + ␊ ␊ "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊ - - Quip & Notion with their great concept of "everything is a block"␊ - - Trello with their Kanban␊ - - Airtable & Miro with their no-code programable datasheets␊ - - Miro & Whimiscal with their edgeless visual whiteboard␊ - - Remnote & Capacities with their object-based tag system␊ - For more details, please refer to our RoadMap␊ + ␊ + * Quip & Notion with their great concept of "everything is a block"␊ + * Trello with their Kanban␊ + * Airtable & Miro with their no-code programable datasheets␊ + * Miro & Whimiscal with their edgeless visual whiteboard␊ + * Remnote & Capacities with their object-based tag system␊ + For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊ + ␊ ## Self Host␊ + ␊ Self host AFFiNE␊ ␊ + ␊ ### Learning From␊ ||Title|Tag|␊ |---|---|---|␊ - |Affine Development|Affine Development||␊ - |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc||␊ - |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"||␊ - |Trello with their Kanban|Trello with their Kanban||␊ - |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets||␊ - |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard||␊ + |Affine Development|Affine Development|AFFiNE|␊ + |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc|Developers|␊ + |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"|Reference|␊ + |Trello with their Kanban|Trello with their Kanban|Reference|␊ + |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets|Reference|␊ + |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|Reference|␊ |Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊ + ␊ ## Affine Development␊ - For developer or installation guides, please go to AFFiNE Development␊ + ␊ + For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊ + ␊ + ␊ ␊ `, title: 'Write, Draw, Plan all at Once.', diff --git a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap index e0bf7a8463..70dca3ed84 100644 Binary files a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap and b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap differ diff --git a/packages/backend/server/src/core/doc/reader.ts b/packages/backend/server/src/core/doc/reader.ts index f8b70226e5..f7cf85e249 100644 --- a/packages/backend/server/src/core/doc/reader.ts +++ b/packages/backend/server/src/core/doc/reader.ts @@ -192,7 +192,12 @@ export class DatabaseDocReader extends DocReader { if (!doc) { return null; } - return parseDocToMarkdownFromDocSnapshot(docId, doc.bin, aiEditable); + return parseDocToMarkdownFromDocSnapshot( + workspaceId, + docId, + doc.bin, + aiEditable + ); } async getDocDiff( diff --git a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md index 608a8d4db9..ea7ffb3ff1 100644 --- a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md +++ b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md @@ -1379,43 +1379,66 @@ Generated by [AVA](https://avajs.dev). > Snapshot 1 { - markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ␊ + markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊ + ␊ + ␊ ␊ # You own your data, with no compromises␊ + ␊ ## Local-first & Real-time collaborative␊ + ␊ We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊ + ␊ AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊ ␊ + ␊ + ␊ ### Blocks that assemble your next docs, tasks kanban or whiteboard␊ - There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ␊ + ␊ + There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊ + ␊ We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊ + ␊ If you want to learn more about the product design of AFFiNE, here goes the concepts:␊ + ␊ To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊ + ␊ ## A true canvas for blocks in any form␊ - Many editor apps claimed to be a canvas for productivity. Since the Mother of All Demos, Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers. ␊ + ␊ + [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊ + ␊ + ␊ ␊ "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊ - - Quip & Notion with their great concept of "everything is a block"␊ - - Trello with their Kanban␊ - - Airtable & Miro with their no-code programable datasheets␊ - - Miro & Whimiscal with their edgeless visual whiteboard␊ - - Remnote & Capacities with their object-based tag system␊ - For more details, please refer to our RoadMap␊ + ␊ + * Quip & Notion with their great concept of "everything is a block"␊ + * Trello with their Kanban␊ + * Airtable & Miro with their no-code programable datasheets␊ + * Miro & Whimiscal with their edgeless visual whiteboard␊ + * Remnote & Capacities with their object-based tag system␊ + For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊ + ␊ ## Self Host␊ + ␊ Self host AFFiNE␊ ␊ + ␊ ### Learning From␊ ||Title|Tag|␊ |---|---|---|␊ - |Affine Development|Affine Development||␊ - |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc||␊ - |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"||␊ - |Trello with their Kanban|Trello with their Kanban||␊ - |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets||␊ - |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard||␊ + |Affine Development|Affine Development|AFFiNE|␊ + |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc|Developers|␊ + |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"|Reference|␊ + |Trello with their Kanban|Trello with their Kanban|Reference|␊ + |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets|Reference|␊ + |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|Reference|␊ |Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊ + ␊ ## Affine Development␊ - For developer or installation guides, please go to AFFiNE Development␊ + ␊ + For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊ + ␊ + ␊ ␊ `, title: 'Write, Draw, Plan all at Once.', @@ -1427,49 +1450,72 @@ Generated by [AVA](https://avajs.dev). { markdown: `␊ - AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ␊ + AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊ + ␊ + ␊ ␊ # You own your data, with no compromises␊ + ␊ ## Local-first & Real-time collaborative␊ + ␊ We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊ + ␊ AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊ ␊ + ␊ + ␊ ␊ ### Blocks that assemble your next docs, tasks kanban or whiteboard␊ + ␊ ␊ - There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ␊ + There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊ + ␊ We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊ + ␊ If you want to learn more about the product design of AFFiNE, here goes the concepts:␊ + ␊ To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊ + ␊ ␊ ## A true canvas for blocks in any form␊ - Many editor apps claimed to be a canvas for productivity. Since the Mother of All Demos, Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers. ␊ + ␊ + [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊ + ␊ + ␊ ␊ "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊ - - Quip & Notion with their great concept of "everything is a block"␊ - - Trello with their Kanban␊ - - Airtable & Miro with their no-code programable datasheets␊ - - Miro & Whimiscal with their edgeless visual whiteboard␊ - - Remnote & Capacities with their object-based tag system␊ + ␊ + * Quip & Notion with their great concept of "everything is a block"␊ + * Trello with their Kanban␊ + * Airtable & Miro with their no-code programable datasheets␊ + * Miro & Whimiscal with their edgeless visual whiteboard␊ + * Remnote & Capacities with their object-based tag system␊ ␊ - For more details, please refer to our RoadMap␊ + For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊ + ␊ ## Self Host␊ + ␊ Self host AFFiNE␊ + ␊ ␊ ␊ ### Learning From␊ ||Title|Tag|␊ |---|---|---|␊ - |Affine Development|Affine Development||␊ - |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc||␊ - |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"||␊ - |Trello with their Kanban|Trello with their Kanban||␊ - |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets||␊ - |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard||␊ + |Affine Development|Affine Development|AFFiNE|␊ + |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc|Developers|␊ + |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"|Reference|␊ + |Trello with their Kanban|Trello with their Kanban|Reference|␊ + |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets|Reference|␊ + |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|Reference|␊ |Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊ + ␊ ␊ ## Affine Development␊ - For developer or installation guides, please go to AFFiNE Development␊ + ␊ + For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊ + ␊ + ␊ ␊ ␊ `, diff --git a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap index 1b99242ee5..fa26beded9 100644 Binary files a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap and b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap differ diff --git a/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts b/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts index f7a126d4cc..15b606dc90 100644 --- a/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts +++ b/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts @@ -79,6 +79,7 @@ test('can read all blocks from doc snapshot without workspace snapshot', async t test('can parse doc to markdown from doc snapshot', async t => { const result = parseDocToMarkdownFromDocSnapshot( + workspace.id, docSnapshot.id, docSnapshot.blob ); @@ -88,6 +89,7 @@ test('can parse doc to markdown from doc snapshot', async t => { test('can parse doc to markdown from doc snapshot with ai editable', async t => { const result = parseDocToMarkdownFromDocSnapshot( + workspace.id, docSnapshot.id, docSnapshot.blob, true diff --git a/packages/backend/server/src/core/utils/blocksuite.ts b/packages/backend/server/src/core/utils/blocksuite.ts index fd12fedf33..94ef14ec32 100644 --- a/packages/backend/server/src/core/utils/blocksuite.ts +++ b/packages/backend/server/src/core/utils/blocksuite.ts @@ -188,14 +188,17 @@ export async function readAllBlocksFromDocSnapshot( } export function parseDocToMarkdownFromDocSnapshot( + workspaceId: string, docId: string, docSnapshot: Uint8Array, aiEditable = false ) { + const docUrlPrefix = workspaceId ? `/workspace/${workspaceId}` : undefined; const parsed = parseYDocToMarkdown( Buffer.from(docSnapshot), docId, - aiEditable + aiEditable, + docUrlPrefix ); return { diff --git a/packages/common/native/Cargo.toml b/packages/common/native/Cargo.toml index 507f9def77..391827bfbd 100644 --- a/packages/common/native/Cargo.toml +++ b/packages/common/native/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2021" -name = "affine_common" -version = "0.1.0" +edition = "2021" +license-file = "LICENSE" +name = "affine_common" +version = "0.1.0" [features] default = [] diff --git a/packages/common/native/LICENSE b/packages/common/native/LICENSE new file mode 100644 index 0000000000..053ad06aa0 --- /dev/null +++ b/packages/common/native/LICENSE @@ -0,0 +1,44 @@ +The AFFiNE Enterprise Edition (EE) license (the “EE License”) +Copyright (c) 2022-present TOEVERYTHING PTE. LTD. and its affiliates. + +With regard to the AFFiNE Software: + +This software and associated documentation files (the "Software") may only be +used in production, if you (and any entity that you represent) have agreed to, +and are in compliance with, the AFFiNE Subscription Terms of Service, available +at https://affine.pro/terms/#subscription (the “EE Terms”), or other +agreement governing the use of the Software, as agreed by you and AFFiNE, +and otherwise have a valid AFFiNE Enterprise Edition subscription for the +correct number of user seats. Subject to the foregoing sentence, you are free to +modify this Software and publish patches to the Software. You agree that AFFiNE +and/or its licensors (as applicable) retain all right, title and interest in and +to all such modifications and/or patches, and all such modifications and/or +patches may only be used, copied, modified, displayed, distributed, or otherwise +exploited with a valid AFFiNE Enterprise Edition subscription for the correct +number of user seats. Notwithstanding the foregoing, you may copy and modify +the Software for development and testing purposes, without requiring a +subscription. You agree that AFFiNE and/or its licensors (as applicable) retain +all right, title and interest in and to all such modifications. You are not +granted any other rights beyond what is expressly stated herein. Subject to the +foregoing, it is forbidden to copy, merge, publish, distribute, sublicense, +and/or sell the Software. + +This EE License applies only to the part of this Software that is not +distributed as part of AFFiNE Community Edition (CE). Any part of this Software +distributed as part of AFFiNE CE or is served client-side as an image, font, +cascading stylesheet (CSS), file which produces or is compiled, arranged, +augmented, or combined into client-side JavaScript, in whole or in part, is +copyrighted under the MPL2.0 license. The full text of this EE License shall +be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +For all third party components incorporated into the AFFiNE Software, those +components are licensed under the original license provided by the owner of the +applicable component. diff --git a/packages/common/native/src/doc_loader/loader/source/parser.rs b/packages/common/native/src/doc_loader/loader/source/parser.rs index f6cbb89dae..2fcb1c7302 100644 --- a/packages/common/native/src/doc_loader/loader/source/parser.rs +++ b/packages/common/native/src/doc_loader/loader/source/parser.rs @@ -110,7 +110,7 @@ fn get_language_parser(language: &Language) -> Parser { }; parser .set_language(&lang.into()) - .unwrap_or_else(|_| panic!("Error loading grammar for language: {:?}", language)); + .unwrap_or_else(|_| panic!("Error loading grammar for language: {language:?}")); parser } diff --git a/packages/common/native/src/doc_parser.rs b/packages/common/native/src/doc_parser/affine.rs similarity index 53% rename from packages/common/native/src/doc_parser.rs rename to packages/common/native/src/doc_parser/affine.rs index 699fa76f98..bbe107d31a 100644 --- a/packages/common/native/src/doc_parser.rs +++ b/packages/common/native/src/doc_parser/affine.rs @@ -1,10 +1,22 @@ -use std::collections::{HashMap, HashSet}; - use serde::{Deserialize, Serialize}; use serde_json::{Map as JsonMap, Value as JsonValue}; use thiserror::Error; use y_octo::{Any, DocOptions, JwstCodecError, Map, Value}; +use super::{ + blocksuite::{ + collect_child_ids, get_block_id, get_flavour, get_list_depth, get_string, nearest_by_flavour, + DocContext, + }, + delta_markdown::{ + delta_value_to_inline_markdown, extract_inline_references, text_to_inline_markdown, + text_to_markdown, DeltaToMdOptions, + }, + value::{ + any_as_string, any_truthy, build_reference_payload, params_value_to_json, value_to_string, + }, +}; + const SUMMARY_LIMIT: usize = 1000; const PAGE_FLAVOUR: &str = "affine:page"; const NOTE_FLAVOUR: &str = "affine:note"; @@ -89,6 +101,7 @@ pub fn parse_doc_to_markdown( doc_bin: Vec, doc_id: String, ai_editable: bool, + doc_url_prefix: Option, ) -> Result { if doc_bin.is_empty() || doc_bin == [0, 0] { return Err(ParseError::InvalidBinary); @@ -107,36 +120,16 @@ pub fn parse_doc_to_markdown( }); } - let mut block_pool: HashMap = HashMap::new(); - let mut parent_lookup: HashMap = HashMap::new(); - - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() { - if let Some(block_id) = get_block_id(&block_map) { - for child_id in collect_child_ids(&block_map) { - parent_lookup.insert(child_id, block_id.clone()); - } - block_pool.insert(block_id, block_map); - } - } - } - - let root_block_id = block_pool - .iter() - .find_map(|(id, block)| { - get_flavour(block) - .filter(|flavour| flavour == PAGE_FLAVOUR) - .map(|_| id.clone()) - }) + let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) .ok_or_else(|| ParseError::ParserError("root block not found".into()))?; - - let mut queue: Vec<(Option, String)> = vec![(None, root_block_id.clone())]; - let mut visited: HashSet = HashSet::from([root_block_id.clone()]); + let root_block_id = context.root_block_id.clone(); + let mut walker = context.walker(); let mut doc_title = String::from("Untitled"); let mut markdown = String::new(); + let md_options = DeltaToMdOptions::new(doc_url_prefix); - while let Some((parent_block_id, block_id)) = queue.pop() { - let block = match block_pool.get(&block_id) { + while let Some((parent_block_id, block_id)) = walker.next() { + let block = match context.block_pool.get(&block_id) { Some(block) => block, None => continue, }; @@ -146,9 +139,9 @@ pub fn parse_doc_to_markdown( None => continue, }; - let parent_id = parent_lookup.get(&block_id); + let parent_id = context.parent_lookup.get(&block_id); let parent_flavour = parent_id - .and_then(|id| block_pool.get(id)) + .and_then(|id| context.block_pool.get(id)) .and_then(get_flavour); if parent_flavour.as_deref() == Some("affine:database") { @@ -156,12 +149,7 @@ pub fn parse_doc_to_markdown( } // enqueue children first to keep traversal order similar to JS implementation - let mut child_ids = collect_child_ids(block); - for child_id in child_ids.drain(..).rev() { - if visited.insert(child_id.clone()) { - queue.push((Some(block_id.clone()), child_id)); - } - } + walker.enqueue_children(&block_id, block); if flavour == PAGE_FLAVOUR { let title = get_string(block, "prop:title").unwrap_or_default(); @@ -171,109 +159,103 @@ pub fn parse_doc_to_markdown( if flavour == "affine:database" { let title = get_string(block, "prop:title").unwrap_or_default(); - markdown.push_str(&format!("\n### {}\n", title)); + markdown.push_str(&format!("\n### {title}\n")); - let columns_array = block.get("prop:columns").and_then(|v| v.to_array()); + let columns = parse_database_columns(block); let cells_map = block.get("prop:cells").and_then(|v| v.to_map()); - if let (Some(columns_array), Some(cells_map)) = (columns_array, cells_map) { - let mut columns = Vec::new(); - for col_val in columns_array.iter() { - if let Some(col_map) = col_val.to_map() { - let id = get_string(&col_map, "id").unwrap_or_default(); - let name = get_string(&col_map, "name").unwrap_or_default(); - let type_ = get_string(&col_map, "type").unwrap_or_default(); - let data = col_map.get("data").and_then(|v| v.to_map()); - columns.push((id, name, type_, data)); - } - } - + if let (Some(columns), Some(cells_map)) = (columns, cells_map) { let escape_table = |s: &str| s.replace('|', "\\|").replace('\n', "
"); + let mut table = String::new(); - markdown.push('|'); - for (_, name, _, _) in &columns { - markdown.push_str(&escape_table(name)); - markdown.push('|'); + table.push('|'); + for column in &columns { + table.push_str(&escape_table(column.name.as_deref().unwrap_or_default())); + table.push('|'); } - markdown.push('\n'); + table.push('\n'); - markdown.push('|'); + table.push('|'); for _ in &columns { - markdown.push_str("---|"); + table.push_str("---|"); } - markdown.push('\n'); + table.push('\n'); let child_ids = collect_child_ids(block); for child_id in child_ids { - markdown.push('|'); + table.push('|'); let row_cells = cells_map.get(&child_id).and_then(|v| v.to_map()); - for (col_id, _, col_type, col_data) in &columns { + for column in &columns { let mut cell_text = String::new(); - if col_type == "title" { - if let Some(child_block) = block_pool.get(&child_id) { - if let Some((text, _)) = text_content(child_block, "prop:text") { + if column.col_type == "title" { + if let Some(child_block) = context.block_pool.get(&child_id) { + if let Some(text_md) = + text_to_inline_markdown(child_block, "prop:text", &md_options) + { + cell_text = text_md; + } else if let Some((text, _)) = text_content(child_block, "prop:text") { cell_text = text; } } } else if let Some(row_cells) = &row_cells { - if let Some(cell_val) = row_cells.get(col_id).and_then(|v| v.to_map()) { - if let Some(value) = cell_val.get("value").and_then(|v| v.to_any()) { - cell_text = format_cell_value(&value, col_type, col_data.as_ref()); + if let Some(cell_val) = row_cells.get(&column.id).and_then(|v| v.to_map()) { + if let Some(value) = cell_val.get("value") { + if let Some(text_md) = delta_value_to_inline_markdown(&value, &md_options) { + cell_text = text_md; + } else { + cell_text = format_cell_value(&value, column); + } } } } - markdown.push_str(&escape_table(&cell_text)); - markdown.push('|'); + table.push_str(&escape_table(&cell_text)); + table.push('|'); } - markdown.push('\n'); + table.push('\n'); } + append_table_block(&mut markdown, &table); } continue; } if flavour == "affine:table" { let contents = gather_table_contents(block); - markdown.push_str(&contents.join("|")); - markdown.push('\n'); + let table = contents.join("|"); + append_table_block(&mut markdown, &table); continue; } if ai_editable && parent_block_id.as_ref() == Some(&root_block_id) { - markdown.push_str(&format!( - "\n", - block_id, flavour - )); + markdown.push_str(&format!("\n")); } if flavour == "affine:paragraph" { - if let Some((text, _)) = text_content(block, "prop:text") { - let type_ = get_string(block, "prop:type").unwrap_or_default(); - let prefix = match type_.as_str() { - "h1" => "# ", - "h2" => "## ", - "h3" => "### ", - "h4" => "#### ", - "h5" => "##### ", - "h6" => "###### ", - "quote" => "> ", - _ => "", - }; - markdown.push_str(prefix); - markdown.push_str(&text); - markdown.push('\n'); + let type_ = get_string(block, "prop:type").unwrap_or_default(); + let prefix = paragraph_prefix(type_.as_str()); + if let Some(text_md) = text_to_markdown(block, "prop:text", &md_options) { + append_paragraph(&mut markdown, prefix, &text_md); + } else if let Some((text, _)) = text_content(block, "prop:text") { + append_paragraph(&mut markdown, prefix, &text); } continue; } if flavour == "affine:list" { - if let Some((text, _)) = text_content(block, "prop:text") { - let depth = get_list_depth(&block_id, &parent_lookup, &block_pool); - let indent = " ".repeat(depth); - markdown.push_str(&indent); - markdown.push_str("- "); - markdown.push_str(&text); - markdown.push('\n'); + let type_ = get_string(block, "prop:type").unwrap_or_default(); + let checked = block + .get("prop:checked") + .and_then(|value| value.to_any()) + .as_ref() + .map(any_truthy) + .unwrap_or(false); + let prefix = list_prefix(type_.as_str(), checked); + let depth = get_list_depth(&block_id, &context.parent_lookup, &context.block_pool); + let indent = list_indent(depth); + if let Some(text_md) = text_to_markdown(block, "prop:text", &md_options) { + append_list_item(&mut markdown, &indent, prefix, &text_md); + } else if let Some((text, _)) = text_content(block, "prop:text") { + append_list_item(&mut markdown, &indent, prefix, &text); } continue; } @@ -281,11 +263,7 @@ pub fn parse_doc_to_markdown( if flavour == "affine:code" { if let Some((text, _)) = text_content(block, "prop:text") { let lang = get_string(block, "prop:language").unwrap_or_default(); - markdown.push_str("```"); - markdown.push_str(&lang); - markdown.push('\n'); - markdown.push_str(&text); - markdown.push_str("\n```\n"); + append_code_block(&mut markdown, &lang, &text); } continue; } @@ -297,27 +275,6 @@ pub fn parse_doc_to_markdown( }) } -fn get_list_depth( - block_id: &str, - parent_lookup: &HashMap, - blocks: &HashMap, -) -> usize { - let mut depth = 0; - let mut current_id = block_id.to_string(); - - while let Some(parent_id) = parent_lookup.get(¤t_id) { - if let Some(parent_block) = blocks.get(parent_id) { - if get_flavour(parent_block).as_deref() == Some("affine:list") { - depth += 1; - current_id = parent_id.clone(); - continue; - } - } - break; - } - depth -} - pub fn parse_doc_from_binary(doc_bin: Vec, doc_id: String) -> Result { if doc_bin.is_empty() || doc_bin == [0, 0] { return Err(ParseError::InvalidBinary); @@ -333,38 +290,16 @@ pub fn parse_doc_from_binary(doc_bin: Vec, doc_id: String) -> Result = HashMap::new(); - let mut parent_lookup: HashMap = HashMap::new(); - - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() { - if let Some(block_id) = get_block_id(&block_map) { - for child_id in collect_child_ids(&block_map) { - parent_lookup.insert(child_id, block_id.clone()); - } - block_pool.insert(block_id, block_map); - } - } - } - - let root_block_id = block_pool - .iter() - .find_map(|(id, block)| { - get_flavour(block) - .filter(|flavour| flavour == PAGE_FLAVOUR) - .map(|_| id.clone()) - }) + let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) .ok_or_else(|| ParseError::ParserError("root block not found".into()))?; - - let mut queue: Vec<(Option, String)> = vec![(None, root_block_id.clone())]; - let mut visited: HashSet = HashSet::from([root_block_id.clone()]); - let mut blocks: Vec = Vec::with_capacity(block_pool.len()); + let mut walker = context.walker(); + let mut blocks: Vec = Vec::with_capacity(context.block_pool.len()); let mut doc_title = String::new(); let mut summary = String::new(); let mut summary_remaining = SUMMARY_LIMIT as isize; - while let Some((parent_block_id, block_id)) = queue.pop() { - let block = match block_pool.get(&block_id) { + while let Some((parent_block_id, block_id)) = walker.next() { + let block = match context.block_pool.get(&block_id) { Some(block) => block, None => continue, }; @@ -374,20 +309,22 @@ pub fn parse_doc_from_binary(doc_bin: Vec, doc_id: String) -> Result continue, }; - let parent_block = parent_block_id.as_ref().and_then(|id| block_pool.get(id)); + let parent_block = parent_block_id + .as_ref() + .and_then(|id| context.block_pool.get(id)); let parent_flavour = parent_block.and_then(get_flavour); - let note_block = nearest_by_flavour(&block_id, NOTE_FLAVOUR, &parent_lookup, &block_pool); + let note_block = nearest_by_flavour( + &block_id, + NOTE_FLAVOUR, + &context.parent_lookup, + &context.block_pool, + ); let note_block_id = note_block.as_ref().and_then(get_block_id); let display_mode = determine_display_mode(note_block.as_ref()); // enqueue children first to keep traversal order similar to JS implementation - let mut child_ids = collect_child_ids(block); - for child_id in child_ids.drain(..).rev() { - if visited.insert(child_id.clone()) { - queue.push((Some(block_id.clone()), child_id)); - } - } + walker.enqueue_children(&block_id, block); let build_block = |database_name: Option<&String>| { BlockInfo::base( @@ -412,7 +349,7 @@ pub fn parse_doc_from_binary(doc_bin: Vec, doc_id: String) -> Result, doc_id: String) -> Result, doc_id: String) -> Result, doc_id: String) -> Result, doc_id: String) -> Result Vec { - block - .get("sys:children") - .and_then(|value| value.to_array()) - .map(|array| { - array - .iter() - .filter_map(|value| value_to_string(&value)) - .collect::>() - }) - .unwrap_or_default() +fn paragraph_prefix(type_: &str) -> &'static str { + match type_ { + "h1" => "# ", + "h2" => "## ", + "h3" => "### ", + "h4" => "#### ", + "h5" => "##### ", + "h6" => "###### ", + "quote" => "> ", + _ => "", + } } -fn get_block_id(block: &Map) -> Option { - get_string(block, "sys:id") +fn list_prefix(type_: &str, checked: bool) -> &'static str { + match type_ { + "bulleted" => "* ", + "todo" => { + if checked { + "- [x] " + } else { + "- [ ] " + } + } + _ => "1. ", + } } -fn get_flavour(block: &Map) -> Option { - get_string(block, "sys:flavour") +fn list_indent(depth: usize) -> String { + " ".repeat(depth) } -fn get_string(block: &Map, key: &str) -> Option { - block.get(key).and_then(|value| value_to_string(&value)) +fn append_paragraph(markdown: &mut String, prefix: &str, text: &str) { + markdown.push_str(prefix); + markdown.push_str(text); + if !text.ends_with('\n') { + markdown.push('\n'); + } + markdown.push('\n'); +} + +fn append_list_item(markdown: &mut String, indent: &str, prefix: &str, text: &str) { + markdown.push_str(indent); + markdown.push_str(prefix); + markdown.push_str(text); + if !text.ends_with('\n') { + markdown.push('\n'); + } +} + +fn append_code_block(markdown: &mut String, lang: &str, text: &str) { + markdown.push_str("```"); + markdown.push_str(lang); + markdown.push('\n'); + markdown.push_str(text); + markdown.push_str("\n```\n\n"); +} + +fn append_table_block(markdown: &mut String, table: &str) { + if table.is_empty() { + markdown.push('\n'); + return; + } + markdown.push_str(table); + if !table.ends_with('\n') { + markdown.push('\n'); + } + markdown.push('\n'); } fn text_content(block: &Map, key: &str) -> Option<(String, usize)> { @@ -601,24 +588,6 @@ fn text_content(block: &Map, key: &str) -> Option<(String, usize)> { }) } -fn nearest_by_flavour( - start: &str, - flavour: &str, - parent_lookup: &HashMap, - blocks: &HashMap, -) -> Option { - let mut cursor = Some(start.to_string()); - while let Some(node) = cursor { - if let Some(block) = blocks.get(&node) { - if get_flavour(block).as_deref() == Some(flavour) { - return Some(block.clone()); - } - } - cursor = parent_lookup.get(&node).cloned(); - } - None -} - fn determine_display_mode(note_block: Option<&Map>) -> String { match note_block.and_then(|block| get_string(block, "prop:displayMode")) { Some(mode) if mode == "both" => "page".into(), @@ -646,19 +615,24 @@ fn compose_additional( Some(JsonValue::Object(payload).to_string()) } -fn embed_ref_payload(block: &Map, page_id: &str) -> Option { - let mut payload = JsonMap::new(); - payload.insert("docId".into(), JsonValue::String(page_id.to_string())); +fn apply_blob_info(info: &mut BlockInfo, blob_id: String, content: String) { + info.blob = Some(vec![blob_id]); + info.content = Some(vec![content]); +} - if let Some(params_value) = block.get("prop:params") { - if let Ok(JsonValue::Object(params)) = serde_json::to_value(¶ms_value) { - for (key, value) in params.into_iter() { - payload.insert(key, value); - } - } +fn apply_doc_ref(info: &mut BlockInfo, page_id: String, payload: Option) { + info.ref_doc_id = Some(vec![page_id]); + if let Some(payload) = payload { + info.ref_info = Some(vec![payload]); } +} - Some(JsonValue::Object(payload).to_string()) +fn embed_ref_payload(block: &Map, page_id: &str) -> Option { + let params = block + .get("prop:params") + .as_ref() + .and_then(params_value_to_json); + Some(build_reference_payload(page_id, params)) } fn gather_surface_texts(block: &Map) -> Vec { @@ -698,22 +672,14 @@ fn gather_database_texts(block: &Map) -> (Vec, Option) { texts.push(title.clone()); } - if let Some(columns) = block.get("prop:columns").and_then(|value| value.to_array()) { - for column_value in columns.iter() { - if let Some(column) = column_value.to_map() { - if let Some(name) = get_string(&column, "name") { - texts.push(name); - } - if let Some(data) = column.get("data").and_then(|value| value.to_map()) { - if let Some(options) = data.get("options").and_then(|value| value.to_array()) { - for option_value in options.iter() { - if let Some(option) = option_value.to_map() { - if let Some(value) = get_string(&option, "value") { - texts.push(value); - } - } - } - } + if let Some(columns) = parse_database_columns(block) { + for column in columns.iter() { + if let Some(name) = column.name.as_ref() { + texts.push(name.clone()); + } + for option in column.options.iter() { + if let Some(value) = option.value.as_ref() { + texts.push(value.clone()); } } } @@ -736,79 +702,118 @@ fn gather_table_contents(block: &Map) -> Vec { contents } -fn format_cell_value(value: &Any, col_type: &str, col_data: Option<&Map>) -> String { - match col_type { +struct DatabaseOption { + id: Option, + value: Option, + color: Option, +} + +struct DatabaseColumn { + id: String, + name: Option, + col_type: String, + options: Vec, +} + +fn parse_database_columns(block: &Map) -> Option> { + let columns = block + .get("prop:columns") + .and_then(|value| value.to_array())?; + let mut parsed = Vec::new(); + for column_value in columns.iter() { + if let Some(column) = column_value.to_map() { + let id = get_string(&column, "id").unwrap_or_default(); + let name = get_string(&column, "name"); + let col_type = get_string(&column, "type").unwrap_or_default(); + let options = parse_database_options(&column); + parsed.push(DatabaseColumn { + id, + name, + col_type, + options, + }); + } + } + Some(parsed) +} + +fn parse_database_options(column: &Map) -> Vec { + let Some(data) = column.get("data").and_then(|value| value.to_map()) else { + return Vec::new(); + }; + let Some(options) = data.get("options").and_then(|value| value.to_array()) else { + return Vec::new(); + }; + + let mut parsed = Vec::new(); + for option_value in options.iter() { + if let Some(option) = option_value.to_map() { + parsed.push(DatabaseOption { + id: get_string(&option, "id"), + value: get_string(&option, "value"), + color: get_string(&option, "color"), + }); + } + } + parsed +} + +fn format_option_tag(option: &DatabaseOption) -> String { + let id = option.id.as_deref().unwrap_or_default(); + let value = option.value.as_deref().unwrap_or_default(); + let color = option.color.as_deref().unwrap_or_default(); + + format!( + "{value}" + ) +} + +fn format_cell_value(value: &Value, column: &DatabaseColumn) -> String { + match column.col_type.as_str() { "select" => { - if let Any::String(id) = value { - if let Some(options) = col_data - .and_then(|d| d.get("options")) - .and_then(|v| v.to_array()) - { - for opt in options.iter() { - if let Some(opt_map) = opt.to_map() { - if let Some(opt_id) = get_string(&opt_map, "id") { - if opt_id == *id { - return get_string(&opt_map, "value").unwrap_or_default(); - } - } - } + let id = match value { + Value::Any(any) => any_as_string(any).map(str::to_string), + Value::Text(text) => Some(text.to_string()), + _ => None, + }; + if let Some(id) = id { + for option in column.options.iter() { + if option.id.as_deref() == Some(id.as_str()) { + return format_option_tag(option); } } } String::new() } "multi-select" => { - if let Any::Array(ids) = value { - let mut selected = Vec::new(); - if let Some(options) = col_data - .and_then(|d| d.get("options")) - .and_then(|v| v.to_array()) - { - for id_val in ids.iter() { - if let Any::String(id) = id_val { - for opt in options.iter() { - if let Some(opt_map) = opt.to_map() { - if let Some(opt_id) = get_string(&opt_map, "id") { - if opt_id == *id { - selected.push(get_string(&opt_map, "value").unwrap_or_default()); - } - } - } - } - } + let ids: Vec = match value { + Value::Any(Any::Array(ids)) => ids + .iter() + .filter_map(any_as_string) + .map(str::to_string) + .collect(), + Value::Array(array) => array + .iter() + .filter_map(|id_val| value_to_string(&id_val)) + .collect(), + _ => Vec::new(), + }; + + if ids.is_empty() { + return String::new(); + } + + let mut selected = Vec::new(); + for id in ids.iter() { + for option in column.options.iter() { + if option.id.as_deref() == Some(id.as_str()) { + selected.push(format_option_tag(option)); } } - return selected.join(", "); } - String::new() + selected.join("") } - _ => any_to_string(value).unwrap_or_default(), - } -} - -fn value_to_string(value: &Value) -> Option { - if let Some(text) = value.to_text() { - return Some(text.to_string()); - } - - if let Some(any) = value.to_any() { - return any_to_string(&any); - } - - None -} - -fn any_to_string(any: &Any) -> Option { - match any { - Any::String(value) => Some(value.to_string()), - Any::Integer(value) => Some(value.to_string()), - Any::Float32(value) => Some(value.0.to_string()), - Any::Float64(value) => Some(value.0.to_string()), - Any::BigInt64(value) => Some(value.to_string()), - Any::True => Some("true".into()), - Any::False => Some("false".into()), - Any::Null | Any::Undefined => None, - Any::Array(_) | Any::Object(_) | Any::Binary(_) => serde_json::to_string(any).ok(), + _ => value_to_string(value).unwrap_or_default(), } } @@ -825,8 +830,8 @@ mod tests { #[test] fn test_parse_doc_from_binary() { - let json = include_bytes!("../fixtures/demo.ydoc.json"); - let doc_bin = include_bytes!("../fixtures/demo.ydoc").to_vec(); + let json = include_bytes!("../../fixtures/demo.ydoc.json"); + let doc_bin = include_bytes!("../../fixtures/demo.ydoc").to_vec(); let doc_id = "dYpV7PPhk8amRkY5IAcVO".to_string(); let result = parse_doc_from_binary(doc_bin, doc_id).unwrap(); @@ -838,4 +843,44 @@ mod tests { config ); } + + #[test] + fn test_paragraph_newlines() { + let mut markdown = String::new(); + append_paragraph(&mut markdown, "# ", "Title\n"); + assert_eq!(markdown, "# Title\n\n"); + + markdown.clear(); + append_paragraph(&mut markdown, "", "Plain"); + assert_eq!(markdown, "Plain\n\n"); + } + + #[test] + fn test_list_newlines() { + let mut markdown = String::new(); + append_list_item(&mut markdown, " ", "* ", "Item\n"); + assert_eq!(markdown, " * Item\n"); + + markdown.clear(); + append_list_item(&mut markdown, "", "- [ ] ", "Task"); + assert_eq!(markdown, "- [ ] Task\n"); + } + + #[test] + fn test_code_block_newlines() { + let mut markdown = String::new(); + append_code_block(&mut markdown, "rs", "fn main() {}"); + assert_eq!(markdown, "```rs\nfn main() {}\n```\n\n"); + } + + #[test] + fn test_table_newlines() { + let mut markdown = String::new(); + append_table_block(&mut markdown, "|a|b|\n|---|---|\n|1|2|\n"); + assert_eq!(markdown, "|a|b|\n|---|---|\n|1|2|\n\n"); + + markdown.clear(); + append_table_block(&mut markdown, "|a|b|"); + assert_eq!(markdown, "|a|b|\n\n"); + } } diff --git a/packages/common/native/src/doc_parser/blocksuite.rs b/packages/common/native/src/doc_parser/blocksuite.rs new file mode 100644 index 0000000000..b7d149c6ea --- /dev/null +++ b/packages/common/native/src/doc_parser/blocksuite.rs @@ -0,0 +1,164 @@ +use std::collections::{HashMap, HashSet}; + +use y_octo::Map; + +use super::value::value_to_string; + +pub(super) struct BlockIndex { + pub(super) block_pool: HashMap, + pub(super) parent_lookup: HashMap, +} + +pub(super) struct DocContext { + pub(super) block_pool: HashMap, + pub(super) parent_lookup: HashMap, + pub(super) root_block_id: String, +} + +pub(super) struct BlockWalker { + queue: Vec<(Option, String)>, + visited: HashSet, +} + +pub(super) fn build_block_index(blocks_map: &Map) -> BlockIndex { + let mut block_pool: HashMap = HashMap::new(); + let mut parent_lookup: HashMap = HashMap::new(); + + for (_, value) in blocks_map.iter() { + if let Some(block_map) = value.to_map() { + if let Some(block_id) = get_block_id(&block_map) { + for child_id in collect_child_ids(&block_map) { + parent_lookup.insert(child_id, block_id.clone()); + } + block_pool.insert(block_id, block_map); + } + } + } + + BlockIndex { + block_pool, + parent_lookup, + } +} + +impl DocContext { + pub(super) fn from_blocks_map(blocks_map: &Map, root_flavour: &str) -> Option { + let BlockIndex { + block_pool, + parent_lookup, + } = build_block_index(blocks_map); + + let root_block_id = find_block_id_by_flavour(&block_pool, root_flavour)?; + Some(Self { + block_pool, + parent_lookup, + root_block_id, + }) + } + + pub(super) fn walker(&self) -> BlockWalker { + BlockWalker::new(&self.root_block_id) + } +} + +impl BlockWalker { + fn new(root_block_id: &str) -> Self { + let mut visited = HashSet::new(); + visited.insert(root_block_id.to_string()); + + Self { + queue: vec![(None, root_block_id.to_string())], + visited, + } + } + + pub(super) fn next(&mut self) -> Option<(Option, String)> { + self.queue.pop() + } + + pub(super) fn enqueue_children(&mut self, parent_block_id: &str, block: &Map) { + let mut child_ids = collect_child_ids(block); + for child_id in child_ids.drain(..).rev() { + if self.visited.insert(child_id.clone()) { + self + .queue + .push((Some(parent_block_id.to_string()), child_id)); + } + } + } +} + +pub(super) fn find_block_id_by_flavour( + block_pool: &HashMap, + flavour: &str, +) -> Option { + block_pool.iter().find_map(|(id, block)| { + get_flavour(block) + .filter(|block_flavour| block_flavour == flavour) + .map(|_| id.clone()) + }) +} + +pub(super) fn collect_child_ids(block: &Map) -> Vec { + block + .get("sys:children") + .and_then(|value| value.to_array()) + .map(|array| { + array + .iter() + .filter_map(|value| value_to_string(&value)) + .collect::>() + }) + .unwrap_or_default() +} + +pub(super) fn get_block_id(block: &Map) -> Option { + get_string(block, "sys:id") +} + +pub(super) fn get_flavour(block: &Map) -> Option { + get_string(block, "sys:flavour") +} + +pub(super) fn get_string(block: &Map, key: &str) -> Option { + block.get(key).and_then(|value| value_to_string(&value)) +} + +pub(super) fn get_list_depth( + block_id: &str, + parent_lookup: &HashMap, + blocks: &HashMap, +) -> usize { + let mut depth = 0; + let mut current_id = block_id.to_string(); + + while let Some(parent_id) = parent_lookup.get(¤t_id) { + if let Some(parent_block) = blocks.get(parent_id) { + if get_flavour(parent_block).as_deref() == Some("affine:list") { + depth += 1; + current_id = parent_id.clone(); + continue; + } + } + break; + } + depth +} + +pub(super) fn nearest_by_flavour( + start: &str, + flavour: &str, + parent_lookup: &HashMap, + blocks: &HashMap, +) -> Option { + let mut cursor = Some(start.to_string()); + while let Some(node) = cursor { + if let Some(block) = blocks.get(&node) { + if get_flavour(block).as_deref() == Some(flavour) { + return Some(block.clone()); + } + } + cursor = parent_lookup.get(&node).cloned(); + } + None +} diff --git a/packages/common/native/src/doc_parser/delta_markdown.rs b/packages/common/native/src/doc_parser/delta_markdown.rs new file mode 100644 index 0000000000..6a0ddeafaa --- /dev/null +++ b/packages/common/native/src/doc_parser/delta_markdown.rs @@ -0,0 +1,791 @@ +use std::{ + cell::RefCell, + collections::{HashMap, HashSet}, + rc::{Rc, Weak}, +}; + +use y_octo::{AHashMap, Any, Map, Text, TextAttributes, TextDeltaOp, TextInsert, Value}; + +use super::value::{ + any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json, + value_to_any, +}; + +#[derive(Debug, Clone)] +struct InlineReference { + ref_type: Option, + page_id: String, + title: Option, + params: Option>, + mode: Option, +} + +#[derive(Debug, Clone)] +pub(super) struct InlineReferencePayload { + pub(super) doc_id: String, + pub(super) payload: String, +} + +#[derive(Debug, Clone)] +pub(super) struct DeltaToMdOptions { + doc_url_prefix: Option, +} + +impl DeltaToMdOptions { + pub(super) fn new(doc_url_prefix: Option) -> Self { + Self { doc_url_prefix } + } + + fn build_reference_link(&self, reference: &InlineReference) -> (String, String) { + let title = reference.title.clone().unwrap_or_default(); + + if let Some(prefix) = self.doc_url_prefix.as_deref() { + let prefix = prefix.trim_end_matches('/'); + return (title, format!("{}/{}", prefix, reference.page_id)); + } + + let mut parts = Vec::new(); + parts.push( + reference + .ref_type + .clone() + .unwrap_or_else(|| "LinkedPage".into()), + ); + parts.push(reference.page_id.clone()); + if let Some(mode) = reference.mode.as_ref() { + parts.push(mode.clone()); + } + + (title, parts.join(":")) + } +} + +pub(super) fn text_to_markdown( + block: &Map, + key: &str, + options: &DeltaToMdOptions, +) -> Option { + block + .get(key) + .and_then(|value| value.to_text()) + .map(|text| delta_to_markdown(&text, options)) +} + +pub(super) fn text_to_inline_markdown( + block: &Map, + key: &str, + options: &DeltaToMdOptions, +) -> Option { + block + .get(key) + .and_then(|value| value.to_text()) + .map(|text| delta_to_inline_markdown(&text, options)) +} + +pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec { + let mut refs = Vec::new(); + let mut seen: HashSet<(String, String)> = HashSet::new(); + + for op in delta { + let attrs = match op { + TextDeltaOp::Insert { + format: Some(format), + .. + } => format, + _ => continue, + }; + + let reference = match attrs.get("reference").and_then(parse_inline_reference) { + Some(reference) => reference, + None => continue, + }; + + let payload = match inline_reference_payload(&reference) { + Some(payload) => payload, + None => continue, + }; + + let key = (reference.page_id.clone(), payload.clone()); + if seen.insert(key.clone()) { + refs.push(InlineReferencePayload { + doc_id: key.0, + payload: key.1, + }); + } + } + + refs +} + +fn parse_inline_reference(value: &Any) -> Option { + let map = match value { + Any::Object(map) => map, + _ => return None, + }; + + let page_id = map + .get("pageId") + .and_then(any_as_string) + .map(str::to_string)?; + let title = map.get("title").and_then(any_as_string).map(str::to_string); + let ref_type = map.get("type").and_then(any_as_string).map(str::to_string); + let params = map.get("params").and_then(|value| match value { + Any::Object(map) => Some(map.clone()), + _ => None, + }); + let mode = params + .as_ref() + .and_then(|params| params.get("mode")) + .and_then(any_as_string) + .map(str::to_string); + + Some(InlineReference { + ref_type, + page_id, + title, + params, + mode, + }) +} + +fn inline_reference_payload(reference: &InlineReference) -> Option { + let params = reference.params.as_ref().map(params_any_map_to_json); + Some(build_reference_payload(&reference.page_id, params)) +} + +fn delta_to_markdown(text: &Text, options: &DeltaToMdOptions) -> String { + delta_to_markdown_with_options(&text.to_delta(), options, true) +} + +fn delta_to_inline_markdown(text: &Text, options: &DeltaToMdOptions) -> String { + delta_to_markdown_with_options(&text.to_delta(), options, false) +} + +fn delta_to_markdown_with_options( + delta: &[TextDeltaOp], + options: &DeltaToMdOptions, + trailing_newline: bool, +) -> String { + let ops = build_delta_ops(delta); + delta_ops_to_markdown_with_options(&ops, options, trailing_newline) +} + +fn delta_ops_to_markdown_with_options( + ops: &[DeltaOp], + options: &DeltaToMdOptions, + trailing_newline: bool, +) -> String { + let root = convert_delta_ops(ops, options); + let mut rendered = render_node(&root); + rendered = rendered.trim_end().to_string(); + if trailing_newline { + rendered.push('\n'); + } + rendered +} + +#[derive(Debug, Clone)] +struct DeltaOp { + insert: DeltaInsert, + attributes: TextAttributes, +} + +#[derive(Debug, Clone)] +enum DeltaInsert { + Text(String), + Embed(Vec), +} + +fn delta_ops_from_any(value: &Any) -> Option> { + let map = match value { + Any::Object(map) => map, + _ => return None, + }; + match map.get("$blocksuite:internal:text$") { + Some(Any::True) => {} + _ => return None, + } + + let delta = map.get("delta")?; + let entries = match delta { + Any::Array(entries) => entries, + _ => return None, + }; + + let mut ops = Vec::new(); + for entry in entries { + if let Some(op) = delta_op_from_any(entry) { + ops.push(op); + } + } + + Some(ops) +} + +fn delta_op_from_any(value: &Any) -> Option { + let map = match value { + Any::Object(map) => map, + _ => return None, + }; + + let insert_value = map.get("insert")?; + let insert = match insert_value { + Any::String(text) => DeltaInsert::Text(text.clone()), + Any::Array(values) => DeltaInsert::Embed(values.clone()), + _ => DeltaInsert::Embed(vec![insert_value.clone()]), + }; + + let attributes = map + .get("attributes") + .and_then(any_to_attributes) + .unwrap_or_default(); + + Some(DeltaOp { insert, attributes }) +} + +fn any_to_attributes(value: &Any) -> Option { + let map = match value { + Any::Object(map) => map, + _ => return None, + }; + + let mut attrs = TextAttributes::new(); + for (key, value) in map.iter() { + attrs.insert(key.clone(), value.clone()); + } + Some(attrs) +} + +fn delta_any_to_inline_markdown(value: &Any, options: &DeltaToMdOptions) -> Option { + delta_ops_from_any(value).map(|ops| delta_ops_to_markdown_with_options(&ops, options, false)) +} + +pub(super) fn delta_value_to_inline_markdown( + value: &Value, + options: &DeltaToMdOptions, +) -> Option { + if let Some(text) = value.to_text() { + return Some(delta_to_inline_markdown(&text, options)); + } + + let any = value_to_any(value)?; + delta_any_to_inline_markdown(&any, options) +} + +fn build_delta_ops(delta: &[TextDeltaOp]) -> Vec { + let mut ops = Vec::new(); + + for op in delta { + let (insert, attrs) = match op { + TextDeltaOp::Insert { insert, format } => (insert, format.clone().unwrap_or_default()), + _ => continue, + }; + + match insert { + TextInsert::Text(text) => ops.push(DeltaOp { + insert: DeltaInsert::Text(text.clone()), + attributes: attrs, + }), + TextInsert::Embed(values) => ops.push(DeltaOp { + insert: DeltaInsert::Embed(values.clone()), + attributes: attrs, + }), + } + } + + ops +} + +#[derive(Debug)] +struct Group { + node: Rc>, + kind: String, + distance: usize, + count: usize, +} + +fn convert_delta_ops(ops: &[DeltaOp], options: &DeltaToMdOptions) -> Rc> { + let root = Node::new_root(); + let mut group: Option = None; + let mut active_inline: HashMap = HashMap::new(); + let mut beginning_of_line = false; + + let mut line = Node::new_line(); + let mut el = line.clone(); + Node::append(&root, line.clone()); + + for index in 0..ops.len() { + let op = &ops[index]; + let next_attrs = ops.get(index + 1).map(|next| &next.attributes); + + match &op.insert { + DeltaInsert::Embed(values) => { + apply_inline_attributes(&mut el, &op.attributes, None, &mut active_inline, options); + for value in values { + match value { + Any::Object(map) => { + for (key, value) in map.iter() { + match key.as_str() { + "image" => { + if let Some(src) = any_as_string(value) { + let url = encode_link(src); + Node::append(&el, Node::new_text(&format!("![]({url})"))); + } + } + "thematic_break" => { + let current_open = el.borrow().open.clone(); + el.borrow_mut().open = format!("\n---\n{current_open}"); + } + _ => {} + } + } + } + Any::String(value) => { + Node::append(&el, Node::new_text(value)); + } + _ => {} + } + } + } + DeltaInsert::Text(text) => { + let lines: Vec<&str> = text.split('\n').collect(); + if has_block_level_attribute(&op.attributes) { + for _ in 1..lines.len() { + for (attr, value) in op.attributes.iter() { + match attr.as_str() { + "header" => { + if let Some(level) = any_as_u64(value) { + let prefix = "#".repeat(level as usize); + let current_open = line.borrow().open.clone(); + line.borrow_mut().open = format!("{prefix} {current_open}"); + new_line(&root, &mut line, &mut el, &mut active_inline); + break; + } + } + "blockquote" => { + let current_open = line.borrow().open.clone(); + line.borrow_mut().open = format!("> {current_open}"); + new_line(&root, &mut line, &mut el, &mut active_inline); + break; + } + "list" => { + if group.as_ref().is_some_and(|g| g.kind != attr.as_str()) { + group = None; + } + if group.is_none() { + let group_node = Node::new_line(); + Node::append(&root, group_node.clone()); + group = Some(Group { + node: group_node, + kind: attr.to_string(), + distance: 0, + count: 0, + }); + } + + if let Some(group) = group.as_mut() { + Node::append(&group.node, line.clone()); + group.distance = 0; + match any_as_string(value) { + Some("bullet") => { + let current_open = line.borrow().open.clone(); + line.borrow_mut().open = format!("- {current_open}"); + } + Some("checked") => { + let current_open = line.borrow().open.clone(); + line.borrow_mut().open = format!("- [x] {current_open}"); + } + Some("unchecked") => { + let current_open = line.borrow().open.clone(); + line.borrow_mut().open = format!("- [ ] {current_open}"); + } + Some("ordered") => { + group.count += 1; + let current_open = line.borrow().open.clone(); + line.borrow_mut().open = format!("{}. {}", group.count, current_open); + } + _ => {} + } + } + + new_line(&root, &mut line, &mut el, &mut active_inline); + break; + } + _ => {} + } + } + } + beginning_of_line = true; + } else { + for (line_index, segment) in lines.iter().enumerate() { + if (line_index > 0 || beginning_of_line) && group.is_some() { + let reset_group = group.as_mut().map(|group| { + group.distance += 1; + group.distance >= 2 + }); + if reset_group == Some(true) { + group = None; + } + } + + apply_inline_attributes( + &mut el, + &op.attributes, + next_attrs, + &mut active_inline, + options, + ); + Node::append(&el, Node::new_text(segment)); + if line_index + 1 < lines.len() { + new_line(&root, &mut line, &mut el, &mut active_inline); + } + } + beginning_of_line = false; + } + } + } + } + + root +} + +fn apply_inline_attributes( + el: &mut Rc>, + attrs: &TextAttributes, + next: Option<&TextAttributes>, + active_inline: &mut HashMap, + options: &DeltaToMdOptions, +) { + let mut first = Vec::new(); + let mut then = Vec::new(); + let mut seen: HashSet = HashSet::new(); + + let mut tag = el.clone(); + loop { + let format = match tag.borrow().format.clone() { + Some(format) => format, + None => break, + }; + seen.insert(format.clone()); + + let should_close = match attrs.get(&format) { + Some(value) => !any_truthy(value) || tag.borrow().open != tag.borrow().close, + None => true, + }; + + if should_close { + for key in seen.iter() { + active_inline.remove(key); + } + let parent = { + let tag_ref = tag.borrow(); + tag_ref.parent.as_ref().and_then(|p| p.upgrade()) + }; + if let Some(parent) = parent { + *el = parent.clone(); + tag = parent; + continue; + } + break; + } + + let parent = { + let tag_ref = tag.borrow(); + tag_ref.parent.as_ref().and_then(|p| p.upgrade()) + }; + if let Some(parent) = parent { + tag = parent; + } else { + break; + } + } + + for (attr, value) in attrs.iter() { + if !is_inline_attribute(attr) || !any_truthy(value) { + continue; + } + if let Some(active) = active_inline.get(attr) { + if active == value { + continue; + } + } + + let next_matches = next + .and_then(|next_attrs| next_attrs.get(attr)) + .map(|next_value| next_value == value) + .unwrap_or(false); + + if next_matches { + first.push(attr.clone()); + } else { + then.push(attr.clone()); + } + active_inline.insert(attr.clone(), value.clone()); + } + + for attr in first.into_iter().chain(then) { + if let Some(node) = inline_node_for_attr(&attr, attrs, options) { + node.borrow_mut().format = Some(attr.clone()); + Node::append(el, node.clone()); + *el = node; + } + } +} + +fn inline_node_for_attr( + attr: &str, + attrs: &TextAttributes, + options: &DeltaToMdOptions, +) -> Option>> { + match attr { + "italic" => Some(Node::new_inline("_", "_")), + "bold" => Some(Node::new_inline("**", "**")), + "link" => attrs + .get(attr) + .and_then(any_as_string) + .map(|url| Node::new_inline("[", &format!("]({url})"))), + "reference" => attrs + .get(attr) + .and_then(parse_inline_reference) + .map(|reference| { + let (title, link) = options.build_reference_link(&reference); + Node::new_inline("[", &format!("{title}]({link})")) + }), + "strike" => Some(Node::new_inline("~~", "~~")), + "code" => Some(Node::new_inline("`", "`")), + _ => None, + } +} + +fn has_block_level_attribute(attrs: &TextAttributes) -> bool { + attrs.contains_key("header") || attrs.contains_key("blockquote") || attrs.contains_key("list") +} + +fn is_inline_attribute(attr: &str) -> bool { + matches!( + attr, + "italic" | "bold" | "link" | "reference" | "strike" | "code" + ) +} + +fn encode_link(link: &str) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + + #[inline] + fn push_pct(out: &mut String, b: u8) { + out.push('%'); + out.push(HEX[(b >> 4) as usize] as char); + out.push(HEX[(b & 0x0f) as usize] as char); + } + + #[inline] + fn is_allowed(b: u8) -> bool { + matches!( + b, + b'A'..=b'Z' + | b'a'..=b'z' + | b'0'..=b'9' + | b'-' + | b'_' + | b'.' + | b'!' + | b'~' + | b'*' + | b'\'' + | b';' + | b',' + | b'/' + | b'?' + | b':' + | b'@' + | b'&' + | b'=' + | b'+' + | b'$' + | b'#' + ) + } + + let mut out = String::with_capacity(link.len()); + + for &b in link.as_bytes() { + match b { + b'(' | b')' => push_pct(&mut out, b), + b if is_allowed(b) => out.push(b as char), + b => push_pct(&mut out, b), + } + } + + if let Some(i) = out.find("?response-content-disposition=attachment") { + out.truncate(i); + } else if let Some(i) = out.find("&response-content-disposition=attachment") { + out.truncate(i); + } + + out +} + +#[derive(Debug)] +struct Node { + open: String, + close: String, + text: String, + children: Vec>>, + parent: Option>>, + format: Option, +} + +impl Node { + fn new_root() -> Rc> { + Rc::new(RefCell::new(Node { + open: String::new(), + close: String::new(), + text: String::new(), + children: Vec::new(), + parent: None, + format: None, + })) + } + + fn new_inline(open: &str, close: &str) -> Rc> { + Rc::new(RefCell::new(Node { + open: open.to_string(), + close: close.to_string(), + text: String::new(), + children: Vec::new(), + parent: None, + format: None, + })) + } + + fn new_text(text: &str) -> Rc> { + Rc::new(RefCell::new(Node { + open: String::new(), + close: String::new(), + text: text.to_string(), + children: Vec::new(), + parent: None, + format: None, + })) + } + + fn new_line() -> Rc> { + Rc::new(RefCell::new(Node { + open: String::new(), + close: "\n".to_string(), + text: String::new(), + children: Vec::new(), + parent: None, + format: None, + })) + } + + fn append(parent: &Rc>, child: Rc>) { + if let Some(old_parent) = child.borrow().parent.as_ref().and_then(|p| p.upgrade()) { + let mut old_parent = old_parent.borrow_mut(); + old_parent + .children + .retain(|existing| !Rc::ptr_eq(existing, &child)); + } + + child.borrow_mut().parent = Some(Rc::downgrade(parent)); + parent.borrow_mut().children.push(child); + } +} + +fn render_node(node: &Rc>) -> String { + let node_ref = node.borrow(); + let mut inner = node_ref.text.clone(); + for child in node_ref.children.iter() { + inner.push_str(&render_node(child)); + } + + if inner.trim().is_empty() + && node_ref.open == node_ref.close + && !node_ref.open.is_empty() + && !node_ref.close.is_empty() + { + return String::new(); + } + + let wrapped = !node_ref.open.is_empty() && !node_ref.close.is_empty(); + let empty_inner = inner.trim().is_empty(); + let mut fragments = Vec::new(); + + if inner.starts_with(' ') && !empty_inner && wrapped { + fragments.push(" ".to_string()); + } + if !node_ref.open.is_empty() { + fragments.push(node_ref.open.clone()); + } + fragments.push(if wrapped { + inner.trim().to_string() + } else { + inner.clone() + }); + if !node_ref.close.is_empty() { + fragments.push(node_ref.close.clone()); + } + if inner.ends_with(' ') && !empty_inner && wrapped { + fragments.push(" ".to_string()); + } + + fragments.join("") +} + +fn new_line( + root: &Rc>, + line: &mut Rc>, + el: &mut Rc>, + active_inline: &mut HashMap, +) { + *line = Node::new_line(); + *el = line.clone(); + Node::append(root, line.clone()); + active_inline.clear(); +} + +#[cfg(test)] +mod tests { + use serde_json::Value; + + use super::*; + + #[test] + fn test_delta_to_inline_markdown_link() { + let mut attrs = TextAttributes::new(); + attrs.insert("link".into(), Any::String("https://example.com".into())); + + let delta = vec![TextDeltaOp::Insert { + insert: TextInsert::Text("AFFiNE".into()), + format: Some(attrs), + }]; + + let options = DeltaToMdOptions::new(None); + let rendered = delta_to_markdown_with_options(&delta, &options, false); + assert_eq!(rendered, "[AFFiNE](https://example.com)"); + } + + #[test] + fn test_extract_inline_references_payload() { + let mut ref_map = AHashMap::default(); + ref_map.insert("pageId".into(), Any::String("doc123".into())); + ref_map.insert("title".into(), Any::String("Doc Title".into())); + ref_map.insert("type".into(), Any::String("LinkedPage".into())); + + let mut attrs = TextAttributes::new(); + attrs.insert("reference".into(), Any::Object(ref_map)); + + let delta = vec![TextDeltaOp::Insert { + insert: TextInsert::Text("Doc Title".into()), + format: Some(attrs), + }]; + + let refs = extract_inline_references(&delta); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].doc_id, "doc123"); + + let payload: Value = serde_json::from_str(&refs[0].payload).unwrap(); + assert_eq!(payload, serde_json::json!({ "docId": "doc123" })); + } +} diff --git a/packages/common/native/src/doc_parser/mod.rs b/packages/common/native/src/doc_parser/mod.rs new file mode 100644 index 0000000000..d3110c94a5 --- /dev/null +++ b/packages/common/native/src/doc_parser/mod.rs @@ -0,0 +1,9 @@ +mod affine; +mod blocksuite; +mod delta_markdown; +mod value; + +pub use affine::{ + get_doc_ids_from_binary, parse_doc_from_binary, parse_doc_to_markdown, BlockInfo, CrawlResult, + MarkdownResult, ParseError, +}; diff --git a/packages/common/native/src/doc_parser/value.rs b/packages/common/native/src/doc_parser/value.rs new file mode 100644 index 0000000000..7bf2383055 --- /dev/null +++ b/packages/common/native/src/doc_parser/value.rs @@ -0,0 +1,121 @@ +use serde_json::{Map as JsonMap, Value as JsonValue}; +use y_octo::{AHashMap, Any, Value}; + +pub(super) fn any_truthy(value: &Any) -> bool { + match value { + Any::True => true, + Any::False | Any::Null | Any::Undefined => false, + Any::String(value) => !value.is_empty(), + Any::Integer(value) => *value != 0, + Any::Float32(value) => value.0 != 0.0, + Any::Float64(value) => value.0 != 0.0, + Any::BigInt64(value) => *value != 0, + Any::Object(_) | Any::Array(_) | Any::Binary(_) => true, + } +} + +pub(super) fn any_as_string(value: &Any) -> Option<&str> { + match value { + Any::String(value) => Some(value), + _ => None, + } +} + +pub(super) fn any_as_u64(value: &Any) -> Option { + match value { + Any::Integer(value) if *value >= 0 => Some(*value as u64), + Any::Float32(value) if value.0 >= 0.0 => Some(value.0 as u64), + Any::Float64(value) if value.0 >= 0.0 => Some(value.0 as u64), + Any::BigInt64(value) if *value >= 0 => Some(*value as u64), + _ => None, + } +} + +pub(super) fn value_to_string(value: &Value) -> Option { + if let Some(text) = value.to_text() { + return Some(text.to_string()); + } + + if let Some(any) = value.to_any() { + return any_to_string(&any); + } + + None +} + +pub(super) fn value_to_any(value: &Value) -> Option { + if let Some(any) = value.to_any() { + return Some(any); + } + + if let Some(text) = value.to_text() { + return Some(Any::String(text.to_string())); + } + + if let Some(array) = value.to_array() { + let mut values = Vec::new(); + for item in array.iter() { + if let Some(any) = value_to_any(&item) { + values.push(any); + } else if let Some(text) = value_to_string(&item) { + values.push(Any::String(text)); + } + } + return Some(Any::Array(values)); + } + + if let Some(map) = value.to_map() { + let mut values = AHashMap::default(); + for key in map.keys() { + if let Some(entry) = map.get(key) { + if let Some(any) = value_to_any(&entry) { + values.insert(key.to_string(), any); + } else if let Some(text) = value_to_string(&entry) { + values.insert(key.to_string(), Any::String(text)); + } + } + } + return Some(Any::Object(values)); + } + + None +} + +pub(super) fn any_to_string(any: &Any) -> Option { + match any { + Any::String(value) => Some(value.to_string()), + Any::Integer(value) => Some(value.to_string()), + Any::Float32(value) => Some(value.0.to_string()), + Any::Float64(value) => Some(value.0.to_string()), + Any::BigInt64(value) => Some(value.to_string()), + Any::True => Some("true".into()), + Any::False => Some("false".into()), + Any::Null | Any::Undefined => None, + Any::Array(_) | Any::Object(_) | Any::Binary(_) => serde_json::to_string(any).ok(), + } +} + +pub(super) fn params_any_map_to_json(params: &AHashMap) -> JsonValue { + let mut values = JsonMap::new(); + for (key, value) in params.iter() { + if let Ok(value) = serde_json::to_value(value) { + values.insert(key.clone(), value); + } + } + JsonValue::Object(values) +} + +pub(super) fn params_value_to_json(params: &Value) -> Option { + serde_json::to_value(params).ok() +} + +pub(super) fn build_reference_payload(doc_id: &str, params: Option) -> String { + let mut payload = JsonMap::new(); + payload.insert("docId".into(), JsonValue::String(doc_id.to_string())); + if let Some(JsonValue::Object(params)) = params { + for (key, value) in params.into_iter() { + payload.insert(key, value); + } + } + JsonValue::Object(payload).to_string() +} diff --git a/packages/common/native/src/hashcash.rs b/packages/common/native/src/hashcash.rs index e3809f3b8a..acd4a0877b 100644 --- a/packages/common/native/src/hashcash.rs +++ b/packages/common/native/src/hashcash.rs @@ -85,10 +85,10 @@ impl Stamp { let hex_digits = ((bits as f32) / 4.).ceil() as usize; let zeros = String::from_utf8(vec![b'0'; hex_digits]).unwrap(); loop { - hasher.update(format!("{}:{:x}", challenge, counter).as_bytes()); + hasher.update(format!("{challenge}:{counter:x}").as_bytes()); let result = format!("{:x}", hasher.finalize_reset()); if result[..hex_digits] == zeros { - break format!("{:x}", counter); + break format!("{counter:x}"); }; counter += 1 } diff --git a/packages/common/y-octo/core/src/doc/codec/any.rs b/packages/common/y-octo/core/src/doc/codec/any.rs index 2723518a9d..40f0c47671 100644 --- a/packages/common/y-octo/core/src/doc/codec/any.rs +++ b/packages/common/y-octo/core/src/doc/codec/any.rs @@ -529,18 +529,18 @@ impl Display for Any { match self { Self::True => write!(f, "true"), Self::False => write!(f, "false"), - Self::String(s) => write!(f, "\"{}\"", s), - Self::Integer(i) => write!(f, "{}", i), - Self::Float32(v) => write!(f, "{}", v), - Self::Float64(v) => write!(f, "{}", v), - Self::BigInt64(v) => write!(f, "{}", v), + Self::String(s) => write!(f, "\"{s}\""), + Self::Integer(i) => write!(f, "{i}"), + Self::Float32(v) => write!(f, "{v}"), + Self::Float64(v) => write!(f, "{v}"), + Self::BigInt64(v) => write!(f, "{v}"), Self::Object(map) => { write!(f, "{{")?; for (i, (key, value)) in map.iter().enumerate() { if i > 0 { write!(f, ", ")?; } - write!(f, "{}: {}", key, value)?; + write!(f, "{key}: {value}")?; } write!(f, "}}") } @@ -550,11 +550,11 @@ impl Display for Any { if i > 0 { write!(f, ", ")?; } - write!(f, "{}", value)?; + write!(f, "{value}")?; } write!(f, "]") } - Self::Binary(buf) => write!(f, "{:?}", buf), + Self::Binary(buf) => write!(f, "{buf:?}"), Self::Undefined => write!(f, "undefined"), Self::Null => write!(f, "null"), } diff --git a/packages/common/y-octo/core/src/doc/codec/update.rs b/packages/common/y-octo/core/src/doc/codec/update.rs index 742ec9ff58..abfe71c719 100644 --- a/packages/common/y-octo/core/src/doc/codec/update.rs +++ b/packages/common/y-octo/core/src/doc/codec/update.rs @@ -166,7 +166,7 @@ impl Update { // merge two nodes, mark the index merged_index.push(index + 1); } else { - debug!("merge failed: {:?} {:?}", cur, next) + debug!("merge failed: {cur:?} {next:?}") } } diff --git a/packages/common/y-octo/core/src/doc/publisher.rs b/packages/common/y-octo/core/src/doc/publisher.rs index 3912d2b1f6..0f6658cc58 100644 --- a/packages/common/y-octo/core/src/doc/publisher.rs +++ b/packages/common/y-octo/core/src/doc/publisher.rs @@ -96,13 +96,13 @@ impl DocPublisher { let mut encoder = RawEncoder::default(); if let Err(e) = update.write(&mut encoder) { - warn!("Failed to encode document: {}", e); + warn!("Failed to encode document: {e}"); continue; } (encoder.into_inner(), history) } Err(e) => { - warn!("Failed to diff document: {}", e); + warn!("Failed to diff document: {e}"); continue; } }; @@ -117,7 +117,7 @@ impl DocPublisher { cb(&binary, &history); })) .unwrap_or_else(|e| { - warn!("Failed to call subscriber: {:?}", e); + warn!("Failed to call subscriber: {e:?}"); }); } } else { diff --git a/packages/common/y-octo/core/src/doc/store.rs b/packages/common/y-octo/core/src/doc/store.rs index 8ae493a17a..23fa03064e 100644 --- a/packages/common/y-octo/core/src/doc/store.rs +++ b/packages/common/y-octo/core/src/doc/store.rs @@ -90,7 +90,7 @@ impl DocStore { if let Some(last_struct) = structs.back() { last_struct.clock() + last_struct.len() } else { - warn!("client {} has no struct info", client); + warn!("client {client} has no struct info"); 0 } } else { @@ -108,7 +108,7 @@ impl DocStore { if let Some(last_struct) = structs.back() { state.insert(*client, last_struct.clock() + last_struct.len()); } else { - warn!("client {} has no struct info", client); + warn!("client {client} has no struct info"); } } state @@ -126,7 +126,7 @@ impl DocStore { return Err(JwstCodecError::StructClockInvalid { expect, actually }); } } else { - warn!("client {} has no struct info", client_id); + warn!("client {client_id} has no struct info"); } structs.push_back(item); } @@ -593,7 +593,7 @@ impl DocStore { self.delete_node(&Node::Item(item_owner_ref.clone()), Some(parent)); } else { // adjust parent length - if this.parent_sub.is_none() { + if this.parent_sub.is_none() && this.countable() { parent.len += this.len(); } } diff --git a/packages/common/y-octo/core/src/doc/types/list/mod.rs b/packages/common/y-octo/core/src/doc/types/list/mod.rs index 502df8582d..380d324adf 100644 --- a/packages/common/y-octo/core/src/doc/types/list/mod.rs +++ b/packages/common/y-octo/core/src/doc/types/list/mod.rs @@ -17,7 +17,7 @@ pub(crate) struct ItemPosition { impl ItemPosition { pub fn forward(&mut self) { if let Some(right) = self.right.get() { - if !right.deleted() { + if right.indexable() { self.index += right.len(); } @@ -103,7 +103,7 @@ pub(crate) trait ListType: AsInner { while remaining > 0 { if let Some(item) = pos.right.get() { - if !item.deleted() { + if item.indexable() { let content_len = item.len(); if remaining < content_len { pos.offset = remaining; @@ -148,7 +148,9 @@ pub(crate) trait ListType: AsInner { content: Content, ) -> JwstCodecResult { if let Some(markers) = &ty.markers { - markers.update_marker_changes(pos.index, content.clock_len() as i64); + if content.countable() { + markers.update_marker_changes(pos.index, content.clock_len() as i64); + } } let item = store.create_item( @@ -214,7 +216,7 @@ pub(crate) trait ListType: AsInner { while remaining > 0 { if let Some(item) = pos.right.get() { - if !item.deleted() { + if item.indexable() { let content_len = item.len(); if remaining < content_len { store.split_node(item.id, remaining)?; diff --git a/packages/common/y-octo/core/src/doc/types/text.rs b/packages/common/y-octo/core/src/doc/types/text.rs index 07bc05d802..1c271ffa0b 100644 --- a/packages/common/y-octo/core/src/doc/types/text.rs +++ b/packages/common/y-octo/core/src/doc/types/text.rs @@ -1,12 +1,43 @@ -use std::fmt::Display; +use std::{collections::BTreeMap, fmt::Display}; -use super::list::ListType; -use crate::{impl_type, Content, JwstCodecResult}; +use super::{list::ListType, AsInner}; +use crate::{ + doc::{DocStore, ItemRef, Node, Parent, Somr, YType, YTypeRef}, + impl_type, Any, Content, JwstCodecError, JwstCodecResult, +}; impl_type!(Text); impl ListType for Text {} +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +pub enum TextInsert { + Text(String), + Embed(Vec), +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +pub enum TextDeltaOp { + Insert { + insert: TextInsert, + #[serde(skip_serializing_if = "Option::is_none")] + format: Option, + }, + Retain { + retain: u64, + #[serde(skip_serializing_if = "Option::is_none")] + format: Option, + }, + Delete { + delete: u64, + }, +} + +pub type TextDelta = Vec; +pub type TextAttributes = BTreeMap; + impl Text { #[inline] pub fn len(&self) -> u64 { @@ -27,13 +58,113 @@ impl Text { pub fn remove(&mut self, char_index: u64, len: u64) -> JwstCodecResult { self.remove_at(char_index, len) } + + pub fn to_delta(&self) -> TextDelta { + let mut ops = Vec::new(); + let mut attrs = TextAttributes::new(); + + for item_ref in self.iter_item() { + if let Some(item) = item_ref.get() { + match &item.content { + Content::Format { key, value } => { + if is_nullish(value) { + attrs.remove(key.as_str()); + } else { + attrs.insert(key.to_string(), value.clone()); + } + } + Content::String(text) => { + push_insert(&mut ops, TextInsert::Text(text.clone()), &attrs); + } + Content::Embed(embed) => { + push_insert(&mut ops, TextInsert::Embed(vec![embed.clone()]), &attrs); + } + Content::Any(any) => { + push_insert(&mut ops, TextInsert::Embed(any.clone()), &attrs); + } + Content::Json(values) => { + let converted = values + .iter() + .map(|value| { + value + .as_ref() + .map(|s| Any::String(s.clone())) + .unwrap_or(Any::Undefined) + }) + .collect::>(); + push_insert(&mut ops, TextInsert::Embed(converted), &attrs); + } + Content::Binary(value) => { + push_insert( + &mut ops, + TextInsert::Embed(vec![Any::Binary(value.clone())]), + &attrs, + ); + } + _ => {} + } + } + } + + ops + } + + pub fn apply_delta(&mut self, delta: &[TextDeltaOp]) -> JwstCodecResult { + let (mut store, mut ty) = self.as_inner().write().ok_or(JwstCodecError::DocReleased)?; + let parent = self.as_inner().clone(); + + let mut pos = TextPosition::new(parent, ty.start.clone()); + + for op in delta { + match op { + TextDeltaOp::Insert { insert, format } => { + let attrs = format.clone().unwrap_or_default(); + match insert { + TextInsert::Text(text) => { + insert_text_content( + &mut store, + &mut ty, + &mut pos, + Content::String(text.clone()), + attrs, + )?; + } + TextInsert::Embed(values) => { + for value in values { + insert_text_content( + &mut store, + &mut ty, + &mut pos, + Content::Embed(value.clone()), + attrs.clone(), + )?; + } + } + } + } + TextDeltaOp::Retain { retain, format } => { + let attrs = format.clone().unwrap_or_default(); + if attrs.is_empty() { + advance_text_position(&mut store, &mut pos, *retain)?; + } else { + format_text(&mut store, &mut ty, &mut pos, *retain, attrs)?; + } + } + TextDeltaOp::Delete { delete } => { + delete_text(&mut store, &mut ty, &mut pos, *delete)?; + } + } + } + + Ok(()) + } } impl Display for Text { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.iter_item().try_for_each(|item| { if let Content::String(str) = &item.get().unwrap().content { - write!(f, "{}", str) + write!(f, "{str}") } else { Ok(()) } @@ -50,15 +181,361 @@ impl serde::Serialize for Text { } } +struct TextPosition { + parent: YTypeRef, + left: ItemRef, + right: ItemRef, + index: u64, + attrs: TextAttributes, +} + +impl TextPosition { + fn new(parent: YTypeRef, right: ItemRef) -> Self { + Self { + parent, + left: Somr::none(), + right, + index: 0, + attrs: TextAttributes::new(), + } + } + + fn forward(&mut self) { + if let Some(right) = self.right.get() { + if !right.deleted() { + if let Content::Format { key, value } = &right.content { + if is_nullish(value) { + self.attrs.remove(key.as_str()); + } else { + self.attrs.insert(key.to_string(), value.clone()); + } + } else if right.countable() { + self.index += right.len(); + } + } + + self.left = self.right.clone(); + self.right = right.right.clone(); + } + } +} + +fn is_nullish(value: &Any) -> bool { + matches!(value, Any::Null | Any::Undefined) +} + +fn push_insert(ops: &mut Vec, insert: TextInsert, attrs: &TextAttributes) { + let format = if attrs.is_empty() { + None + } else { + Some(attrs.clone()) + }; + + if let Some(TextDeltaOp::Insert { + insert: TextInsert::Text(prev), + format: prev_format, + }) = ops.last_mut() + { + if let TextInsert::Text(text) = insert { + if prev_format.as_ref() == format.as_ref() { + prev.push_str(&text); + return; + } + ops.push(TextDeltaOp::Insert { + insert: TextInsert::Text(text), + format, + }); + return; + } + } + + ops.push(TextDeltaOp::Insert { insert, format }); +} + +fn advance_text_position( + store: &mut DocStore, + pos: &mut TextPosition, + mut remaining: u64, +) -> JwstCodecResult { + while remaining > 0 { + let Some(item) = pos.right.get() else { + return Err(JwstCodecError::IndexOutOfBound(pos.index + remaining)); + }; + + if item.deleted() { + pos.forward(); + continue; + } + + if matches!(item.content, Content::Format { .. }) { + pos.forward(); + continue; + } + + let item_len = item.len(); + if remaining < item_len { + let (left, right) = store.split_node(item.id, remaining)?; + pos.left = left.as_item(); + pos.right = right.as_item(); + pos.index += remaining; + break; + } + + remaining -= item_len; + pos.forward(); + } + + Ok(()) +} + +fn minimize_attribute_changes(pos: &mut TextPosition, attrs: &TextAttributes) { + loop { + let Some(item) = pos.right.get() else { + break; + }; + + if item.deleted() { + pos.forward(); + continue; + } + + if let Content::Format { key, value } = &item.content { + let attr = attrs.get(key.as_str()).cloned().unwrap_or(Any::Null); + if attr == *value { + pos.forward(); + continue; + } + } + + break; + } +} + +fn insert_item( + store: &mut DocStore, + ty: &mut YType, + pos: &mut TextPosition, + content: Content, +) -> JwstCodecResult { + if let Some(markers) = &ty.markers { + if content.countable() { + markers.update_marker_changes(pos.index, content.clock_len() as i64); + } + } + + let item = store.create_item( + content, + pos.left.clone(), + pos.right.clone(), + Some(Parent::Type(pos.parent.clone())), + None, + ); + let item_ref = item.clone(); + store.integrate(Node::Item(item), 0, Some(ty))?; + pos.right = item_ref; + pos.forward(); + + Ok(()) +} + +fn insert_attributes( + store: &mut DocStore, + ty: &mut YType, + pos: &mut TextPosition, + attrs: &TextAttributes, +) -> JwstCodecResult { + let mut negated = TextAttributes::new(); + + for (key, value) in attrs { + let current = pos.attrs.get(key.as_str()).cloned().unwrap_or(Any::Null); + if current == *value { + continue; + } + + negated.insert(key.to_string(), current); + insert_item( + store, + ty, + pos, + Content::Format { + key: key.to_string(), + value: value.clone(), + }, + )?; + } + + Ok(negated) +} + +fn insert_negated_attributes( + store: &mut DocStore, + ty: &mut YType, + pos: &mut TextPosition, + mut negated: TextAttributes, +) -> JwstCodecResult { + loop { + let Some(item) = pos.right.get() else { + break; + }; + + if item.deleted() { + pos.forward(); + continue; + } + + if let Content::Format { key, value } = &item.content { + if let Some(negated_value) = negated.get(key.as_str()) { + if negated_value == value { + negated.remove(key.as_str()); + pos.forward(); + continue; + } + } + } + + break; + } + + for (key, value) in negated { + insert_item( + store, + ty, + pos, + Content::Format { + key: key.to_string(), + value, + }, + )?; + } + + Ok(()) +} + +fn insert_text_content( + store: &mut DocStore, + ty: &mut YType, + pos: &mut TextPosition, + content: Content, + mut attrs: TextAttributes, +) -> JwstCodecResult { + for key in pos.attrs.keys() { + if !attrs.contains_key(key.as_str()) { + attrs.insert(key.to_string(), Any::Null); + } + } + + minimize_attribute_changes(pos, &attrs); + let negated = insert_attributes(store, ty, pos, &attrs)?; + insert_item(store, ty, pos, content)?; + insert_negated_attributes(store, ty, pos, negated)?; + + Ok(()) +} + +fn format_text( + store: &mut DocStore, + ty: &mut YType, + pos: &mut TextPosition, + mut remaining: u64, + attrs: TextAttributes, +) -> JwstCodecResult { + if remaining == 0 { + return Ok(()); + } + + minimize_attribute_changes(pos, &attrs); + let mut negated = insert_attributes(store, ty, pos, &attrs)?; + + while remaining > 0 { + let Some(item) = pos.right.get() else { + break; + }; + + if item.deleted() { + pos.forward(); + continue; + } + + match &item.content { + Content::Format { key, value } => { + if let Some(attr) = attrs.get(key.as_str()) { + if attr == value { + negated.remove(key.as_str()); + } else { + negated.insert(key.to_string(), value.clone()); + } + store.delete_item(item, Some(ty)); + pos.forward(); + } else { + pos.forward(); + } + } + _ => { + let item_len = item.len(); + if remaining < item_len { + store.split_node(item.id, remaining)?; + remaining = 0; + } else { + remaining -= item_len; + } + pos.forward(); + } + } + } + + insert_negated_attributes(store, ty, pos, negated)?; + + Ok(()) +} + +fn delete_text( + store: &mut DocStore, + ty: &mut YType, + pos: &mut TextPosition, + mut remaining: u64, +) -> JwstCodecResult { + if remaining == 0 { + return Ok(()); + } + + let start = remaining; + + while remaining > 0 { + let Some(item) = pos.right.get() else { + break; + }; + + if item.indexable() { + let item_len = item.len(); + if remaining < item_len { + store.split_node(item.id, remaining)?; + remaining = 0; + } else { + remaining -= item_len; + } + store.delete_item(item, Some(ty)); + } + + pos.forward(); + } + + if let Some(markers) = &ty.markers { + markers.update_marker_changes(pos.index, -((start - remaining) as i64)); + } + + Ok(()) +} + #[cfg(test)] mod tests { use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha20Rng; use yrs::{Options, Text, Transact}; + use super::{TextAttributes, TextDeltaOp, TextInsert}; #[cfg(not(loom))] use crate::sync::{Arc, AtomicUsize, Ordering}; - use crate::{loom_model, sync::thread, Doc}; + use crate::{loom_model, sync::thread, Any, Doc}; #[test] fn test_manipulate_text() { @@ -290,4 +767,103 @@ mod tests { assert_eq!(text.to_string(), "hello great world!"); }); } + + #[test] + fn test_text_delta_insert_format() { + loom_model!({ + let doc = Doc::new(); + let mut text = doc.get_or_create_text("text").unwrap(); + + let mut attrs = TextAttributes::new(); + attrs.insert("bold".to_string(), Any::True); + + text + .apply_delta(&[TextDeltaOp::Insert { + insert: TextInsert::Text("abc".to_string()), + format: Some(attrs.clone()), + }]) + .unwrap(); + + assert_eq!(text.to_string(), "abc"); + assert_eq!( + text.to_delta(), + vec![TextDeltaOp::Insert { + insert: TextInsert::Text("abc".to_string()), + format: Some(attrs), + }] + ); + }); + } + + #[test] + fn test_text_delta_retain_format() { + loom_model!({ + let doc = Doc::new(); + let mut text = doc.get_or_create_text("text").unwrap(); + + text + .apply_delta(&[TextDeltaOp::Insert { + insert: TextInsert::Text("abc".to_string()), + format: None, + }]) + .unwrap(); + + let mut attrs = TextAttributes::new(); + attrs.insert("bold".to_string(), Any::True); + + text + .apply_delta(&[TextDeltaOp::Retain { + retain: 1, + format: Some(attrs.clone()), + }]) + .unwrap(); + + assert_eq!( + text.to_delta(), + vec![ + TextDeltaOp::Insert { + insert: TextInsert::Text("a".to_string()), + format: Some(attrs), + }, + TextDeltaOp::Insert { + insert: TextInsert::Text("bc".to_string()), + format: None, + } + ] + ); + }); + } + + #[test] + fn test_text_delta_utf16_retain() { + loom_model!({ + let doc = Doc::new(); + let mut text = doc.get_or_create_text("text").unwrap(); + + text + .apply_delta(&[TextDeltaOp::Insert { + insert: TextInsert::Text("😀".to_string()), + format: None, + }]) + .unwrap(); + + let mut attrs = TextAttributes::new(); + attrs.insert("bold".to_string(), Any::True); + + text + .apply_delta(&[TextDeltaOp::Retain { + retain: 2, + format: Some(attrs.clone()), + }]) + .unwrap(); + + assert_eq!( + text.to_delta(), + vec![TextDeltaOp::Insert { + insert: TextInsert::Text("😀".to_string()), + format: Some(attrs), + }] + ); + }); + } } diff --git a/packages/common/y-octo/core/src/doc/types/value.rs b/packages/common/y-octo/core/src/doc/types/value.rs index b0cd6a5883..420aa0bfc2 100644 --- a/packages/common/y-octo/core/src/doc/types/value.rs +++ b/packages/common/y-octo/core/src/doc/types/value.rs @@ -131,8 +131,8 @@ impl From for Value { impl Display for Value { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Value::Any(any) => write!(f, "{}", any), - Value::Text(text) => write!(f, "{}", text), + Value::Any(any) => write!(f, "{any}"), + Value::Text(text) => write!(f, "{text}"), _ => write!(f, ""), } } diff --git a/packages/common/y-octo/core/src/lib.rs b/packages/common/y-octo/core/src/lib.rs index 13c886abcc..ba7cf0bf08 100644 --- a/packages/common/y-octo/core/src/lib.rs +++ b/packages/common/y-octo/core/src/lib.rs @@ -9,7 +9,8 @@ pub use doc::{ encode_awareness_as_message, encode_update_as_message, merge_updates_v1, Any, Array, Awareness, AwarenessEvent, Client, ClientMap, Clock, CrdtRead, CrdtReader, CrdtWrite, CrdtWriter, Doc, DocOptions, HashMap as AHashMap, HashMapExt, History, HistoryOptions, Id, Map, RawDecoder, - RawEncoder, StateVector, StoreHistory, Text, Update, Value, + RawEncoder, StateVector, StoreHistory, Text, TextAttributes, TextDelta, TextDeltaOp, TextInsert, + Update, Value, }; pub(crate) use doc::{Content, Item}; use log::{debug, warn}; diff --git a/packages/common/y-octo/core/src/protocol/sync.rs b/packages/common/y-octo/core/src/protocol/sync.rs index 3f57fbf9f6..96fc2439f5 100644 --- a/packages/common/y-octo/core/src/protocol/sync.rs +++ b/packages/common/y-octo/core/src/protocol/sync.rs @@ -62,7 +62,7 @@ pub fn read_sync_message(input: &[u8]) -> IResult<&[u8], SyncMessage> { let (awareness_tail, awareness) = read_awareness(update)?; let tail_len = awareness_tail.len(); if tail_len > 0 { - debug!("awareness update has trailing bytes: {}", tail_len); + debug!("awareness update has trailing bytes: {tail_len}"); debug_assert!(tail_len > 0, "awareness update has trailing bytes"); } awareness