From afa984da543a5d49c3b649794836b639619953cb Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Wed, 14 May 2025 14:52:41 +0000 Subject: [PATCH] feat(server): handle workspace doc updates (#11937) --- .docker/selfhost/schema.json | 12 + .github/actions/server-test-env/action.yml | 6 + .github/workflows/build-images.yml | 2 + .github/workflows/build-test.yml | 5 + docs/developing-server.md | 6 + packages/backend/server/package.json | 1 + .../__fixtures__/test-doc.snapshot.bin | Bin 0 -> 36447 bytes .../__fixtures__/test-root-doc.snapshot.bin | Bin 0 -> 296 bytes .../src/__tests__/mocks/doc-snapshot.mock.ts | 42 ++ .../server/src/__tests__/mocks/index.ts | 2 + .../server/src/__tests__/mocks/queue.mock.ts | 1 + .../src/__tests__/mocks/workspace.mock.ts | 34 +- .../server/src/base/job/queue/config.ts | 8 + .../backend/server/src/base/job/queue/def.ts | 1 + .../server/src/base/job/queue/executor.ts | 6 +- packages/backend/server/src/core/doc/event.ts | 45 +- packages/backend/server/src/core/doc/job.ts | 11 +- .../__snapshots__/blocksute.spec.ts.md | 644 ++++++++++++++++++ .../__snapshots__/blocksute.spec.ts.snap | Bin 0 -> 6255 bytes .../core/utils/__tests__/blocksute.spec.ts | 57 ++ .../server/src/core/utils/blocksuite.ts | 52 +- packages/backend/server/src/models/doc.ts | 20 +- .../__snapshots__/service.spec.ts.md | 40 ++ .../__snapshots__/service.spec.ts.snap | Bin 2758 -> 3696 bytes .../src/plugins/indexer/__tests__/job.spec.ts | 108 +++ .../plugins/indexer/__tests__/service.spec.ts | 526 +++++++++++++- .../server/src/plugins/indexer/index.ts | 15 + .../backend/server/src/plugins/indexer/job.ts | 110 +++ .../server/src/plugins/indexer/service.ts | 213 +++++- packages/backend/server/tsconfig.json | 1 + packages/frontend/admin/src/config.json | 4 + tools/utils/src/workspace.gen.ts | 1 + yarn.lock | 1 + 33 files changed, 1940 insertions(+), 34 deletions(-) create mode 100644 packages/backend/server/src/__tests__/__fixtures__/test-doc.snapshot.bin create mode 100644 packages/backend/server/src/__tests__/__fixtures__/test-root-doc.snapshot.bin create mode 100644 packages/backend/server/src/__tests__/mocks/doc-snapshot.mock.ts create mode 100644 packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md create mode 100644 packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap create mode 100644 packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts create mode 100644 packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts create mode 100644 packages/backend/server/src/plugins/indexer/job.ts diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index 60d65f7ebf..a1b50b112b 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -70,6 +70,18 @@ "concurrency": 1 } }, + "queues.indexer": { + "type": "object", + "description": "The config for indexer job queue\n@default {\"concurrency\":1}", + "properties": { + "concurrency": { + "type": "number" + } + }, + "default": { + "concurrency": 1 + } + }, "queues.notification": { "type": "object", "description": "The config for notification job queue\n@default {\"concurrency\":10}", diff --git a/.github/actions/server-test-env/action.yml b/.github/actions/server-test-env/action.yml index 3f08733e43..b54ed8ba1f 100644 --- a/.github/actions/server-test-env/action.yml +++ b/.github/actions/server-test-env/action.yml @@ -4,6 +4,11 @@ description: 'Prepare Server Test Environment' runs: using: 'composite' steps: + - name: Bundle @affine/reader + shell: bash + run: | + yarn affine @affine/reader build + - name: Initialize database shell: bash run: | @@ -21,6 +26,7 @@ runs: yarn affine @affine/server prisma generate yarn affine @affine/server prisma migrate deploy yarn affine @affine/server data-migration run + - name: Import config shell: bash run: | diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 7492f97a6f..c8fbad4ce6 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -174,6 +174,8 @@ jobs: path: ./packages/backend/native - name: List server-native files run: ls -alh ./packages/backend/native + - name: Build @affine/reader + run: yarn workspace @affine/reader build - name: Build Server run: yarn workspace @affine/server build - name: Upload server dist diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 7447d07905..c77de2f63c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -172,6 +172,11 @@ jobs: name: server-native.node path: ./packages/backend/native + - name: Bundle @affine/reader + shell: bash + run: | + yarn workspace @affine/reader build + - name: Run Check run: | yarn affine init diff --git a/docs/developing-server.md b/docs/developing-server.md index 478179e88d..4fe6c98515 100644 --- a/docs/developing-server.md +++ b/docs/developing-server.md @@ -35,6 +35,12 @@ Server also requires native packages to be built, you can build them by running yarn affine @affine/server-native build ``` +## Build @affine/reader package + +```sh +yarn affine @affine/reader build +``` + ## Prepare dev environment ```sh diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index c534723c00..4dc509a61d 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -26,6 +26,7 @@ "postinstall": "prisma generate" }, "dependencies": { + "@affine/reader": "workspace:*", "@affine/server-native": "workspace:*", "@ai-sdk/anthropic": "^1.2.10", "@ai-sdk/google": "^1.2.10", diff --git a/packages/backend/server/src/__tests__/__fixtures__/test-doc.snapshot.bin b/packages/backend/server/src/__tests__/__fixtures__/test-doc.snapshot.bin new file mode 100644 index 0000000000000000000000000000000000000000..7c59832629a7757109a3b15ae19d626fe0862230 GIT binary patch literal 36447 zcmb__36NaXm8G(fsZ||&Rc^) zztb1;`uu*6%jNQfJZ+9Z(C2jfLVlk|3VPg<&!c?n4*BWZUoI0hOCGn|84O7wp9hVQ zd_Es)b_WCaEhPCQugm9gdDWV|UKf4+{Ay8iC>V5lQ95c41^sT(?2yy%bGiK?ugB*` zn_OPC)dAM(?=;(r(yb0D;B)#VuiG2&1cO0OND828)a-S;UHB^?h1@>BPx;*Mkv{#x z!ZK#_{=(K6Nu;Awxy8e&k%|21M4+o+`@}pwy)jqH-5ihEiY%QaFA>S(iSTr~khK+; zgvZC@Df#A1ctWnsG%aUy@pQ^|u;DML^rhnS=wv(*%gU*FEMajbo6g*vkLMF|!+bse zf{kO@cwTO^cV@%IHv5i5IAsqf683Q3zC9I{oz$7ZUEYo!_krBzDb|@AsWX&L?Yz;v zGpTf*bw)q^;<`)$0j5AtqPI|i(+FXU_o7g7^ki`Ar|_yO5=a6U^|q9rJ06&K{v zi8$-DYjJKe97`96OBu9nosN!>q(e<|jK>p6Dsa=nXgZMyXL7P_W}!~E`CvT}g~-*N z-mx>&bxhS1#2pGw=LLVvF@I*G8nXJrd#;|}D!vZS9@=AZ_=jE9^$)cQw zAK6IyyN$iba~LPVDJ%dkN*)^Kv+8FUF_hZT6{Dx|k58uqVSQsEDMT z%A?$vJrWM<2yiHZTzpJdo4>zFq)Di0aL@?~=3_14g8e+Y_hxMK^P)+yL-+L{qS?}+$?EO84 z-ZxB2#AQ9nNap(+ZrYtL*we+7O1W4#A8rHZ@{{&d+8#|O!HZ-(C+B!4tl_#_z(qYL zxV8opz1yUo!M#2P*I5P^WuThQ)(@AK)eNq)mT;Y|fXhvA=}AVo&Ncjbe>xgYKnG@X zdHYTFK^gy(k0)hd1uY2Osth;Y2WzyR7HCn&39Xrq!dPs;HQIlGp>>I&MH#53OZB6r zWi>wqZ=RC(=R#LDR)!a@a0ZO*$vX?2(dvOKJ)z4uNpd zNjo&B$TS{KrVCJI*?c^jfKH913wgm#!BU|m72T*3H74gWuxgagqKR}N=CpSgvM3~* zOlP5aNl_~WD?)?S1wF;*N}Knj;KP)&yx)U9x}tY)k@w7^Q? z9$HXC;p&M-RvcdN2%kbh4;oV8OePVJVh{nwLI(fn80;L*sRFQqJ6FhL(%HPz4po-5 z3uY)&bD|h7C=rQGs-|RJKb2f%(Sd#GCSR9C9~6wHDsnI3YqaX zS@-@ic#n&zBh>pLDLEAl9vDfFFvgx?y{8OR)0v9impvn+Goj@&>-d?9j{m7{m7dW$ zuJ{wY#%jl4wQl+f$Dkx966qrUqQW^C;c2zK1GHQ*wyNEOGmW$R2WJ{B9fWg@RUX3k z8m(P~?=@Qa2;Xb0?IhsK`n&|)jPipZBx@o_=T6sPDdHH)j4^_oX9S@P1n~I^g4A#b zw5(<H2cUq)?_*IPBc}GystD_kG!umS&h7}G}Rt?_;QXT?>WI3 z530M>k>wzJrT~rONPOjMNmSx(y;vE@P+g6W1{vODtzr5gs)tb;lU$Y4#s9C zlXKcpTnY9k%mW#o5Ikw_G2 z3a~(O!U2yKV2mUYk#X8#6c@vA;qj|dRxX{C^OM55RV-2QB`gy32O&DNz&anDW)r)TSAPW3?#AVmekD%Evp$Lo?BwYi078nV1%A% zWW>KL8QG!)*Yweski%I(3LimpG{tDcEDq{TC4X5oxg3E1it`r3qj)gOD&d3)RqbILuNCem8)A1NaM-E{C5$}YCn*@i& zm{lEr;0*Px#ifvxmB$NE)YEb*hCwH+E_m}I#u`(;G-y>t0jDvt(Q~0jQ4nJ480~Qk zM~sU^j*X13F1422BTFGU5&}uipXiJiI`{AC>1UF=xQrx68Ax&$ml-8zZg*+9%!GDv znNetDcU@ej3a!HKy11;`?pmeWUBbhe$YK)YO{_4p26$R+GYbv3FtaW$o88R1xXjYb zdT&{knf04x)@IgkmRXruzgbq>%)*y*G_y{M$&_SG1li;6jYs+pv}YZRAXgYcC<7sU zrGg+eCQ~h|8B4BIu!NV-hjzmx3w&PD2vlKY{chQf?e=`O0G$_3O^0*J=+!1aSU}-a zN&Lc=g{~~C8chpzH|F24=)0IYPH2r3r^jNLkPi$nv`#Fi;Y1mzHz$@Ghf@u-w5(=m zomg%hQ@nhRR)tY|V!4(m6k(&@!;{PBC*rB8RX0rL^O>7BZGtL9M9Y~=-*9JnY2o4J zD+dH_B2WlTCM-|;;d0`))bHn~3WoG&j2^N2*myou0E z-^jknt@|X#yn9v<^C$z=bkB;qnWtqnW8OV0teAJtiWYKq2%fv+tSu9E*i{hjS|E<(! zmz@(%X8k>|!W#VJ0(`=ez@O>PW@GUk9>*x7-qDprJ<34fA6;prUfG67#B=$oTxPf* zU1`O-qboV<7Dk|gh5ll~pM7?^Mf0*%@K`D}ChOyAtpdk_7*7CNxvp z=G{J(VDP`m;HL~!%bRt|4mEy-<%D8J%bVce_OeBI4k1Rs>i6}-lxPGdAlu9 zeX|y-Z>}^)^$me40Ys?w?e$KM9O&EAy_2DO`zj(eWuVsGzN#KlYq`u&y?vDxsc&C3 z3#s)CMpTci3bn_xd1d~($u9OyaClQ`#3vE$G0bvUf67hD*p0zEW(oW4t7?$>_EqMv zA6f<22|wOnSHA6{)m^~0-Yp}L;Ii22{H4h+gkvCIY)-Vw&$C9GhJkfYcQqNAi&`!-(4VaUogD9HHP9;@?`Kz6JKl3!@OV~3M%#yYO?X#=cc`{2j;<(U zpn9$G+10bV;?J(Obj5$Xy2=%QX|=U0{?ckISNx^bwOw(1IY(FgvT!a*`b3%C`%^u> z?f&*IKcmcRj53sg5Pq$KGBun_Evp%CUaQ~@FP{(A3g_~*@<=ug)QhWc>W0Ut1TAAS z777w@*RY!bqf&&&M64^V*toh2ceq(`N4<(Wnu9+og4IrSBc&4zQT~5Z6j4yjXKGAy zQ#KurCBqqVSdTO}4dSvYLtKXtR|rJ=)C0V~CXKNk-uuZ(bz!1KWGk21C5L3KFtR zu@QYSb&e3(ISOB1&W%U*F+?6>h)@PXLo$r-dF69w+DBp^9-7&7&MfDYI>>ynzGMiRR3BoGgO|cK!w!* zQyMB2>i?-~{Y-jmvTAAPXU`%g8` zuJ@m6w$%I2HdpEWA2eI*{U0=2>HQxx*Vg;^a*lfcCql81*oiA6dwbHo-JPL!&XpG# zS11D^{9*-HuGMy(i-ibb{#51E`Em_8$}d*%g{P>}osP@J+`J}TJHFJ+`22EngE&62 zrY9fljT~@IZ!UQ}V?zmFaJ1LGn;P(P^ZXRnFwy&A#Z4JSnTFJ)a?oIb;>1uBUT$u} ziew>~;`RT8*Z-PY|Kez3TQCCY)84yD0>B;PeX8k%vWH}CKy zy#vM3Qg^Skhbp|-yf}`+%h{D?9Vy3FU*>&$OT*afk+)2Fa#MXf$GQ{YWN9L}WeZym zd8>@^64j%&8$m|03UZ^mxrci5mKn;w;w^YbYr!mCdZ&3ltT@?L();laFXcV0lw3zB zw55HsduH#>0oQ@>j;?)fzH#lnaw)up)sp^M12-eTi6O^N}eyOsV6FY_ya7_%|s-n>cbEMG+CN$zTT&r|%2lg||*sLsb z7Qd*e@5*yn|F3yjzgTh>OaI;2TTjK6zlb7+le5v(a@U$QTiZtmGnwRWX=iem&$}ZU zNlvsUZS&?EG7a_w`%_)fcyF?6+tzGXQVQ)33|D6IdS!X{Ol146QmJEeHZkS*rejyH_G`S_)8%Rnyrm+MuGW6daCn01 z#?MOK^vgHaG^I1D@@&%QkH>jc-_$_so#@T=Z6DdPjcLelu30c0P82X}EE*IpgV>-> zjX)|!VjAZXubU2MKcQPJVx>5Oul`c++c$ZIXS51k`zAvZ+lt=Z%vC+3SGc$nvEPIk zGb}=eWUa=vRW;&J1=g3S&NIBur?ooUXNF{l&oi@c7j@%lz0O8;_kkhFvWM80Gxe90 z^|V@pxk6f~L;ughRwYw{1a;S%#raHW`<}>-sI-F}==<&(^CB7T&#jqNBT_x*wBF96 zHhq5$+C<1uUbJa+U?!C4EqaCrS(|<^dz*eZmp1*dx=rUrn}{KXHtqEHr$$SC>7F6h zrWa;!(~EOy(~H$@`WMkA8YYG|bxn2mW;Q#LTjQ)vKc2l!FU_S*FIBhcl^SjGga;Ct zN!gvqur|Fqdz*eTmp1*Rx=pXwXwyJ^Pktmm;TZ2{ZF*z&HoZBQHoaNhri&GA(obcx zK}Uo67i%b>KpA*?-2sJPtWmP+hCl1}FVb7FvlqN?s8g{#4Xeh7M)m>2;%X)f%0vmgroqg^r$RMCY%syG2C9 zV)~>acGhFf7TX=LiBmWi_K5;4XNr+cq(y*C+mF_nj`IFhL8*sn{vdcppb(NwX;De% zt|OjN2A*CwlIO0o)>wY`I$%k#5SC2Y5>|I# zPgqh0o?bVWcVBOb<=xj?VR`rUHL%nZjac4yeI2W@cVBM}(HE`-@N{hv9$VF4Gh07FeEq#LpE0tMpqbyl!2$$jo}q* z46j&Xc%>GGdZH1--(CM7UMIa$1Hj7yK*E9ml(W0L+#P{v)W@vV(`#v*QU(I>^xC=s z)Uuk*t*6&ojnmU>Ym8Gp(FowT*8ayAJx{N-9>$NY1%d$?f%egCQ@j z^>GKiZ2?@qg1>!Uapg}SfO~dum5!|qEwN_E_t7~gV#qni*QE>w04{O;4 zExIxDf-nn-VMNPqrQTiHfnAW=*X6+@P~N!WxOecn(yAL*E8O78^2hB6Q~C+f#W%W8(r z2}^8F7_g~uDD*@lIw#k83?_4>#cXac>#M%6u2WW%{k2zpf+lntoT& z`{mm5SFVs9$DKx18TGy|pxDXpD*Db-G*y~VE;7oo;dFJK7R+tEQ9WLOJ>l}r@sx6t zQQ~IxHlrJ&tOsC2UV099qAye32S!$sOw# zC!%{YlUugRevfU2x{L(hn8Pu$vwxc;*=Be-3o66?f3v=+N34a}CHnr*dNr0!jpS(y z^b5zl{O9>O8Jl1ae(~>r|F=sGzpL|gKQ6L!c|Ewv&L0W}ytphcE} zK9>|kt9??)=@Pg0!Hfum+v*|p}1=w!8n){G{PlwtLrWNm`M0?h(*G(>hgon;Y5*@=Z*p#7%Oy-SWY> z@`txbWg^L#rOyuyngwZxDp6k(<@gF9$a5fM|r~+j^jADdK2Fvrp+H*zgQ%6hm-L{$yVIJ^qr<= zZ{Cg#-SR|udy3IW`)o{(PfX^qs1DPcSGGv0OeJ|E&Z;NFjN=ckUnDx(p1`qITs@eO z$Jq#ZaQzaT#~BFEsQhFX8a}9LED{*6vCNZ(@K*k_Mw2fb|Jg2s)Nw$ebIRAbePBCN zC}+!3Ur?b?kf4fv;8u^%;|#*F5Z;C-;PLpy?T@%H)Cqm(#;>??RPwY*L9f^85f?(b z{NnCUm%9yDmxi2PKYW&eR|-Jug%~5wGDcAI&#tebO3tpIMU{v$t5peptyU$dh^dn5 zp`)r2vF64bXQ@ceemq5@e`cXb(8`*MB#956{#0B0vBGeZCvqq@T{OZ^L*l1u> zPch=%voBT}*vc4nL1$FOY=6OYw*S%k#nc$C>ybdTuSlR9E-Mn~!i|aqIwHyqhYRbg z42M_ChJ%M7xS*j%{pD$d+BW{PhT4gbF1^hbC@`__33Ph)XNL;ePKMfBWz;IPFMd~t z;inK72;geffE(^VCUD%CE9P~NM}8G>GqP{e**-pc3l zx7f%dd5RVob@8v(uNagO%Ey-TcJ)p;wr>2^9GUmldZvxPZRq*O(#CJ8O59u(J0}uSLHzkkca^kAzU?3Rq2M`4cw0SD0;5%B% zCLEuFMTJJOA8xU+L!EcD%x%`aqlL+t&$@TCd@MO@pP9?~j+UBoMzXb>?`W}_b?<0V ztuqo3$r*5voY@2q7*#oc{U1L1XSU4-a_*Yizq4N&81ZqR=%|I9@q5|YGPzw_3c|9N zLVjEhj@y{Qfnb{t;;$OvM@>a7>NJY_{uX7=v_tFqsWhPk@KN_^y7(K`^;SA zM{A25$<`u2Y7{whEstsYBHuHPX zMBoBH4NhM+XDb-KUp1aZT30HTNh2hobz=TDoE9cIooZQ$!>-wiObgnxS-uhTRLcUz zqT=UTPifL8vGG#N1HK_V#(&m?|MY9$`S~J)@b?dPOgW~rJ-dobub(LkztYY|f;!-P zZ$(_d>BS@;z?crftcK%`dEFU+VI7cMaIbLDy0;Bi>pKH3Ul8vExZIK-UOV0n2udDY z#18-7CovQI3{wNt^fN6rOzbl)vzXYT%xV)GzgC;rsEC=^8>@J#n%GI)s*P))c>9dB zXP2m-DUXP`x$F987VJkuYg*Z7=BN-ws2{_$e3DF?LKK%i_7SR^+`J9LXczjr+bOrI! zN1M;y26p>y6nImMF7YVd2+mg@uZk!mE9LEc~lVeu=-RVB#jA6@w`2FREOW^C{7J=SM9?x1{JOvepiRce-dl?At&C}kq`k3 zf%14sCn(4t1o82_jVFM0{7@T0W^B^_kc|Oqgs9ADyWi2;q^;sn&pw>vMD-!J+_|&% z;m5Pwwa?5g_YZ4Z?nt(_+&{Fm+&|O=O(HVDp$nQB7{F2$^ka9P`bHCB3}0oN&o8<0d{Zc7BI;wUW*)hy&M4~RtxMQSE%onM z>)d?vW33jx`LR}mNl|Y^T>H#i+{apL`sPTs7Wc7MgKy3h#<5n_Hzz?E;Lyb#;XiBQ zKK1kazitf0CBma!{RnnDi`HVMriSRwPjKjR|YK zfr&tF^YpgS;UQOkYj`NN+mmv0{r6~VmHvCYRnvby>hz-;PSiu5M*5HK!9~9{8{wg^ zovZL?J%jnZo84R5`Psl{$~c)>k4UVt9-nD78urY3e5Q47{;Xb<#SHmOYwa0Q)Mv0B zX%=F(-$7f2(TC5_qw9U#ge5SP?%kY`jdK}qjTB{w|XEdqjb$Rh-kSL3Mt7nri-zt(VO zFSO3$%!)FrZA1K8Z5v*II-{j$yM;5G7YC-YSaszsG_szVhZo8N!qPl6J)y3DW5 zwWfJ^VUE+r1&vk22i`kVR)GvEtL}dMu~%4lM6ky=xBa};iPuAvo0ydw$Hav$vMp9N``53=WmCWs3`Z3cQi`Ln%w2%KUtBSIEA`asrYo zr!tx}Fpm0rCqujTm3l(ldc1FgW>lDIPb5}p&-*qQdBn8meH-ScJ@uk22z1|u+5{5y z8MG&@;4$ra+mrWxx9`91`?K3BlqkL~D^b<&?&`?qBhzF4nHl!=LF}8jSn0B(;Z~~k zz76%L*84WpqgwCVP_0_;+n^dABupX?^6Q2N8~Ko;Itb2OI)1;=&dhD=?C#jywfn#z zGdxb#q68AFC~?x75+~i$8f3hy& zpEMEvNlo}9Hr^}}{y6_x6aLv#C;tz-ngaTu+tITt9|;yBd_m~(vhXXsN&K!(tN$50 z&4L~$-c|KrS=a5t)2ZUASC0pqGw=edgq`A88p4Wj8`g!Kc*`~5#`doO77KlCAwH2l zzQN4v5cL~1^FM7^)u~;&VK+Id{P>2sIUkR2V9v)~jP{RjFsSkCP1JZlP9hVu+6U&k z`tc1lbv2SJbu};8%I$buBLInxe?|G3yNA?zO8KAsKk$YduN46l?4 zQ0aEOvcb&lcy$Bk5%Z9cK$o#_&*7B~Ri4A^8>&5rSG2BDBdJd)LRWPA_|IBb|KX*_ zUNCY!+2;s#wfiQ#QP$PB%Uv~_`;b_r=-#$gbZ^g5(dk85D7v?2QFNj{gQ7F{9OhL_ zdicz!;Ydb=Bid%A;oh!G!@XUXhI`vY!@aFZo#e%vMN*IRpEaqU_`K4tH#*maBDGI&HATW~`GI2riE9EuN^9^}>5a45tl)&2s0 zt@al_fF2@$p?Y7by7;BspUEpX+IVc$=Y#SPn47Ple`Z1V4>ru=BB38Ooe79ZyL(9K z5$}s?FcBAcT0L`zfl@uAJ7W8@`~AcG62T)IiJ6R2M>cBA{Gb2&@Xy%B_#+$5H0ftI zE~Yl}a10W*CMHI=#>Pe@LktGhR_-_d{6eX^P@fDR{qbq~G&6DS5{-dD#uJS6VJ3-Q>j=68DBe(Vm#qw#Kp#M$sXzLCrnUTuxx zDL$zlp7^zTcpl&QF^1>yjh4exPnz5C)IYNrp2s)NGCa|bn!^*~GasJEH4YLNILP%3 zPyGbUUyBnkbeM%sz>s6dCjV0#=}a4Cpvq4H>PXwEjp{+hoVqt~A-=|Qw|uYmsf`Pj(~5;`!d6@@-jNU&cd|P>c?+~l zl^ zmir7|9>eAcaZF5k+(EYQ!&7G313h@dLb=A>$tzXQ>&0P40J%-$;c4-oj*#D(Nio~B z3<6$ym=Mp+n{+ZgVg8At_-w^L(bk?E_IA8nu_a~4R(X40`LXovL%dCflcNn`b|UB0 zMzy`FCh5D?;df!V7@OPRyZGH$9~N61aR%1u4Z2NM4vh)o4mD5;9oHt&*SN;STUxKYSn_K2feO$2B8zN`3pJgj>Dv0HL^R)R7NQ;r=)A&iYhbywE0|1#Q#rGRO0P zF+6&b5_j*a4_9cTPrb_ouT+5OqP2LTBBs2G0*J(GWy-@ZN}II@Snv*vIKLvg_os_F zh-sq-ya-uwx_j=qjIC)_h&1K1Jqxp!fTy!b# zaZaSu6A2kFcV&fZ0cD~Hxi{n(OegjZkL>r3BnN}hSjo{pIn$jSizN^E`nM$a_6_e& zkM3}H2Bj^IVrW=$=kr^8#|OInQ)45kE~uq#qiJt67;FDz4qHGRQOA@c>Y8!6)RRsQ z+|Qq74q~~5n)0Z+6s(-CJD2Wi*i;?yM?G)HE2*s{Fd~ggLZAH3Y=n-W;Z8b&onfSU z&NCfB8F;#J@>{AM|5QXqo{+oE zhnWPwz^;_A+5~6O(+0;8M=;$!Dd53(I3(d^=H~fXrit^F4B#D82X+oB#CGub=a%c_ zcutUmV;OTXd<8^hOF(b$8mgGQ+B{^QNCP(ksbjOzbP$1ZOU`J1+ z7%hZ*qT^GU(aw?S{!nnsbgyq{OJC5@9_? zRag*Av%GE~NB^yEtTN&z3^*OwVCDrs#3ovth{Z`;K@aS?afbqJZkJ1Jqz5ywLqiJs zy*_NC6~BlbX?`4sb-8$?;PqL!0mO~kfw!G=<4a#`)VcAR;0B4AxRIF52io^<-k<7b z#>Kmg80YC z5DZce-?aGdEbIX0oE^7)>d!CkJ^#o@@AFP@l05z9LY*YPRul*kqI{KdqWC~@G)pG{MmWu^~SwV*_pUY3sMqGQd3-j1Y@;RWo+W4e-juuVe$;> zj4ZjSC5fyBiRr1u3>pv~OI~7bDr32DNNQrPLU?{rc5y*sa;kz$YHmI|+-L?yBx{&5 zQy9y+O!I;*oq`LD11c2Z%9vP7GD~t&kri_#7o`H7gXE+&8yM6PI#@~)(~B828QH+r z79$yp#4KZ20rLb4W4T;-QD#Z1j)F^3V!4h&Ku%(wLSjyiLSl)6e_nE`9wSUCBMTz{ D-uQP( literal 0 HcmV?d00001 diff --git a/packages/backend/server/src/__tests__/mocks/doc-snapshot.mock.ts b/packages/backend/server/src/__tests__/mocks/doc-snapshot.mock.ts new file mode 100644 index 0000000000..16a1bbb1ad --- /dev/null +++ b/packages/backend/server/src/__tests__/mocks/doc-snapshot.mock.ts @@ -0,0 +1,42 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +import { faker } from '@faker-js/faker'; +import type { Snapshot } from '@prisma/client'; + +import { Mocker } from './factory'; + +export type MockDocSnapshotInput = { + user: { id: string }; + workspaceId: string; + docId?: string; + blob?: Uint8Array; +}; + +export type MockedDocSnapshot = Snapshot; + +export class MockDocSnapshot extends Mocker< + MockDocSnapshotInput, + MockedDocSnapshot +> { + override async create(input: MockDocSnapshotInput) { + if (!input.blob) { + const snapshot = await readFile( + path.join(import.meta.dirname, '../__fixtures__/test-doc.snapshot.bin') + ); + input.blob = snapshot; + } + const snapshot = await this.db.snapshot.create({ + data: { + id: input.docId ?? faker.string.nanoid(), + workspaceId: input.workspaceId, + blob: input.blob, + createdAt: new Date(), + updatedAt: new Date(), + createdBy: input.user.id, + updatedBy: input.user.id, + }, + }); + return snapshot; + } +} diff --git a/packages/backend/server/src/__tests__/mocks/index.ts b/packages/backend/server/src/__tests__/mocks/index.ts index 50e82b334a..b6abf141c8 100644 --- a/packages/backend/server/src/__tests__/mocks/index.ts +++ b/packages/backend/server/src/__tests__/mocks/index.ts @@ -6,6 +6,7 @@ export * from './workspace-user.mock'; import { MockCopilotProvider } from './copilot.mock'; import { MockDocMeta } from './doc-meta.mock'; +import { MockDocSnapshot } from './doc-snapshot.mock'; import { MockEventBus } from './eventbus.mock'; import { MockMailer } from './mailer.mock'; import { MockJobQueue } from './queue.mock'; @@ -22,6 +23,7 @@ export const Mockers = { WorkspaceUser: MockWorkspaceUser, UserSettings: MockUserSettings, DocMeta: MockDocMeta, + DocSnapshot: MockDocSnapshot, }; export { MockCopilotProvider, MockEventBus, MockJobQueue, MockMailer }; diff --git a/packages/backend/server/src/__tests__/mocks/queue.mock.ts b/packages/backend/server/src/__tests__/mocks/queue.mock.ts index 25a3ceff4f..d44a9b6321 100644 --- a/packages/backend/server/src/__tests__/mocks/queue.mock.ts +++ b/packages/backend/server/src/__tests__/mocks/queue.mock.ts @@ -5,6 +5,7 @@ import { JobQueue } from '../../base'; export class MockJobQueue { add = Sinon.createStubInstance(JobQueue).add.resolves(); + remove = Sinon.createStubInstance(JobQueue).remove.resolves(); last(name: Job): { name: Job; payload: Jobs[Job] } { const addJobName = this.add.lastCall?.args[0]; diff --git a/packages/backend/server/src/__tests__/mocks/workspace.mock.ts b/packages/backend/server/src/__tests__/mocks/workspace.mock.ts index bfc1fd9446..5a95b78604 100644 --- a/packages/backend/server/src/__tests__/mocks/workspace.mock.ts +++ b/packages/backend/server/src/__tests__/mocks/workspace.mock.ts @@ -1,3 +1,6 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + import { faker } from '@faker-js/faker'; import type { Prisma, Workspace } from '@prisma/client'; import { omit } from 'lodash-es'; @@ -7,6 +10,7 @@ import { Mocker } from './factory'; export type MockWorkspaceInput = Prisma.WorkspaceCreateInput & { owner?: { id: string }; + snapshot?: Uint8Array | true; }; export type MockedWorkspace = Workspace; @@ -14,8 +18,18 @@ export type MockedWorkspace = Workspace; export class MockWorkspace extends Mocker { override async create(input?: Partial) { const owner = input?.owner; - input = omit(input, 'owner'); - return await this.db.workspace.create({ + if (input?.snapshot === true) { + const snapshot = await readFile( + path.join( + import.meta.dirname, + '../__fixtures__/test-root-doc.snapshot.bin' + ) + ); + input.snapshot = snapshot; + } + const snapshot = input?.snapshot; + input = omit(input, 'owner', 'snapshot'); + const workspace = await this.db.workspace.create({ data: { name: faker.animal.cat(), public: false, @@ -31,5 +45,21 @@ export class MockWorkspace extends Mocker { : undefined, }, }); + + // create a rootDoc snapshot + if (snapshot) { + await this.db.snapshot.create({ + data: { + id: workspace.id, + workspaceId: workspace.id, + blob: snapshot, + createdAt: new Date(), + updatedAt: new Date(), + createdBy: owner?.id, + updatedBy: owner?.id, + }, + }); + } + return workspace; } } diff --git a/packages/backend/server/src/base/job/queue/config.ts b/packages/backend/server/src/base/job/queue/config.ts index 0f32961fce..5aeb0f5de7 100644 --- a/packages/backend/server/src/base/job/queue/config.ts +++ b/packages/backend/server/src/base/job/queue/config.ts @@ -61,6 +61,14 @@ defineModuleConfig('job', { schema, }, + 'queues.indexer': { + desc: 'The config for indexer job queue', + default: { + concurrency: 1, + }, + schema, + }, + 'queues.notification': { desc: 'The config for notification job queue', default: { diff --git a/packages/backend/server/src/base/job/queue/def.ts b/packages/backend/server/src/base/job/queue/def.ts index d1f2aa60eb..50e18eb8ea 100644 --- a/packages/backend/server/src/base/job/queue/def.ts +++ b/packages/backend/server/src/base/job/queue/def.ts @@ -27,6 +27,7 @@ export enum Queue { NOTIFICATION = 'notification', DOC = 'doc', COPILOT = 'copilot', + INDEXER = 'indexer', } export const QUEUES = Object.values(Queue); diff --git a/packages/backend/server/src/base/job/queue/executor.ts b/packages/backend/server/src/base/job/queue/executor.ts index 93689bab20..71e8e2e69a 100644 --- a/packages/backend/server/src/base/job/queue/executor.ts +++ b/packages/backend/server/src/base/job/queue/executor.ts @@ -27,11 +27,15 @@ export class JobExecutor implements OnModuleDestroy { @OnEvent('config.init') async onConfigInit() { - const queues = env.flavors.graphql ? difference(QUEUES, [Queue.DOC]) : []; + const queues = env.flavors.graphql + ? difference(QUEUES, [Queue.DOC, Queue.INDEXER]) + : []; // NOTE(@forehalo): only enable doc queue in doc service if (env.flavors.doc) { queues.push(Queue.DOC); + // NOTE(@fengmk2): Once the index task cannot be processed in time, it needs to be separated from the doc service and deployed independently. + queues.push(Queue.INDEXER); } await this.startWorkers(queues); diff --git a/packages/backend/server/src/core/doc/event.ts b/packages/backend/server/src/core/doc/event.ts index b61c030d32..d148cf5ff1 100644 --- a/packages/backend/server/src/core/doc/event.ts +++ b/packages/backend/server/src/core/doc/event.ts @@ -1,14 +1,17 @@ import { Injectable } from '@nestjs/common'; -import { OnEvent } from '../../base'; +import { JobQueue, OnEvent } from '../../base'; import { Models } from '../../models'; +import { PgWorkspaceDocStorageAdapter } from './adapters/workspace'; import { DocReader } from './reader'; @Injectable() export class DocEventsListener { constructor( private readonly docReader: DocReader, - private readonly models: Models + private readonly models: Models, + private readonly workspace: PgWorkspaceDocStorageAdapter, + private readonly queue: JobQueue ) {} @OnEvent('doc.snapshot.updated') @@ -26,6 +29,17 @@ export class DocEventsListener { return; } await this.models.doc.upsertMeta(workspaceId, docId, content); + await this.queue.add( + 'indexer.indexDoc', + { + workspaceId, + docId, + }, + { + jobId: `${workspaceId}/${docId}`, + priority: 100, + } + ); } else { // update workspace content to database const content = this.docReader.parseWorkspaceContent(blob); @@ -33,6 +47,33 @@ export class DocEventsListener { return; } await this.models.workspace.update(workspaceId, content); + await this.queue.add( + 'indexer.indexWorkspace', + { + workspaceId, + }, + { + jobId: workspaceId, + priority: 100, + } + ); + } + } + + @OnEvent('user.deleted') + async clearUserWorkspaces(payload: Events['user.deleted']) { + for (const workspace of payload.ownedWorkspaces) { + await this.workspace.deleteSpace(workspace); + await this.queue.add( + 'indexer.deleteWorkspace', + { + workspaceId: workspace, + }, + { + jobId: workspace, + priority: 0, + } + ); } } } diff --git a/packages/backend/server/src/core/doc/job.ts b/packages/backend/server/src/core/doc/job.ts index dc1cde21f7..1d0a5c184c 100644 --- a/packages/backend/server/src/core/doc/job.ts +++ b/packages/backend/server/src/core/doc/job.ts @@ -1,9 +1,8 @@ import { Injectable } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; -import { JobQueue, OnEvent, OnJob } from '../../base'; +import { JobQueue, OnJob } from '../../base'; import { Models } from '../../models'; -import { PgWorkspaceDocStorageAdapter } from './adapters/workspace'; declare global { interface Jobs { @@ -15,7 +14,6 @@ declare global { export class DocStorageCronJob { constructor( private readonly models: Models, - private readonly workspace: PgWorkspaceDocStorageAdapter, private readonly queue: JobQueue ) {} @@ -34,11 +32,4 @@ export class DocStorageCronJob { async cleanExpiredHistories() { await this.models.history.cleanExpired(); } - - @OnEvent('user.deleted') - async clearUserWorkspaces(payload: Events['user.deleted']) { - for (const workspace of payload.ownedWorkspaces) { - await this.workspace.deleteSpace(workspace); - } - } } diff --git a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md new file mode 100644 index 0000000000..a263260663 --- /dev/null +++ b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md @@ -0,0 +1,644 @@ +# Snapshot report for `src/core/utils/__tests__/blocksute.spec.ts` + +The actual snapshot is saved in `blocksute.spec.ts.snap`. + +Generated by [AVA](https://avajs.dev). + +## can read all doc ids from workspace snapshot + +> Snapshot 1 + + [ + '5nS9BSp3Px', + ] + +## can read all blocks from doc snapshot + +> Snapshot 1 + + { + blocks: [ + { + additional: { + displayMode: 'edgeless', + noteBlockId: undefined, + }, + blockId: 'TnUgtVg7Eu', + content: 'Write, Draw, Plan all at Once.', + docId: 'doc-0', + flavour: 'affine:page', + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'RX4CG2zsBk', + }, + blockId: 'FoPQcAyV_m', + content: 'AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'RX4CG2zsBk', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'RX4CG2zsBk', + }, + blockId: 'oz48nn_zp8', + content: '', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'RX4CG2zsBk', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'RX4CG2zsBk', + }, + blockId: 'g8a-D9-jXS', + content: 'You own your data, with no compromises', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'RX4CG2zsBk', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'RX4CG2zsBk', + }, + blockId: 'J8lHN1GR_5', + content: 'Local-first & Real-time collaborative', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'RX4CG2zsBk', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'RX4CG2zsBk', + }, + blockId: 'xCuWdM0VLz', + content: 'We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'RX4CG2zsBk', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'RX4CG2zsBk', + }, + blockId: 'zElMi0tViK', + content: 'AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'RX4CG2zsBk', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'RX4CG2zsBk', + }, + blockId: 'Z4rK0OF9Wk', + content: '', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'RX4CG2zsBk', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'S1mkc8zUoU', + }, + blockId: 'DQ0Ryb-SpW', + content: 'Blocks that assemble your next docs, tasks kanban or whiteboard', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'S1mkc8zUoU', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'yGlBdshAqN', + }, + blockId: 'HAZC3URZp_', + content: 'There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'yGlBdshAqN', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'yGlBdshAqN', + }, + blockId: '0H87ypiuv8', + content: 'We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'yGlBdshAqN', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'yGlBdshAqN', + }, + blockId: 'Sp4G1KD0Wn', + content: 'If you want to learn more about the product design of AFFiNE, here goes the concepts:', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'yGlBdshAqN', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'yGlBdshAqN', + }, + blockId: 'RsUhDuEqXa', + content: 'To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'yGlBdshAqN', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: '6lDiuDqZGL', + }, + blockId: 'Z2HibKzAr-', + content: 'A true canvas for blocks in any form', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: '6lDiuDqZGL', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: '6lDiuDqZGL', + }, + blockId: 'UwvWddamzM', + content: 'Many editor apps claimed to be a canvas for productivity. Since the Mother of All Demos, Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers. ', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: '6lDiuDqZGL', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: '6lDiuDqZGL', + }, + blockId: 'g9xKUjhJj1', + content: '', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: '6lDiuDqZGL', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: '6lDiuDqZGL', + }, + blockId: 'wDTn4YJ4pm', + content: '"We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: '6lDiuDqZGL', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: '6lDiuDqZGL', + }, + blockId: 'xFrrdiP3-V', + content: 'Quip & Notion with their great concept of "everything is a block"', + docId: 'doc-0', + flavour: 'affine:list', + parentBlockId: '6lDiuDqZGL', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: '6lDiuDqZGL', + }, + blockId: 'Tp9xyN4Okl', + content: 'Trello with their Kanban', + docId: 'doc-0', + flavour: 'affine:list', + parentBlockId: '6lDiuDqZGL', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: '6lDiuDqZGL', + }, + blockId: 'K_4hUzKZFQ', + content: 'Airtable & Miro with their no-code programable datasheets', + docId: 'doc-0', + flavour: 'affine:list', + parentBlockId: '6lDiuDqZGL', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: '6lDiuDqZGL', + }, + blockId: 'QwMzON2s7x', + content: 'Miro & Whimiscal with their edgeless visual whiteboard', + docId: 'doc-0', + flavour: 'affine:list', + parentBlockId: '6lDiuDqZGL', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: '6lDiuDqZGL', + }, + blockId: 'FFVmit6u1T', + content: 'Remnote & Capacities with their object-based tag system', + docId: 'doc-0', + flavour: 'affine:list', + parentBlockId: '6lDiuDqZGL', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'cauvaHOQmh', + }, + blockId: 'YqnG5O6AE6', + content: 'For more details, please refer to our RoadMap', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'cauvaHOQmh', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'cauvaHOQmh', + }, + blockId: 'sbDTmZMZcq', + content: 'Self Host', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'cauvaHOQmh', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'cauvaHOQmh', + }, + blockId: 'QVvitesfbj', + content: 'Self host AFFiNE', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'cauvaHOQmh', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: 'Learning From', + displayMode: 'page', + noteBlockId: '2jwCeO8Yot', + }, + blockId: 'U_GoHFD9At', + content: [ + 'Learning From', + 'Title', + 'Tag', + 'Reference', + 'Developers', + 'AFFiNE', + ], + docId: 'doc-0', + flavour: 'affine:database', + }, + { + additional: { + databaseName: 'Learning From', + displayMode: 'page', + noteBlockId: '2jwCeO8Yot', + }, + blockId: 'tpyOZbPc1P', + content: 'Affine Development', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'U_GoHFD9At', + parentFlavour: 'affine:database', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: 'Learning From', + displayMode: 'page', + noteBlockId: '2jwCeO8Yot', + }, + blockId: 'VMx9lHw3TR', + content: 'For developers or installations guides, please go to AFFiNE Doc', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'U_GoHFD9At', + parentFlavour: 'affine:database', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: 'Learning From', + displayMode: 'page', + noteBlockId: '2jwCeO8Yot', + }, + blockId: 'Q6LnVyKoGS', + content: 'Quip & Notion with their great concept of "everything is a block"', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'U_GoHFD9At', + parentFlavour: 'affine:database', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: 'Learning From', + displayMode: 'page', + noteBlockId: '2jwCeO8Yot', + }, + blockId: 'EkFHpB-mJi', + content: 'Trello with their Kanban', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'U_GoHFD9At', + parentFlavour: 'affine:database', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: 'Learning From', + displayMode: 'page', + noteBlockId: '2jwCeO8Yot', + }, + blockId: '3aMlphe2lp', + content: 'Airtable & Miro with their no-code programable datasheets', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'U_GoHFD9At', + parentFlavour: 'affine:database', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: 'Learning From', + displayMode: 'page', + noteBlockId: '2jwCeO8Yot', + }, + blockId: 'MiZtUig-fL', + content: 'Miro & Whimiscal with their edgeless visual whiteboard', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'U_GoHFD9At', + parentFlavour: 'affine:database', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: 'Learning From', + displayMode: 'page', + noteBlockId: '2jwCeO8Yot', + }, + blockId: 'erYE2C7cc5', + content: 'Remnote & Capacities with their object-based tag system', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'U_GoHFD9At', + parentFlavour: 'affine:database', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'c9MF_JiRgx', + }, + blockId: 'NyHXrMX3R1', + content: 'Affine Development', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'c9MF_JiRgx', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'c9MF_JiRgx', + }, + blockId: '9-K49otbCv', + content: 'For developer or installation guides, please go to AFFiNE Development', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'c9MF_JiRgx', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + databaseName: undefined, + displayMode: 'page', + noteBlockId: 'c9MF_JiRgx', + }, + blockId: 'faFteK9eG-', + content: '', + docId: 'doc-0', + flavour: 'affine:paragraph', + parentBlockId: 'c9MF_JiRgx', + parentFlavour: 'affine:note', + ref: [], + refDocId: [], + }, + { + additional: { + displayMode: 'edgeless', + noteBlockId: undefined, + }, + blockId: '6x7ALjUDjj', + content: [ + 'What is AFFiNE', + 'Related Articles', + ' ', + 'Self-host', + '', + 'AFFiNE ', + 'Development', + 'You can check these URLs to learn about AFFiNE', + 'Database Reference', + ], + docId: 'doc-0', + flavour: 'affine:surface', + parentBlockId: 'TnUgtVg7Eu', + parentFlavour: 'affine:page', + }, + { + additional: { + displayMode: 'edgeless', + noteBlockId: undefined, + }, + blockId: 'ECrtbvW6xx', + docId: 'doc-0', + flavour: 'affine:bookmark', + }, + { + additional: { + displayMode: 'edgeless', + noteBlockId: undefined, + }, + blockId: '5W--UQLN11', + docId: 'doc-0', + flavour: 'affine:bookmark', + }, + { + additional: { + displayMode: 'edgeless', + noteBlockId: undefined, + }, + blob: [ + 'BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA=', + ], + blockId: 'lcZphIJe63', + docId: 'doc-0', + flavour: 'affine:image', + parentBlockId: '6x7ALjUDjj', + parentFlavour: 'affine:surface', + }, + { + additional: { + displayMode: 'edgeless', + noteBlockId: undefined, + }, + blob: [ + 'HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8=', + ], + blockId: 'JlgVJdWU12', + docId: 'doc-0', + flavour: 'affine:image', + parentBlockId: '6x7ALjUDjj', + parentFlavour: 'affine:surface', + }, + { + additional: { + displayMode: 'edgeless', + noteBlockId: undefined, + }, + blob: [ + 'ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0=', + ], + blockId: 'lht7AqBqnF', + docId: 'doc-0', + flavour: 'affine:image', + parentBlockId: '6x7ALjUDjj', + parentFlavour: 'affine:surface', + }, + ], + summary: 'AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. You own your data, with no compromisesLocal-first & Real-time collaborativeWe love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.Blocks that assemble your next docs, tasks kanban or whiteboardThere is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ', + title: 'Write, Draw, Plan all at Once.', + } diff --git a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap new file mode 100644 index 0000000000000000000000000000000000000000..06ec55a74f57afad4eca109cc10010b69fcf7a6a GIT binary patch literal 6255 zcmV-#7?9^dRzVKC<&&X#sZuVy{->} zwxycxx4Ub)x~iwDdPXDG8^DkN3;05IYz)Q_Ls)Z}vl3*{!99crLS-gqaYz`+e ztYfg(VB-DoVX5kw(ahVeYNzE$7LoqZGgWW>eoy_L=e?_{ABl|WW+-mI>l4D!99^6# z4ylG>2{o*!x~_!HkfMccC1RNgWzw|bc2W%q#WvKW9Wxzm#<1V{NI0xI>hbP{6yp91 zEDkIv1}cEnf#7w4f_lqRrviZ)YsHEc1eOp85V)4rfd#AyEF@5HU0_L-(O=ilpWJu! z%#2ez<~f!2IPH_`;03)oG5H&F+*wo&90Ue{_bi%rE7&_aCPI$u*R;8ES#t(v8QZDB$SA0d8M70P)1Xl9(K=7-?$l3rp%P3I6_S*lUiIWR^o;^ zsf%z_C<)b2qaq;;M^TNi64s)cqw0!k+rqKEL#i?=97nkLj<5{X(I$lAm`ac7Xr|#6 z>DDZ>R5@X$6m!x@leu~pE0dZNQw&oHnTe!jCNx{vU1mttOCp+OJ4%t#Cse(}(Go%l znYylynwEz@EEL^zX<|ar!a`M&mYFnd5mrX0ly)Po6e<0aniGn-^0;}Xj@wFHHAdYW zwphJ{z7LZ;zZ+Gxr# zE!$1`P|9|_I4eq(VWCW_hT|Ufuxv9SoS5sECxLsVsu}idrHlyGNm;^HCZ|)|jd#@a zazZG!Ba%uaorR@})L;~NjoQp#TZS#o5yeWArA`(rM|IWkI8;aJb@Rn-x!(5v0=TOH z9xQ-I3*d_d@Rb7iMgjb30sNo<{;>dFDS%%Tz~UgR4#Jio><&U@5Do{SBM5^*I30v| z5GI1~&LDgs2=@o!6G8Y~5dL!z{%a7v8-%|I!t+7+aS&b)LO~&{EQA{hp{Nkb3*lfP zv=%~N;rzyaf!x^dA+R(MxYcWuUF*Yzcs;<$I?wEM%~hUxk87^>%)fBWHJ+&~gTPwPjJoDJ&-}b=uJ_EJ zy5;AGXpCiu*oyqTywK$-szfKJoEdmxz#f_uY`c&nZh-1 z^vut>=C(AkYi{?Xd)LK*CAY7JJ66NPtKnO#;qf)_=WAfmTBuwLVlA9qE5Q_G zz)>7M;P=+SH`hrp*JQx_?K*g29V}W8rR%|25BIN! zXV$}O>tWXhIJ7~6S(X7aya8_A08<;_lN;cL4X|`09M}l)jquP$czmM-b6p0^cQ(SG zZiIi@2nN%ukEq{}sdf5~weM(IzUR;O+`|ssf&^kYL&~U|z0(*W6 zV7fD4oJu%b3HMdP*M9Z?x+fn8K@CSA9!#cQrKkVHv!Q7ew)4m^i_d|R?oZAmi?}r!m!-fOUb^xXhz=sY< zFscj_{M`fa`2+Cn1MunrSauM~4oa7a4E9q8K^%m49)!mZ!vAt7o`<095S%^)=MKSt zJS4%yGGM-b2)=y?{^1a;ycx`!;hvk}shi=In_=f+ICxls(KB1S!*I)CICB_2aTpdJ zfubWYbOe6$2z=!TeD8<^lgxnm(GmE`5xBM^K!XJ1WWWqI zz{v)fY=B1^;PnRB+z8!`aC;+ssS%!Olwi(e!2C@k{9_|5Y=V*|xT6U^)db&ff?zWo zXojQB63oBLfEjCsWHWrA8NS>M%UYnU1x~lXT`ll*3;bove3%9AAhQy+!0Rort`&~7 zg4GJ=TH)zd_-QK?wZVZl3FqCJID>6)vJEEN;E^^6w!`jrIMEL8YlkP=;ZNEnocCtp z{IDH<+zv}SpsE7~I$*K`KHUM&cEI9J*xV`Md>{j7aZM*2>Vy-Wa7QOR)Cqso3IF6? z-z8mtSH|@PJzX%+1<5XWunWE_UvqEfH7|C-D_yX>8*01ZWVdw9xlBLb-wk(l!)LqU zscv|&8&>qdfgXtWNY~$=dHusZ@MsS_*#pn^K%f_H>V>1d@Qz;T`Uf+w|NUNgq8Gm3 z3orJ%{-HEF(oHsn%oIg_yo1yb8?q4NkGC@RFC(M*J^Y>CU644B?KdDBAcVE|d z_w{Oi_tjz^Jr=5;8akEmzpn+)@N2T@cJUEdIU4~>jaq6lCY4{7R4rjR4U*J-$159K zE6&;t@$BVWvd7sl9pOcI`?2#8UTBGkcku9>gNrQ@X>#voPFkEimRDMsXDe$BNfvX!9GaD9H zY-asYw#<4iM`pd2UuK}VK*pAhY_vDvc zCkL`5*JO_5x;MY%Dr>8)nM!J@i5xSy1wnTx=H|j|=1g~>j;V~gNf1dHVbz^_tNOe= zN$RQ-F|EV}A}Y`C-Gn*cY{)bE3C&Ka`uyIvSc#d*>7kwCKA>dB)a2Y&$Q`IfbaOJ@ zho`2SE4O4qjZYWUwTPHq8gSPMblV-U8HNZsis>el>yc?*zV7&9v+AjAS@l;rvg-Bx zvZ_B>*;?M&R5onn=s*VBz2%(QMnD&;Whn0OdwS&VrbyBN$tj@PxQ;kISz)GJoJ z=FY4B-Wc9ALL%wd`>)s-?HX7^#DK#i-97j5Vd2$B(NyYX0XLFqM9{P8BPL z>4ixRt4XJH&IHM=#g12ZH6yG|XyKHq+e(qQN@pvRF;fYtb7QK_E?dM*#S)Q}EtCmi zgiXsGC&x=Pqu8s4?&*0Qgx8mty4SHy0}0y}hNG#vTR(OC%B`VyWsCl2b434}`9*)E zqD>p^JX>#-6Th^r>ax#5qVMg(n(&eH)*M+@sZZ9{3L%Z zo8+s9{3YKzl$Yd#lM}64F&IA;Pnr?VM;#w~pF` z=1i3;{hARO-NhD0WSZOj-QC+o_W+N)}QPpyaT`)_e*TLNym$b~N zr6yE&AV0HyetxARrrOG=J4{rSWXiHrTDn_S4aJ?$CR9f#3C*_MIkD?oR2U-NNqYXX6>liD4tIlgLR?VlgsrffK)ci$$ zYECu{7?meFDw8>Ou>#h%VKF_7^!A$Ee#J|4w@L{$;*Ju}pGX%>*?;?vkCrO+if%e? zvL`ju5W=$4{g!6fNzGjbN~PDEycv!=`I#zKL}|42@{L(8Hh{&$SpwKL>>t4Ja9#mC z(_&d+?dZOep&S9MKbF#xO3}==o;UDMj~1eCV>dHYbmO#5ObBZ#J?Zpj&K~KuH*W`4 z*KFs~gyP<8q4+|MP<$u9Pz)sN&P?@G_Qo?xN}<>=Uxy=FOHhh5}tUeA@=0i1 z5U9tqgy8lZA$TCa5FDHAKHJ+6lb3C<6H-$XYFqEI#0ARd(*C||+J7mB_TSAUe$M#pmKy3*4%X4=m5O4k)D zZ+Ik&yzjW6yy-_EAIeYO!BeefTT4@2y^|hZ&Owc#UD zd8qqLo!&ONZ=f&N#+T2pI%LhrnIm)-^oNxw}+Wtk8UzUS9C-k z%odR^=ZMIkzTv1!i@Sd#w2h{mY8h5#Xlx>x8xO*=6Fk6vY{l= zp!t^R>9@(YBpPBd3E)`UX29*Mquv+m0niH~KA#8+}e z;(z8BiJqyp<5u_aeSJB$f8J7q(okJ@%c%~nFDfsZy}`-K;@h$ayyG^1fg5hiOW?Ya z&dNH|8Eu@%A#n4&0jWG5^&gJ@8pOoP7Lz-3#N@O2#U!G(IHI#owB~p;;qvWyUaXow z$fo8$Ur^1$^e)4xywt2dQ&ZnHHrO;amaZqK{xzoK%ct+qPFWE(B)$GwsZoh$xgX66ADWdi&8J7Rs$uu{GVNV#k!;lI~G+X>= zw))U0GR-H$@R>0DaTs2eX}0;%Z1X&w>qX#w9B z@QO^c$B*W1J~YKLjS_*P2((9FJR;GQ`qAw5p(>9*)4H5%^{Vej?LU_|fe1p{bN< zwnX8kD6~ewj7l`sel#^cG_^9#N2Bn`D10LdFUd6f{b&yO&>WO$HpO6D44Pw*h)Faz z`_UZsp*bScd?W^s#Ng{Ocu}Tl@S|z;p=pw7uGc`(pizUkCegI`(X{%|w8=CNXz;KG zPigQYnWn>!rqhR}OQzW{23y9UehjoRiKfSorq_q&s7!PJ7<_aLzBUFg$TWR^H2pp_ z12WCJIBbf;;W$L&63vhw&9D#6ahc{^96l0o`O%#9p?SMZQ!oxo$KlX8h;fPLc0Zbb??dw&85L(79vFx3jKlwxXcoLv{&3b@ ztOf7l!bA)1KxVqpf}Iw0SYTNa(|dfGe$$WXw=$SMZo#K4_?87P%S`X{WqQ9K(|^cd z+G@iN8`^A0+7i~Hl&~_B{6uq~L46D*vKf_O~<2 zE?7PRYbNG<`ap)K$0lHS0`8cQ5I>TM_~-;YCLunQf%w7%yfgvVPfCa%%S1dl2@R9; z5g(q9xXk-j_Pk|;rD4rZ>grUt8J_*P_~MyuNx7`MCOmh6qHx*FZywAmckRp;RF9ev zvjua~E=p#uieymAU#GZAf0h0!{UxHG`yrCcKtK1D<*W4nw&>@+hj5ktD*aXZtMr$E ze(t*rSLt)oyZe$|?pJZ#r>R>kQ+{~vqP(dXn`{)lwI@u+`)Qt_{1b38vv)QSm^ocA z;QoNp%;8z8g#&7I=HSAagV*%A?}52rvYa`!Xx5FU*$4e+&K1m@3(mZG^VSmUsyA1? zx$4bj@a9c!jpsbb_ivv5O<%`a;ynmDzeia)#Sd#eNnR{=Dh$rzi%OsK{Rhfw9bOTb*?w@gxS?evzAseK+?On|b**or zzNY)wspP(%a6H`DTlx~gmbtdfwv$3{zytz9ewBwz}$EPME_2pLA#IV_= ZR$0mBSlyYr-m$Vn{~zHBGqYcK002xAaHjwO literal 0 HcmV?d00001 diff --git a/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts b/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts new file mode 100644 index 0000000000..f760d51acc --- /dev/null +++ b/packages/backend/server/src/core/utils/__tests__/blocksute.spec.ts @@ -0,0 +1,57 @@ +import test from 'ava'; +import { omit } from 'lodash-es'; + +import { createModule } from '../../../__tests__/create-module'; +import { Mockers } from '../../../__tests__/mocks'; +import { Models } from '../../../models'; +import { + readAllBlocksFromDocSnapshot, + readAllDocIdsFromWorkspaceSnapshot, +} from '../blocksuite'; + +const module = await createModule({}); +const models = module.get(Models); + +const owner = await module.create(Mockers.User); +const workspace = await module.create(Mockers.Workspace, { + snapshot: true, + owner, +}); + +const docSnapshot = await module.create(Mockers.DocSnapshot, { + workspaceId: workspace.id, + user: owner, +}); + +test.after.always(async () => { + await module.close(); +}); + +test('can read all doc ids from workspace snapshot', async t => { + const rootDoc = await models.doc.get(workspace.id, workspace.id); + t.truthy(rootDoc); + + const docIds = readAllDocIdsFromWorkspaceSnapshot(rootDoc!.blob); + + t.deepEqual(docIds, ['5nS9BSp3Px']); + t.snapshot(docIds); +}); + +test('can read all blocks from doc snapshot', async t => { + const rootDoc = await models.doc.get(workspace.id, workspace.id); + t.truthy(rootDoc); + const doc = await models.doc.get(workspace.id, docSnapshot.id); + t.truthy(doc); + + const result = await readAllBlocksFromDocSnapshot( + workspace.id, + rootDoc!.blob, + 'doc-0', + docSnapshot.blob + ); + + t.snapshot({ + ...result, + blocks: result!.blocks.map(block => omit(block, ['yblock'])), + }); +}); diff --git a/packages/backend/server/src/core/utils/blocksuite.ts b/packages/backend/server/src/core/utils/blocksuite.ts index 0317bf9232..22ee899023 100644 --- a/packages/backend/server/src/core/utils/blocksuite.ts +++ b/packages/backend/server/src/core/utils/blocksuite.ts @@ -1,12 +1,17 @@ // TODO(@forehalo): // Because of the `@affine/server` package can't import directly from workspace packages, -// this is a temprory solution to get the block suite data(title, description) from given yjs binary or yjs doc. +// this is a temporary solution to get the block suite data(title, description) from given yjs binary or yjs doc. // The logic is mainly copied from // - packages/frontend/core/src/modules/docs-search/worker/in-worker.ts // - packages/frontend/core/src/components/page-list/use-block-suite-page-preview.ts // and it's better to be provided by blocksuite -import { Array, Doc, Map } from 'yjs'; +// eslint-disable-next-line @typescript-eslint/no-restricted-imports -- import from bundle +import { + readAllBlocksFromDoc, + readAllDocIdsFromRootDoc, +} from '@affine/reader/dist'; +import { applyUpdate, Array as YArray, Doc as YDoc, Map as YMap } from 'yjs'; export interface PageDocContent { title: string; @@ -31,7 +36,7 @@ type KnownFlavour = | 'affine:callout' | 'affine:table'; -export function parseWorkspaceDoc(doc: Doc): WorkspaceDocContent | null { +export function parseWorkspaceDoc(doc: YDoc): WorkspaceDocContent | null { // not a workspace doc if (!doc.share.has('meta')) { return null; @@ -50,7 +55,7 @@ export interface ParsePageOptions { } export function parsePageDoc( - doc: Doc, + doc: YDoc, opts: ParsePageOptions = { maxSummaryLength: 150 } ): PageDocContent | null { // not a page doc @@ -58,7 +63,7 @@ export function parsePageDoc( return null; } - const blocks = doc.getMap>('blocks'); + const blocks = doc.getMap>('blocks'); if (!blocks.size) { return null; @@ -71,7 +76,7 @@ export function parsePageDoc( let summaryLenNeeded = opts.maxSummaryLength; - let root: Map | null = null; + let root: YMap | null = null; for (const block of blocks.values()) { const flavour = block.get('sys:flavour') as KnownFlavour; if (flavour === 'affine:page') { @@ -86,8 +91,8 @@ export function parsePageDoc( const queue: string[] = [root.get('sys:id')]; - function pushChildren(block: Map) { - const children = block.get('sys:children') as Array | undefined; + function pushChildren(block: YMap) { + const children = block.get('sys:children') as YArray | undefined; if (children?.length) { for (let i = children.length - 1; i >= 0; i--) { queue.push(children.get(i)); @@ -157,3 +162,34 @@ export function parsePageDoc( return content; } + +export function readAllDocIdsFromWorkspaceSnapshot(snapshot: Uint8Array) { + const rootDoc = new YDoc(); + applyUpdate(rootDoc, snapshot); + return readAllDocIdsFromRootDoc(rootDoc, { + includeTrash: false, + }); +} + +export async function readAllBlocksFromDocSnapshot( + workspaceId: string, + workspaceSnapshot: Uint8Array, + docId: string, + docSnapshot: Uint8Array, + maxSummaryLength?: number +) { + const rootYDoc = new YDoc({ + guid: workspaceId, + }); + applyUpdate(rootYDoc, workspaceSnapshot); + const ydoc = new YDoc({ + guid: docId, + }); + applyUpdate(ydoc, docSnapshot); + return await readAllBlocksFromDoc({ + ydoc, + rootYDoc, + spaceId: workspaceId, + maxSummaryLength, + }); +} diff --git a/packages/backend/server/src/models/doc.ts b/packages/backend/server/src/models/doc.ts index 67ea6fddcb..d013261bcd 100644 --- a/packages/backend/server/src/models/doc.ts +++ b/packages/backend/server/src/models/doc.ts @@ -158,14 +158,7 @@ export class DocModel extends BaseModel { * Get a doc by workspaceId and docId. */ async get(workspaceId: string, docId: string): Promise { - const row = await this.db.snapshot.findUnique({ - where: { - workspaceId_id: { - workspaceId, - id: docId, - }, - }, - }); + const row = await this.getSnapshot(workspaceId, docId); if (!row) { return null; } @@ -178,6 +171,17 @@ export class DocModel extends BaseModel { }; } + async getSnapshot(workspaceId: string, docId: string) { + return await this.db.snapshot.findUnique({ + where: { + workspaceId_id: { + workspaceId, + id: docId, + }, + }, + }); + } + async getAuthors(workspaceId: string, docId: string) { return await this.db.snapshot.findUnique({ where: { diff --git a/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.md index 9a0864619b..e472c3492e 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.md +++ b/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.md @@ -454,3 +454,43 @@ Generated by [AVA](https://avajs.dev). ], }, ] + +## should index doc work + +> Snapshot 1 + + { + summary: [ + 'AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. You own your data, with no compromisesLocal-first & Real-time collaborativeWe love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.Blocks that assemble your next docs, tasks kanban or whiteboardThere is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ', + ], + title: [ + 'Write, Draw, Plan all at Once.', + ], + } + +> Snapshot 2 + + [ + { + blockId: [ + 'VMx9lHw3TR', + ], + content: [ + 'For developers or installations guides, please go to AFFiNE Doc', + ], + flavour: [ + 'affine:paragraph', + ], + }, + { + blockId: [ + '9-K49otbCv', + ], + content: [ + 'For developer or installation guides, please go to AFFiNE Development', + ], + flavour: [ + 'affine:paragraph', + ], + }, + ] diff --git a/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.snap index c0fe57b9872ee1a84c6e6e4f7d200bbfbf59d98b..80e888c3acb26f4cc8a4c0f7fccd2ec7c862d6a0 100644 GIT binary patch literal 3696 zcmV-$4v+CcRzV_s&vWlR@80$8-LvfOeK(Jk+JJR*M8|Oi0Ubm9?bInlRSGDz%-E{quNg-R#<8VJ z5go@Ff0P!5(wyD9_w7CVE5Sg?Ox|Dj?f0|ie9!mS`F_9W?7_LFD_Zm6yFbDsCvy4T zSB72DciVIjI1#5Ykq(@wOIZ-GMcNWx#Jz|PL@@7!9`nPlh{F8`+ie!HJ!C?M1^Gv8 zqD=$X2w)8{Pa=&A<@%l^q93$)Zowm|O%U>%%=1RfVEp(rvh>&pMGy-|znB2K2ry5^1*DA$$R@@u1b8O_ z4iex21*|Jz?Ry0H0Z|rcj4#k8##01%M#41UbZvZ0Q`L~BYG|btd+4(kpLNEYa!?T)fShBdKNi;dQJub9{Dl~1N0XJ!iY^@@vn3ThhR?zp`EaL517LBZ+&BIn|+ozA& zHs9-nQCMx>kDf5^YV-az0iGtnlm?u39Gdqz8S^$1i>SBl>#A&xtaJLGx%QB%1mrDyj3I~V{B27MR=Tqg0E{%{d<5;;I4MttExN_h*H^*Cl{QfOpmS z-pY>>;L`+nm;m1=z<&|oWDPi91FqJ9NCWQBfG=vmqgst`{ze0StJ+T9H>Y&q6dl;0 z1Fz9vINw}xY<)AK=_zf(X94%3S!GNkrB4p&aYd9FpfMVpwcAeQ2#>ib%}i-iJr>Nj z#lV~O1HRz!L5i?a#OhYSV~=Vs&h|qdWM;5OnAGd?m}Mg_(s5Xn(v}x^ZN}*@*N-|h zg?eRZx=RP{R-Hc|njX-BFR8Gy(Da0U3__E($^h0FwP7ij9n?mhGt?d5<;<&$JNAJS zap#4LJExq8JI7VrkskA6#hv?f;EO8kWhHUv$5}3W{V+P}pkquKz$pem4d7KqO*N;- z3q9Y61H%eJ-ee%{k-bg+)RHR40Pa=en3Y@UR}A2rDr|WU_P7E3Oogo~5eOQ&?P5eb zJ40*4saJ-$tC7>UO|*Uk*w?7u1Z<++*#HixuoXGj{SDwBRoFzyO8zy&kP;0>m}fJX zrQYC({0cJ@WRoo*=Hm(zxW-iYSe9e=CKI?tg{@R!`X@}_GbV7KIc_|YW5!dvZ!N9y zlJvXzh#{tHsOpar21F!?Yn*j6yycfVH z>+_sOEbNEM@|-71@|-o9gl5@@(U`QQI0a&bDbRZkKYc#|?yYNke2oC#CcsY!@LXNn zLw5DftW9yW%^Gll3d_Y2<1aMe%^Dy(yf-V^)r#YwJ94{Lpu-XMq378w|3P&FMbiPh zE?cPEI?vRBv(-UXmc)}?dU-rK^7hK24!l(d?mD9O$^;>w%S^k>7hyCTu}Imkw7mLIS=jmK48dTXp*(+H+)!Op7+GFNfSm+r)kGF;9|88OJyJbqi+2*> zfC^ii3lDv072DsT+SQcR` z_gyJc4LG0N0$RvK5dD@WaN_g}xPjz6u4qRAU+!%XwVD_-@zD)-{sKQs5 zaDJaYriQ(#q$ddZpeWqs<5)u^L-g+^&Z=sAw7^G*^%&PyG~TB%q1D-TTo9Lz%Ob`H^XcI zD;vOB4dAtEMYFtw`8PDO|^L+Wpz#kLf92HiUAMVU-(N@^+_1G|Nna|cqu{<-T_1*%aEU*di1tqI6RFxkh zz+(!ip@QW79doGsOyB(2`Qrym)*L$D_X^f)mxj}Vvsbm1ejXO`V-dj;#$k2 z1o*c)Yxy0Co1;f*Ei0e=(w$G-anBQ-D!O6Z`mCwqlA8EiP%tgvpEv ztV#|wE=tLVH}zK&U`w4TTtk38m8MXnAA88gdDY~NsmkcM${1~n79E;;nYzw=NvP}h zxcEA9<>!6z_4G_sou*51csOz@-I<|umn!<$C*+9$l?N=do~XVpWuK9ewSF#u^M~bq zrM#0K_GL(W*b7K|*!sa?C-TcR;ie!MgA~eBDF-{8}?#{&N9{*7?l&1 zFD9znnt#zIy6j+f2yjDP`|&Ri@2xkJnw%>U*sUZL*k?&1u#xQ^BB|}l$ZOTNGtba~ zvs75VsrFh8*sj9zwQC}YH&*rz-&09_xQJ?#MtT4(PnZ8x2d=BFyBmES*rx*@(1H7P z;1Tte++kJYDIIu52VQOfn+@O%^)+hJlp16T(XrEC*|5scCezd3tJ1??xn`q049j+U zp1d=2+ah6LMWdX7a~m}bT-&H*;Kn>T&%l8S2EJU!z)!0gm@vy3u!*+G1U4%bs;TZ$tp-5!yQsUr>LN(E#Nr?^s?G5YjWSZIU&|g$Y;40kwy+Etv#XNCV{&-fCU>;J}NHlL7r+iL*)dHoSVvWIsIv#1F|-i^YVM`qLaE6GZ(<$uTk9y6+T@d80; zQ^timcRIUXAB)wa!si}MnqKiYM~+8@$LXlGH9Z6cEONXK4Hv_R_voAmVszBywApvu zwxlH=iBT~}7ezmy5obL*aORz9I`4^r%iA4!*Vbbl-jiR4F|SSAPREIuOIaB5D2yL5 z+T>Bh<@gZ~JQg_%oJNA~5;A_rkSm-(%+NiePsPAXCX-5D8RM8zQI2X-l{+Yla;ndejt1i@-}TQ(o4K~x}IJ$ zaH9B3I|j|Mp6G{^1(DNoIb}`Jk0c|KAt{OsPi-Fh@+&#{TFVvv_6*(G527v)dLrP{ zbV(cWz%irDbvHQPhA~CZsLLWc&$%Bi4KL@D4J^kmT00dU4T@Gglp~jWqSv8(ug?RA zdo4bbu?^~GM8|yJby~@*eP4KV`_(&khsmU6Q-por7eN$jYRIk}*B!9MkcN>6csrTH zU8IAq<8s>TyOHC&oO+_oLmKcNb37U@dMz$pbZMrUixOLlWg?BI*K~Oz7LV_Zq$7vZ zG-Ba=NavZ?lr|j5Nh019ENJiU@_-Ll$$}22Vu1%P^QG}dUG4;wMWW}l=sLx^*GcOf za2}h1JaY8?aE9*g@>_;L9rOCJYq2%I5K0~kPbilkwK8?o;MUO{a$yY1_`6hF`aO9M8oJIYBhjcJB zZK>~$h;4$?FycO)ORR8)mQ-rj%a1#!>5hO6rs-8K^Wwo-M6Z;Fl75vke@yDE8%K4P z@o50}0Qf3^|ER6CjNisX0H+dQV|{-rIpw&Fl=L6`EMe3OpCIJ6EFpWfI2rK4vH}0S zY(S&%GYZ*g(|~j8Zogit0as~N#kWl~`ANqo6i}`wCO_x+W@S%Io=0iGFSMF0bcYUH zs#j&9MP|1Ax-0f>b}t@m+I@8~;DVhZpl!auUAYT|(vKZ4jF{`jQ6{9FzSHL6H1%E1 zLQXqEI>|7O?GUZvQKv?im;%-bnBOhoZu5rAUcFgF&Fu@tfY*<4r!rT~Tk=*fwumF* O@&5qH(t_!TJpce9&?hYb literal 2758 zcmV;%3OV&bRzVHgRT=+%=gh0S+k3Z9AZUBrP*7>YMXTbx;x)*&V29lecw4lJ>504wA|kCUBjt5HDl#R?21{hl?#Jb z)iDYlu`pP5Dh1uL^i_qD+3y(rPGQirmRIe5eZa1mj=grMT-F_Z8Ra-sxPPP(n+aeZ zfC;3GBl!yVeIox?KA#7*fe6%2)D^r`0#Nl|Qm*U%l2LMq1wT7GJ3(W;*K$N_0Zb$g zlE;zKQ#ER3+gs#e06!KN;R%a88(Ktd%IB@}I)FC?{-qfHJ1~J*uta;ssMSnYLAA24 z)bpE49ndY?DCo6X;kLSAtxC`$`(8+k@*)7216T&&HUM{ta#2Js$&>3lVY!O7S+`0? zMBqScIij~i&t;E6@A3IG@>sG>%To@&nt*EwSWc}9Vl4{NNLo+8HwYLaV3z>q0+>BV zzzZY@D76Zxk@OA$M_d>Krm)sE712UWv`~kuc6&fC8O3V3NhAt1QZ`H8^ES9oT3=Or zYWfPZZY96fc+ULiZI;nf^l5?1&0`}f9n+^frAkq+)#5*?6z!5}8D5xN&oJU~BOhVz z;##ZHWKn5skO6C$pj$OADDv*|fdut|vhEn=qV5Dc+4GYWsYVqN zA8ZMUZqu}#)QI3$o{fm~hCGuG&hM^ZL-hYxxeC{P569FsqTHOdr zExo6w+HWix(0h&e*z@+*)Yy}{{y(udkz!--ld-W^A)udtyE9_1<3R#`NWdNfo+sd+ z1WaPU0tS4J0S*H;G2kf%>|+_R`8EUoBl=D>Haj^mnFF&p@Cp9@#b*1-jm-v)`?TDE zZW;YfQCL&*>G>wz>xe=N$id=zxmYy?P<_Ps zl4W=i)xD}%w+$=w1uek%Q5VIG3Apgr(H)=G78A9hpudKXIyKomH8pMFz&0`Xqp4{Z z2c8yTNvY`|KMARcjg^22Qf69;R0o+^XR5g4*9^TsHScI!&*q)?FYin~n|IEryyHg9 zsmeP$IPjDRJ0~IUyd37TuWmb|PC8Pj1WcBIf&^SFWps1g6Lf*(C5CpQTP652vOC?M ztkLC2z;-c@sS!z^m4M%iu(k;7fCRiI!p0^j1af4%2xu3DXtGy&g^1V65qoRI>N2oe zPTvGHVw+`Ps|ah4z;?>O&qY`+L6X0O7!tBUfH@k%jLZfD@&!sL$u_nCkB`qPzzvGP z$CwDaYZPF;2SF0(0WU3Jd)QD|Vfd|C($3@t9Tm_yKVG|SB z__La7{CCy3l0+kRp$5#y%)(X^uvVOr@`x{PCSa=wn;1zJPZ0160`?N{H{sJKt!)Z?{+*EM zie=dKT6hbcc7maOS!!tW9tPCHxiBfirtZ70N*SX8f z$QL+pAqTF=Y;L4w94HRQ?wuU?wg?}e!1)f|VuqEzrgKCu@_764)B}`}eIZBWrjY<< zT4lo$-j(d??Tr#d9qXyx?ULqZjA=AtOC;bL5f-hodnBMH!jekQ`$Ej}K7D0~v@1l? z9CM&psXFn-9MNHP1a(LP{*l?bNs0_~$iNI4xK!+D+7g(*Sq`@&C966R7%N9~hn$eh z9+u-vsORMP5=tX>L=j)_*E*fR0YanFO*M65noXg zi--r+q$1+iD)0xj)ta56HD6YNR|U|xlT<`l%^ITFwX~KB>-_r&Gv4c^CEg3555U*5 zFD^CIIh4?~{4VsoEeJVJoBS;RhXIVwE(JbJz-$qgR39!0ZPBV-@9Wc>Wy|t#pXBK? z$0wB#UsXd zbuacnuA4Q0zPDU>)>}@kx9lU}uUX#mE_uVzQF=?qp{F+=-1O+d$L~J$==}+h1G~5V z{rkHQ?A-Xu{Tt#T6T7+>H&C9pG@yPPbAgb8+^dwwaY>4-)8Zy2e`NT}KLPOD?Aw$L z|9CCjF?oj;OUoW1A98`QjYEwqee&?8{!s!JWx2u)1T0H+g*f|YO*ZDHljr#+E3}$S zFOHat(x!%V&F27E&}{Eh_1&0Ze+K>xdj}nzeapNfR_Lq6``GR@zB~L zmCSm7a~9hSmX^Ul&`zYj*f3pJ^hJ%hJHhNF;I^#sP ze0M+jzUyXQHG7;nl>swESahg%DFdz&VbR{Tp@~~kXNUKsQuoABHIn@UX!msa;~cmt zv+tJc99YePZ*gEJ2lk4uL=LOucQ|l_1LsP>LJ7DzyGLydr3OQdXv^uZJFp7T^8V@X z_5R_nE3=fGhTU;`l)NN#+oHihyPV9x965u58|736ZjX|q3~Wtd;F&B2UQK5przA6= z5u2|73xy6<5#wu_0u+_BYD6QpK>_X)K$-~Jt^hw1Kx0JEUIlnDt7|^009@#^#)_ah zDzHERxD)5j1nk0hUW4QeSXtD@8tOCab&^eh~R(|#L z9@Ct@s4%@-x8_e@6OZ~p!p7FDV`JNp7Qad;Jo3%cHg{L+*6oC)P7#mu&Hx_gA0LnZ M0|{8!*3>ir0M6|=5&!@I diff --git a/packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts new file mode 100644 index 0000000000..ab338a3ccb --- /dev/null +++ b/packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts @@ -0,0 +1,108 @@ +import { randomUUID } from 'node:crypto'; +import { mock } from 'node:test'; + +import test from 'ava'; +import Sinon from 'sinon'; + +import { createModule } from '../../../__tests__/create-module'; +import { Mockers } from '../../../__tests__/mocks'; +import { ServerConfigModule } from '../../../core/config'; +import { IndexerModule, IndexerService } from '..'; +import { SearchProviderFactory } from '../factory'; +import { IndexerJob } from '../job'; +import { ManticoresearchProvider } from '../providers'; + +const module = await createModule({ + imports: [IndexerModule, ServerConfigModule], + providers: [IndexerService], +}); +const indexerService = module.get(IndexerService); +const indexerJob = module.get(IndexerJob); +const searchProviderFactory = module.get(SearchProviderFactory); +const manticoresearch = module.get(ManticoresearchProvider); + +const user = await module.create(Mockers.User); +const workspace = await module.create(Mockers.Workspace, { + snapshot: true, + owner: user, +}); + +test.after.always(async () => { + await module.close(); +}); + +test.afterEach.always(() => { + Sinon.restore(); + mock.reset(); +}); + +test.beforeEach(() => { + mock.method(searchProviderFactory, 'get', () => { + return manticoresearch; + }); +}); + +test('should handle indexer.indexDoc job', async t => { + const spy = Sinon.spy(indexerService, 'indexDoc'); + await indexerJob.indexDoc({ + workspaceId: workspace.id, + docId: randomUUID(), + }); + t.is(spy.callCount, 1); +}); + +test('should handle indexer.deleteDoc job', async t => { + const spy = Sinon.spy(indexerService, 'deleteDoc'); + await indexerJob.deleteDoc({ + workspaceId: workspace.id, + docId: randomUUID(), + }); + t.is(spy.callCount, 1); +}); + +test('should handle indexer.indexWorkspace job', async t => { + const count = module.queue.count('indexer.deleteDoc'); + const spy = Sinon.spy(indexerService, 'listDocIds'); + await indexerJob.indexWorkspace({ + workspaceId: workspace.id, + }); + t.is(spy.callCount, 1); + const { payload } = await module.queue.waitFor('indexer.indexDoc'); + t.is(payload.workspaceId, workspace.id); + t.is(payload.docId, '5nS9BSp3Px'); + // no delete job + t.is(module.queue.count('indexer.deleteDoc'), count); +}); + +test('should not sync existing doc', async t => { + const count = module.queue.count('indexer.indexDoc'); + mock.method(indexerService, 'listDocIds', async () => { + return ['5nS9BSp3Px']; + }); + await indexerJob.indexWorkspace({ + workspaceId: workspace.id, + }); + t.is(module.queue.count('indexer.indexDoc'), count); +}); + +test('should delete doc from indexer when docId is not in workspace', async t => { + const count = module.queue.count('indexer.deleteDoc'); + mock.method(indexerService, 'listDocIds', async () => { + return ['mock-doc-id1', 'mock-doc-id2']; + }); + await indexerJob.indexWorkspace({ + workspaceId: workspace.id, + }); + const { payload } = await module.queue.waitFor('indexer.indexDoc'); + t.is(payload.workspaceId, workspace.id); + t.is(payload.docId, '5nS9BSp3Px'); + t.is(module.queue.count('indexer.deleteDoc'), count + 2); +}); + +test('should handle indexer.deleteWorkspace job', async t => { + const spy = Sinon.spy(indexerService, 'deleteWorkspace'); + await indexerJob.deleteWorkspace({ + workspaceId: workspace.id, + }); + t.is(spy.callCount, 1); +}); diff --git a/packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts index 7c7ff44b0d..0ec43fb7eb 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts @@ -27,7 +27,10 @@ const indexerService = module.get(IndexerService); const searchProviderFactory = module.get(SearchProviderFactory); const manticoresearch = module.get(ManticoresearchProvider); const user = await module.create(Mockers.User); -const workspace = await module.create(Mockers.Workspace); +const workspace = await module.create(Mockers.Workspace, { + snapshot: true, + owner: user, +}); mock.method(searchProviderFactory, 'get', () => { return manticoresearch; @@ -1580,3 +1583,524 @@ test('should throw error when field is not allowed in aggregate input', async t }); // #endregion + +// #region deleteWorkspace() + +test('should delete workspace work', async t => { + const workspaceId = randomUUID(); + const docId1 = randomUUID(); + const docId2 = randomUUID(); + await indexerService.write( + SearchTable.doc, + [ + { + workspaceId, + docId: docId1, + title: 'hello world', + summary: 'this is a test', + createdByUserId: user.id, + updatedByUserId: user.id, + createdAt: new Date(), + updatedAt: new Date(), + }, + { + workspaceId, + docId: docId2, + title: 'hello world', + summary: 'this is a test', + createdByUserId: user.id, + updatedByUserId: user.id, + createdAt: new Date(), + updatedAt: new Date(), + }, + ], + { + refresh: true, + } + ); + await indexerService.write( + SearchTable.block, + [ + { + workspaceId, + docId: docId1, + blockId: randomUUID(), + content: 'hello world', + flavour: 'affine:text', + createdByUserId: user.id, + updatedByUserId: user.id, + createdAt: new Date(), + updatedAt: new Date(), + }, + ], + { + refresh: true, + } + ); + + let result = await indexerService.search({ + table: SearchTable.doc, + query: { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + options: { + fields: ['workspaceId', 'docId', 'title', 'summary'], + }, + }); + + t.is(result.total, 2); + t.is(result.nodes.length, 2); + + let result2 = await indexerService.search({ + table: SearchTable.block, + query: { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + options: { + fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], + }, + }); + + t.is(result2.total, 1); + t.is(result2.nodes.length, 1); + + await indexerService.deleteWorkspace(workspaceId, { + refresh: true, + }); + + result = await indexerService.search({ + table: SearchTable.doc, + query: { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + options: { + fields: ['workspaceId', 'docId', 'title', 'summary'], + }, + }); + t.is(result.total, 0); + t.is(result.nodes.length, 0); + + result2 = await indexerService.search({ + table: SearchTable.block, + query: { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + options: { + fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], + }, + }); + + t.is(result2.total, 0); + t.is(result2.nodes.length, 0); +}); + +// #endregion + +// #region deleteDoc() + +test('should delete doc work', async t => { + const workspaceId = randomUUID(); + const docId1 = randomUUID(); + const docId2 = randomUUID(); + await indexerService.write( + SearchTable.doc, + [ + { + workspaceId, + docId: docId1, + title: 'hello world', + summary: 'this is a test', + createdByUserId: user.id, + updatedByUserId: user.id, + createdAt: new Date(), + updatedAt: new Date(), + }, + { + workspaceId, + docId: docId2, + title: 'hello world', + summary: 'this is a test', + createdByUserId: user.id, + updatedByUserId: user.id, + createdAt: new Date(), + updatedAt: new Date(), + }, + ], + { + refresh: true, + } + ); + await indexerService.write( + SearchTable.block, + [ + { + workspaceId, + docId: docId1, + blockId: randomUUID(), + content: 'hello world', + flavour: 'affine:text', + createdByUserId: user.id, + updatedByUserId: user.id, + createdAt: new Date(), + updatedAt: new Date(), + }, + { + workspaceId, + docId: docId2, + blockId: randomUUID(), + content: 'hello world', + flavour: 'affine:text', + createdByUserId: user.id, + updatedByUserId: user.id, + createdAt: new Date(), + updatedAt: new Date(), + }, + ], + { + refresh: true, + } + ); + + let result1 = await indexerService.search({ + table: SearchTable.doc, + query: { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + { + type: SearchQueryType.match, + field: 'docId', + match: docId1, + }, + ], + }, + options: { + fields: ['workspaceId', 'docId', 'title', 'summary'], + }, + }); + + t.is(result1.total, 1); + t.is(result1.nodes.length, 1); + t.deepEqual(result1.nodes[0].fields.docId, [docId1]); + + let result2 = await indexerService.search({ + table: SearchTable.doc, + query: { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + { + type: SearchQueryType.match, + field: 'docId', + match: docId2, + }, + ], + }, + options: { + fields: ['workspaceId', 'docId', 'title', 'summary'], + }, + }); + + t.is(result2.total, 1); + t.is(result2.nodes.length, 1); + t.deepEqual(result2.nodes[0].fields.docId, [docId2]); + + let result3 = await indexerService.search({ + table: SearchTable.block, + query: { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + { + type: SearchQueryType.match, + field: 'docId', + match: docId1, + }, + ], + }, + options: { + fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], + }, + }); + t.is(result3.total, 1); + t.is(result3.nodes.length, 1); + t.deepEqual(result3.nodes[0].fields.docId, [docId1]); + + let result4 = await indexerService.search({ + table: SearchTable.block, + query: { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + { + type: SearchQueryType.match, + field: 'docId', + match: docId2, + }, + ], + }, + options: { + fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], + }, + }); + t.is(result4.total, 1); + t.is(result4.nodes.length, 1); + t.deepEqual(result4.nodes[0].fields.docId, [docId2]); + + const count = module.event.count('doc.indexer.deleted'); + + await indexerService.deleteDoc(workspaceId, docId1, { + refresh: true, + }); + t.is(module.event.count('doc.indexer.deleted'), count + 1); + + // make sure the docId1 is deleted + result1 = await indexerService.search({ + table: SearchTable.doc, + query: { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + { + type: SearchQueryType.match, + field: 'docId', + match: docId1, + }, + ], + }, + options: { + fields: ['workspaceId', 'docId', 'title', 'summary'], + }, + }); + + t.is(result1.total, 0); + t.is(result1.nodes.length, 0); + + // make sure the docId2 is not deleted + result2 = await indexerService.search({ + table: SearchTable.doc, + query: { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + { + type: SearchQueryType.match, + field: 'docId', + match: docId2, + }, + ], + }, + options: { + fields: ['workspaceId', 'docId', 'title', 'summary'], + }, + }); + + t.is(result2.total, 1); + t.is(result2.nodes.length, 1); + t.deepEqual(result2.nodes[0].fields.docId, [docId2]); + + // make sure the docId1 block is deleted + result3 = await indexerService.search({ + table: SearchTable.block, + query: { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + { + type: SearchQueryType.match, + field: 'docId', + match: docId1, + }, + ], + }, + options: { + fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], + }, + }); + + t.is(result3.total, 0); + t.is(result3.nodes.length, 0); + + // docId2 block should not be deleted + result4 = await indexerService.search({ + table: SearchTable.block, + query: { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + { + type: SearchQueryType.match, + field: 'docId', + match: docId2, + }, + ], + }, + options: { + fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], + }, + }); + + t.is(result4.total, 1); + t.is(result4.nodes.length, 1); + t.deepEqual(result4.nodes[0].fields.docId, [docId2]); +}); + +// #endregion + +// #region listDocIds() + +test('should list doc ids work', async t => { + const workspaceId = randomUUID(); + const docs = []; + const docCount = 20011; + for (let i = 0; i < docCount; i++) { + docs.push({ + workspaceId, + docId: randomUUID(), + title: `hello world ${i} ${randomUUID()}`, + summary: `this is a test ${i} ${randomUUID()}`, + createdByUserId: user.id, + updatedByUserId: user.id, + createdAt: new Date(), + updatedAt: new Date(), + }); + } + await indexerService.write(SearchTable.doc, docs, { + refresh: true, + }); + + const docIds = await indexerService.listDocIds(workspaceId); + + t.is(docIds.length, docCount); + t.deepEqual(docIds.sort(), docs.map(doc => doc.docId).sort()); + + await indexerService.deleteWorkspace(workspaceId, { + refresh: true, + }); + const docIds2 = await indexerService.listDocIds(workspaceId); + + t.is(docIds2.length, 0); +}); + +// #endregion + +// #region indexDoc() + +test('should index doc work', async t => { + const count = module.event.count('doc.indexer.updated'); + const docSnapshot = await module.create(Mockers.DocSnapshot, { + workspaceId: workspace.id, + user, + }); + + await indexerService.indexDoc(workspace.id, docSnapshot.id, { + refresh: true, + }); + + const result = await indexerService.search({ + table: SearchTable.doc, + query: { + type: SearchQueryType.match, + field: 'docId', + match: docSnapshot.id, + }, + options: { + fields: ['workspaceId', 'docId', 'title', 'summary'], + }, + }); + + t.is(result.total, 1); + t.deepEqual(result.nodes[0].fields.workspaceId, [workspace.id]); + t.deepEqual(result.nodes[0].fields.docId, [docSnapshot.id]); + t.snapshot(omit(result.nodes[0].fields, ['workspaceId', 'docId'])); + + // search blocks + const result2 = await indexerService.search({ + table: SearchTable.block, + query: { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspace.id, + }, + { + type: SearchQueryType.match, + field: 'content', + match: + 'For developers or installations guides, please go to AFFiNE Doc', + }, + ], + }, + options: { + fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'], + highlights: [ + { + field: 'content', + before: '', + end: '', + }, + ], + pagination: { + limit: 2, + }, + }, + }); + + t.is(result2.nodes.length, 2); + t.snapshot( + result2.nodes.map(node => omit(node.fields, ['workspaceId', 'docId'])) + ); + t.is(module.event.count('doc.indexer.updated'), count + 1); +}); +// #endregion diff --git a/packages/backend/server/src/plugins/indexer/index.ts b/packages/backend/server/src/plugins/indexer/index.ts index d98c806973..6235e9ace0 100644 --- a/packages/backend/server/src/plugins/indexer/index.ts +++ b/packages/backend/server/src/plugins/indexer/index.ts @@ -5,6 +5,7 @@ import { Module } from '@nestjs/common'; import { ServerConfigModule } from '../../core/config'; import { PermissionModule } from '../../core/permission'; import { SearchProviderFactory } from './factory'; +import { IndexerJob } from './job'; import { SearchProviders } from './providers'; import { IndexerResolver } from './resolver'; import { IndexerService } from './service'; @@ -14,6 +15,7 @@ import { IndexerService } from './service'; providers: [ IndexerResolver, IndexerService, + IndexerJob, SearchProviderFactory, ...SearchProviders, ], @@ -22,3 +24,16 @@ import { IndexerService } from './service'; export class IndexerModule {} export { IndexerService }; + +declare global { + interface Events { + 'doc.indexer.updated': { + workspaceId: string; + docId: string; + }; + 'doc.indexer.deleted': { + workspaceId: string; + docId: string; + }; + } +} diff --git a/packages/backend/server/src/plugins/indexer/job.ts b/packages/backend/server/src/plugins/indexer/job.ts new file mode 100644 index 0000000000..d94fc9e5a5 --- /dev/null +++ b/packages/backend/server/src/plugins/indexer/job.ts @@ -0,0 +1,110 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { JobQueue, OnJob } from '../../base'; +import { readAllDocIdsFromWorkspaceSnapshot } from '../../core/utils/blocksuite'; +import { Models } from '../../models'; +import { IndexerService } from './service'; + +declare global { + interface Jobs { + 'indexer.indexDoc': { + workspaceId: string; + docId: string; + }; + 'indexer.deleteDoc': { + workspaceId: string; + docId: string; + }; + 'indexer.indexWorkspace': { + workspaceId: string; + }; + 'indexer.deleteWorkspace': { + workspaceId: string; + }; + } +} + +@Injectable() +export class IndexerJob { + private readonly logger = new Logger(IndexerJob.name); + + constructor( + private readonly models: Models, + private readonly service: IndexerService, + private readonly queue: JobQueue + ) {} + + @OnJob('indexer.indexDoc') + async indexDoc({ workspaceId, docId }: Jobs['indexer.indexDoc']) { + // delete the 'indexer.deleteDoc' job from the queue + await this.queue.remove(`${workspaceId}/${docId}`, 'indexer.deleteDoc'); + await this.service.indexDoc(workspaceId, docId); + } + + @OnJob('indexer.deleteDoc') + async deleteDoc({ workspaceId, docId }: Jobs['indexer.deleteDoc']) { + // delete the 'indexer.updateDoc' job from the queue + await this.queue.remove(`${workspaceId}/${docId}`, 'indexer.indexDoc'); + await this.service.deleteDoc(workspaceId, docId); + } + + @OnJob('indexer.indexWorkspace') + async indexWorkspace({ workspaceId }: Jobs['indexer.indexWorkspace']) { + await this.queue.remove(workspaceId, 'indexer.deleteWorkspace'); + const snapshot = await this.models.doc.getSnapshot( + workspaceId, + workspaceId + ); + if (!snapshot) { + this.logger.warn(`workspace ${workspaceId} not found`); + return; + } + const docIdsInWorkspace = readAllDocIdsFromWorkspaceSnapshot(snapshot.blob); + const docIdsInIndexer = await this.service.listDocIds(workspaceId); + const docIdsInWorkspaceSet = new Set(docIdsInWorkspace); + const docIdsInIndexerSet = new Set(docIdsInIndexer); + // diff the docIdsInWorkspace and docIdsInIndexer + const missingDocIds = docIdsInWorkspace.filter( + docId => !docIdsInIndexerSet.has(docId) + ); + const deletedDocIds = docIdsInIndexer.filter( + docId => !docIdsInWorkspaceSet.has(docId) + ); + for (const docId of deletedDocIds) { + await this.queue.add( + 'indexer.deleteDoc', + { + workspaceId, + docId, + }, + { + jobId: `${workspaceId}/${docId}`, + // the delete job should be higher priority than the update job + priority: 0, + } + ); + } + for (const docId of missingDocIds) { + await this.queue.add( + 'indexer.indexDoc', + { + workspaceId, + docId, + }, + { + jobId: `${workspaceId}/${docId}`, + priority: 100, + } + ); + } + this.logger.debug( + `indexed workspace ${workspaceId} with ${missingDocIds.length} missing docs and ${deletedDocIds.length} deleted docs` + ); + } + + @OnJob('indexer.deleteWorkspace') + async deleteWorkspace({ workspaceId }: Jobs['indexer.deleteWorkspace']) { + await this.queue.remove(workspaceId, 'indexer.indexWorkspace'); + await this.service.deleteWorkspace(workspaceId); + } +} diff --git a/packages/backend/server/src/plugins/indexer/service.ts b/packages/backend/server/src/plugins/indexer/service.ts index 2a9949aa8b..c1bbb48876 100644 --- a/packages/backend/server/src/plugins/indexer/service.ts +++ b/packages/backend/server/src/plugins/indexer/service.ts @@ -1,7 +1,13 @@ import { Injectable, Logger } from '@nestjs/common'; import { camelCase, chunk, mapKeys, snakeCase } from 'lodash-es'; -import { InvalidIndexerInput, SearchProviderNotFound } from '../../base'; +import { + EventBus, + InvalidIndexerInput, + SearchProviderNotFound, +} from '../../base'; +import { readAllBlocksFromDocSnapshot } from '../../core/utils/blocksuite'; +import { Models } from '../../models'; import { SearchProviderType } from './config'; import { SearchProviderFactory } from './factory'; import { @@ -30,6 +36,7 @@ import { SearchHighlight, SearchInput, SearchQuery, + SearchQueryOccur, SearchQueryType, } from './types'; @@ -99,7 +106,11 @@ export interface SearchNodeWithMeta extends SearchNode { export class IndexerService { private readonly logger = new Logger(IndexerService.name); - constructor(private readonly factory: SearchProviderFactory) {} + constructor( + private readonly models: Models, + private readonly factory: SearchProviderFactory, + private readonly event: EventBus + ) {} async createTables() { let searchProvider: SearchProvider | undefined; @@ -161,6 +172,204 @@ export class IndexerService { return result; } + async listDocIds(workspaceId: string) { + const docIds: string[] = []; + let cursor: string | undefined; + do { + const result = await this.search({ + table: SearchTable.doc, + query: { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + options: { + fields: ['docId'], + pagination: { + limit: 10000, + cursor, + }, + }, + }); + docIds.push(...result.nodes.map(node => node.fields.docId[0] as string)); + cursor = result.nextCursor; + this.logger.debug( + `get ${result.nodes.length} new / ${docIds.length} total doc ids for workspace ${workspaceId}, nextCursor: ${cursor}` + ); + } while (cursor); + return docIds; + } + + async indexDoc( + workspaceId: string, + docId: string, + options?: OperationOptions + ) { + const workspaceSnapshot = await this.models.doc.getSnapshot( + workspaceId, + workspaceId + ); + if (!workspaceSnapshot) { + this.logger.debug(`workspace ${workspaceId} not found`); + return; + } + const docSnapshot = await this.models.doc.getSnapshot(workspaceId, docId); + if (!docSnapshot) { + this.logger.debug(`doc ${workspaceId}/${docId} not found`); + return; + } + if (docSnapshot.blob.length <= 2) { + this.logger.debug(`doc ${workspaceId}/${docId} is empty, skip indexing`); + return; + } + const result = await readAllBlocksFromDocSnapshot( + workspaceId, + workspaceSnapshot.blob, + docId, + docSnapshot.blob + ); + if (!result) { + this.logger.warn( + `parse doc ${workspaceId}/${docId} failed, workspaceSnapshot size: ${workspaceSnapshot.blob.length}, docSnapshot size: ${docSnapshot.blob.length}` + ); + return; + } + await this.write( + SearchTable.doc, + [ + { + workspaceId, + docId, + title: result.title, + summary: result.summary, + // NOTE(@fengmk): journal is not supported yet + // journal: result.journal, + createdByUserId: docSnapshot.createdBy ?? '', + updatedByUserId: docSnapshot.updatedBy ?? '', + createdAt: docSnapshot.createdAt, + updatedAt: docSnapshot.updatedAt, + }, + ], + options + ); + await this.deleteBlocksByDocId(workspaceId, docId, options); + await this.write( + SearchTable.block, + result.blocks.map(block => ({ + workspaceId, + docId, + blockId: block.blockId, + content: block.content ?? '', + flavour: block.flavour, + blob: block.blob, + refDocId: block.refDocId, + ref: block.ref, + parentFlavour: block.parentFlavour, + parentBlockId: block.parentBlockId, + additional: block.additional + ? JSON.stringify(block.additional) + : undefined, + markdownPreview: block.markdownPreview, + createdByUserId: docSnapshot.createdBy ?? '', + updatedByUserId: docSnapshot.updatedBy ?? '', + createdAt: docSnapshot.createdAt, + updatedAt: docSnapshot.updatedAt, + })), + options + ); + this.event.emit('doc.indexer.updated', { + workspaceId, + docId, + }); + this.logger.debug( + `synced doc ${workspaceId}/${docId} with ${result.blocks.length} blocks` + ); + } + + async deleteDoc( + workspaceId: string, + docId: string, + options?: OperationOptions + ) { + await this.deleteByQuery( + SearchTable.doc, + { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + { + type: SearchQueryType.match, + field: 'docId', + match: docId, + }, + ], + }, + options + ); + this.logger.debug(`deleted doc ${workspaceId}/${docId}`); + await this.deleteBlocksByDocId(workspaceId, docId, options); + this.event.emit('doc.indexer.deleted', { + workspaceId, + docId, + }); + } + + async deleteBlocksByDocId( + workspaceId: string, + docId: string, + options?: OperationOptions + ) { + await this.deleteByQuery( + SearchTable.block, + { + type: SearchQueryType.boolean, + occur: SearchQueryOccur.must, + queries: [ + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + { + type: SearchQueryType.match, + field: 'docId', + match: docId, + }, + ], + }, + options + ); + this.logger.debug(`deleted all blocks in doc ${workspaceId}/${docId}`); + } + + async deleteWorkspace(workspaceId: string, options?: OperationOptions) { + await this.deleteByQuery( + SearchTable.doc, + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + options + ); + this.logger.debug(`deleted all docs in workspace ${workspaceId}`); + await this.deleteByQuery( + SearchTable.block, + { + type: SearchQueryType.match, + field: 'workspaceId', + match: workspaceId, + }, + options + ); + this.logger.debug(`deleted all blocks in workspace ${workspaceId}`); + } + async deleteByQuery( table: T, query: SearchQuery, diff --git a/packages/backend/server/tsconfig.json b/packages/backend/server/tsconfig.json index 6997a939f7..fec2cae0e5 100644 --- a/packages/backend/server/tsconfig.json +++ b/packages/backend/server/tsconfig.json @@ -12,6 +12,7 @@ }, "include": ["./src"], "references": [ + { "path": "../../common/reader" }, { "path": "../native" }, { "path": "../../../tools/cli" }, { "path": "../../../tools/utils" }, diff --git a/packages/frontend/admin/src/config.json b/packages/frontend/admin/src/config.json index e4f8ac5bdb..9760727906 100644 --- a/packages/frontend/admin/src/config.json +++ b/packages/frontend/admin/src/config.json @@ -31,6 +31,10 @@ "type": "Object", "desc": "The config for doc job queue" }, + "queues.indexer": { + "type": "Object", + "desc": "The config for indexer job queue" + }, "queues.notification": { "type": "Object", "desc": "The config for notification job queue" diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts index b7387e5970..3a29cc84c2 100644 --- a/tools/utils/src/workspace.gen.ts +++ b/tools/utils/src/workspace.gen.ts @@ -1084,6 +1084,7 @@ export const PackageList = [ location: 'packages/backend/server', name: '@affine/server', workspaceDependencies: [ + 'packages/common/reader', 'packages/backend/native', 'tools/cli', 'tools/utils', diff --git a/yarn.lock b/yarn.lock index 6ad19327af..ae1daf0f6f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -906,6 +906,7 @@ __metadata: "@affine-tools/cli": "workspace:*" "@affine-tools/utils": "workspace:*" "@affine/graphql": "workspace:*" + "@affine/reader": "workspace:*" "@affine/server-native": "workspace:*" "@ai-sdk/anthropic": "npm:^1.2.10" "@ai-sdk/google": "npm:^1.2.10"