From f5c67171863fef0b2540e917e7f60ecb176c84e4 Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Tue, 18 Aug 2026 20:34:58 +0800 Subject: [PATCH] change sentence cut method --- ios/OralTrainer.xcodeproj/project.pbxproj | 2 + .../UserInterfaceState.xcuserstate | Bin 8717 -> 13399 bytes sentence_api/DEPLOYMENT.md | 8 + sentence_api/processing.py | 47 ++--- sentence_api/tests/test_processing.py | 84 +++++++++ sentence_api/tests/test_transcription.py | 174 +++++++++++++++++- sentence_api/transcription.py | 154 +++++++++++++++- 7 files changed, 440 insertions(+), 29 deletions(-) create mode 100644 sentence_api/tests/test_processing.py diff --git a/ios/OralTrainer.xcodeproj/project.pbxproj b/ios/OralTrainer.xcodeproj/project.pbxproj index 497dd62..7d25a60 100644 --- a/ios/OralTrainer.xcodeproj/project.pbxproj +++ b/ios/OralTrainer.xcodeproj/project.pbxproj @@ -367,6 +367,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 28FY92RBGB; GENERATE_INFOPLIST_FILE = NO; INFOPLIST_FILE = OralTrainer/Resources/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 16.0; @@ -390,6 +391,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 28FY92RBGB; GENERATE_INFOPLIST_FILE = NO; INFOPLIST_FILE = OralTrainer/Resources/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 16.0; diff --git a/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate b/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate index bfeddf00a1b97059184c76eb7ccddb83783dec5a..9a0e1b3dacba362fe5857753999c1b946cf83135 100644 GIT binary patch literal 13399 zcmcgy34Bw<)}NW%rVDL))23;gv}xL#rmfvq$|7`!LMer`g|fxa+_sT4DM<T>7i9zqxq#Gn8ah-4@Tg-+Km;GHh3!``enJ6qbV@YJApJ36Q99dobbTimWl zgs)p`Vk7E|6Zvl5Qr1uOHVQ@|jYhZG%~QNpfs@fN6oz6@EQ&)}C>!OVT$G3MQ2{DM zMW`4JM&O30;BAr~}!N19hVLs0&?%mZ0V6I&?j{8LdRO zp?lG4bRSxeHlr=*QM4UBg`PprqUX>{=wzo#Ctxj3#5$abvv3|Bj!W=pd^xVbm3Ta^#gp(< z+=OS~EAc#R!yVX;9k>(E$4Qb#6}dNPGfB~4@onM0P4rQ~X|j8Nhs%gG9I4Y`(F zN3JI~kiU^z$X(=aau4Yt50Uj`BiTf@lBdZ($TQ?wvWvV#UM6pn{p2ljh#V$I$Wihx zd7petJ|ka}@5vA30{NN2Odu1%u#AF>)lUcw}#=|UURxsBv*D@=a+nC## zyO`C?24*9(iP_9-VIF0+G20vU_AZ<4APPrv6oFV2g;dk(jM?+}Zr2R>?bpaAeT&WP za`hqwill@xy-10oX#jnKawb+^Se7@uB(JbAt7Ld_MOI-|Sz%Uresx(^VRdnJaaD0u zSxIq$iB;BAR!ws_=eat~EqtY;rK^LtyS)>u5r;A#K^hc~5|9=pA{|OX$uyA4Xb=si zAvBZ@qhXIAJxW0a_|~Cx;g^O}IgNntP|AwG=_WQ>n8#|jI2P8ps$p%$7AMc!=lEKy zo#qbS#AZRmc(ZGgwap0zWoa_ox_DQ$)6r39oXWf04ySppjTeor((E>ySc7kX2J_r< zvvZo+)ns*9foPi5-EOqn+iX1anAk)hENgLtS^HKC?G+BY+v%{`c&CXSBMjx0gnwUQ zWqxIOd2U`-)rj&DS%t-AMOo!VMdeuq)g!9P%Bu4#DvI+>Y}^1Z4u@^7*;(V|#l*%A z^oe}>x`KhVg8UL)-iRVyUSV#oZUp=nKKZ#t#k#!w5xKe$?{u3{se(q*D5|8qia(Ky(M2pUU^>2NxRj)m=jZD4^9W4mKvoq2(^ z%?$FCntYFCP9CJH0IEi})n;{DdDk?n#oaEE)!1E~;0hhi0W#m4gUG5ZRxeH7@PRpf zBYM{}6-`5#>yQyOphnb0Agb4_gDWhTg!AH5vT0qn;-U3$3H(T9&hq%Jo zuKq2nffm;sSBqea^$r*4ox@&G_G;XZt#2*+VO>x-8+jl&Bs8p{)9h|<1~0-d8gI3` zdxp)N>@?dNoMx+?ce3aJmPdrm;TLt9?Lr_Irw~S2ZJ$hn8^PH9r$5l(0-+teNy$?UO z?eQmf?S1{QM1isN3m((|-$`lDZa{wnr&YD6(bi7ni|~WDw^_&E1Mby<^T=PFCKUtG-X3~C+`~GKb@(`Xe!7XpcBPs1V3SK ztFgd7TU)K*CZ%QEf^IdjQFX>iU2cKl0e$O3w+I^d-?OvZ(VZx>7u|tY(M+1v3l52SCy-Y_gIMw6sZ&B&C(|KV_X1gnUQ zQbw!#BmFE#Cl7Ja;S@FwVyfFQkGEG_d7DMDZ!uhg*KzVKuwjxr5lzd)YI(b`QW)3}~=01$yvRKAoKljNAIoKA-m2Fc0>=1ri`5*_F#xE-H) ziz{B2lbfgemsAj-2Zm`7BqwTkzsxS}6QwHl0lrrtHK3bxK=H#-4IuX^prW%OF4zI3 zF9%HiDB6$S2JC$p9Yyb;lju`)4p8{l=ocJ_!vSl@i6}b>8*mXGfydxdTn-4k6;Sof z_;$PXOV4qU4Jpd&!)lY(~< zhtUTpb0aze%5x0V=Qw&7oj~t_8of_TX&Ei26||C8(P}!5j;A#n(J6Eqok1U>v*;uA zG5SQ%HadY$q_uQ9ZKiGXF1nrWpgU>$oJr>?s*1VQzKW(m1TI7Ly(&+Te!15g7o-7Q3@8H2mIPN%g4fP)w8iAQFOKPmth z07&`=t{2YpF4*Zyu!@U?6O~u^7tzlsvj_b|C-tCTXq})r3UD^QHH+?3++8rv{k<4t zLMPKkdU^iAG9cLU{S!*Bzhc9!roa4ulsQ4sAU zbDQ1j?y~UBAQ)+BT&Y#kfA3669Q%iv;*cI|XcK_@X~Ll|LKLgCZGuR-(q?H479GJ! zI2m&&0_$-Kc%oFChSTw7I0LNSd+w`sw1{YK&H%rXNTYo?dn$w~@Wi~-4sUh<_!=M= zPC>#>Om6j#InYP1pfhO;ZKX4)$;9UPBDRD_Avg)LmIfTa^}ZM{?Lju$+kU;dIN zYT577zk_%TurL-JdKoUCY3V;cH%_f>c3V5(ylU>~q-HvoA`@HR3Q`-HlanKun=8j= z?Q*qub->IyUP-xfD%#ENarF&39lXVA?zEY^d1sE*;mR3=a-hgNJw%xXeWH;ODF3DI97MrEW+Ws z9`YMg`T2l3-Qa44go0NZSrXp#Oq466onat}Icm{^Tn%wqI= zj!WsIz=#B>?7t#~-#fzWgUuGk^l~{6|9yuaa4O5Fhb{zBu7bapQ0nhg@5cA^JJtK? zRdi9muamF={<|Qr2Wa;to=b96Lpv17bsWSSL9CkqGH$_-3R%FpW*0vbex1Axy!8T} zB6>Al`hVtOw&CrKbw==My?C2oWHBv{j%;%$#AnGDWed&-5-%opxa88s?}B=%)!5b9 z>2SKUB~A4@scb(QZ-9FO|3e6HPvWQW({wpqL9gk<&){eAbM#tz8@*i!aG8EDN`rcJ z*E_h^0|Lw#iX1TE0;{77jLO-@yQLv2ggh!>#vw*GNo__e1ZaDkL@rsti^7inlU_$h z9m20b5D-Gat9UPd4ex7~5*tEdP;2-ft{1d&8K1d&-8)*9UTE11p%Tm_c2PJ(#cZLLJ1(@AD)!NoB68BF- zFp4LPspf^^ zp%s)_?cdvDL88(Gyxl)hcv5%yatUtV(4X@_X z_3-M$lp7E$ygOcmz%2H1UQEV9Rf7yCC1eB{Nk);;7bA$?#6GC+BJ2-O1B1ee3c3&6Y9I@c*2j-;eZeQ{H`talfTT_8$fXJ8N)=7IcspNdt-?h3Eol1RPao><9Gj_FCTFCKfOT zwTU)D9~WH&nMS5VIQ7x@l4gLYgBrYgGn34c9FVucL^@FB7IFnKlewgYSO`yANgHV= zR&phoM{M*-`V@Vd{)0Y4pQX>y=jjV{7u`+w&=vj0 z`!cq)^FoAviW*(Tfs%EGo-^ zw|SioC?vQRZX}5L(GfavOb_zS2wXAgk!B^rYbR{!`Rr zV)MjM>Md&d?`U=nZ0nR zg6fhH!^?^by-VIqw)k7$W8`r`2HVJXvV-oYZ_&5=$WHPEd6FKW@6r>()(i#zyn5hO z+5acBNS*_oBG1!`k94*}L+*$?QJHhI_l3VBV?x>w0wdW0VBCHu(h z^cX!pG@v7I3o{&`@BBM}?k}MhIY!>`m-Pwqo`1s!-R} zHz9ckZ9U|7`VmbR2^j+CdXdnZq+?`EC;$c~hzVvw=qL13dajQd#)L89^fUS;{Z=?x zXkqq0cp%B4xWrm+(NF|}BZ7pE#ot4fcS|pd?+I5F{xGmVy)Z%lo-9@cEO1-ktklA1K@P`S-|nzO=@+hvv^#7T z-kD|X6j~gvk&vnKHk!PR5;vJl7Pwwt5lGtEJ|>gN7OT?IvzNzQCSTw&kABy~6wvPl z1OA_}APQ?Z6E6wrwtK#z=e=`{WGazr12c*l&0NlmVa770Oc_(oRL~3bM|zR|M1Q8g z&|m3q^!E+Oz*HkWQ_a*c6PSriEu?-TJQ&e%`0`-v!Nh|Z4-SA}&CRg7s%+La;rIN#ZZS;$-kv58s4bTf-R7}SOJ zU_~FZgjvd1JUG&WqhO7Kf(y>Ue+7P1L-E7m6rq@K()bI)a>$_?d8q10cY4zQnp&De z1i_qivO;Kz&bvd`F*gpy{7uZw%q<=q?ZGM!j-lz?*ssKc71Tj>DFo>bFn54hRxx)9 zzDYQ6K$)CeT4VfQD3fm-78a!xip!Zf{nzBAIyqFJ``0Dign@f7Wo~e0bGzKU3!+8{ zWtytZkY}_Uj8@5G;xq!E^Zakr^~BPE9yMHW3kD}z4Pf+;Hc=-}O6K(NY=lZ0Oww9y zGq<_mS%_6!sX-EgBA69!#etD!mkGHsqj)fCtPut&2Y-1WH7)(Jp=#*;6ZNKN43yAQ z)3UPtukmDMzyP^R$2gkyZI!~)kCw-`{!Y(3SQakBIbes(s2l0v z+YR3%;L0Cd9@gccGBio@K0CxpGwg62N=BJ*!FVKG2AqPL&@40uuDWi9Yp%P=>*OFn zx#Q#nIYrKpv*cq&&ZIE;Oa)WLjDv7Ck(mV7SEs<$RSUS?cDSl)gKMgt;MLr4QT0}k z{~OE?0kHwi0bK#N1>7F6BjE9X-2ra}90)iRa3tVZ!0~_+0Ve~#2>3GK>ws?qz7Mz% za53QLfL{aIKrV22;Do?Af%5~e3A{b%F1OEWs_u+Wm9BES)Jbev|xO`4;(B`8N3u`A+$h@~7p`$e)wH zAm1%NB0nZSEEh*YE~3Kg{qlj17HQpGZbN3lY2t>Svc(~3QcmlUrk z_9~7jPANWCe5&|N@rB}IB#BIlG(=WK)YY>S*9>5A-%Tp0OUJ`;N)nV09)jKhnF-0+> zVoGCX#M~Lv6Z1&S#+c19kH+kbc{1kdm}g?1i}^fO6I&Hq7ds`^7~2@z8rvD`jCIE@ zh+P!BFZNLEso0NVKZ!jT`(^CcvERmiANy-uKwNkn8y6X;jMK#x$Bm6M#kIy=7q=#E zd)y0gyW)1oy%={c?pt-JIznww=csel`RcLi@#+ccT6LYeUfryosh+JismWrv5||s7ce5Y3ejnG)7I6X1ZpE#;#eY zS)#dGLp94aD>Zj(?$z93-KyQD-J#v7eM_i2;cj ziN-`e@v6jo6E`QmnD~P(SeK|v(`D$gbUC`wx-wmbu1YsSSF5Yj)$4B5J+IrRJD@wH zJEA+LdtY}-cSd(s_p$Cf-4D7Sbw4H5CN(F`Oq!i!O4^q6e9{X^yOQ=K?N2(AbS&w3 z(ut%qNoSKjPWm+Ivt)I0O|mO_N%FE}Px6Z7JCpBDzBl>4}uel&DfxC>$;4-;vZWK3~yPO-#)pAofBiG2y;AU}i zxGT60&dxcw`P|jqGLCY~xs}`<+@0J#+-mNAZUeWId!F0Py~w@99pDagN4ewN3GO8K zF?WvpoIB6`tS9;ay-XjhkI*ahQTk|ojJ`zQsb8&Muiv2Gq<>WZn0~wddHpN;{rY42 zNj6)88StV&s%(wFj3$|EToQ#Pl3 zmhxT7uPMJ9!VHOqBm-wiF{B#O4HI_p14Tfok8HQPgIR>kN z8de)N8(uUVGn_X3lp2*9pE@a9Uy7$h5MwinQ7^V_H*MbK0!5Icf9KoM~NYi_)%2Tb_1p+6`$prQMSDMB0bx zYh!3F*f48R;$Q^D^dVxHFb!+>~)g#_Ejq89Oqb z$ap&A*^C!5_GG-2aUkPx#?g%98SiDhpK&VVOlELqR_3%!Pv(u8t1{PQK9IRP^R>)< TQphGm`ekO1GB`tw&hEQ5WGVGgx zaFreGMnwb#L~sR_eSJJo5y9sM;>IQ}AR_N1rTTo{d^b5cXU@!=`Dgw!r^hZ#ekPZR zN0;Px3yg$OFdD|dI2aElPzq(>gDR+o0L*}yFb|%FrJ%yIum;w`I;e#PcmX!TORxv_ z!amp!2jCzag2QkGj>0iG1^(aX9=sRt!=9LinK&Fr;b- z@mYKhm*GZy2{++pd>LQCEw~l8;p_MY?!tq32oK{CJc=LTX*`SP@Jqai*YHQYj<@j- zyu$)+7R6eymaG+vX02Hai)C@F4U1=OSx455-N$;dhgfg+2uopUY#1BPa@Yv=ILl)m zHkOsKayEfYVo$NDY#Lk6RT2{yESp!?26<0Q~q~sKszzh}; z!3OfM^o*qO>Mj-^wwz+ri);}lyjmGH1q~f2k-AZLGEsthTeBo48xrdw12Q3toa7>R zJq+W2b0~`cZAS^YVzjzNkk#Yb&Rv=ft z>VbUKYOFP!M6=DyqJ7UYVy^o+EL{=EDWPqMeb?=3IO^HtaLCGK-`3>Ly-NUViL)V>xnc&HN8 z%|b%^*gh3~Aow(kJqIfwu@08Oa_UG)b+8gvQ77uc#V?vN1J47GOXsGz)ImMRtgp|P zS6v5nFzI9n%k|J$LtUFBHo%Kx{&XxPSPvV5qOpCR`225HH^ELw+z6ZDWq1X)z*g7> zufleC4R+8ybT8dU_frpgfV%di2dNi5v=Ls1H{eZp3*LrzU>EF$ci}ziO?~NMdW4cG zgDPktEuzKT?Om2O>TPpJb*-hf`k*CWJ#UHbJSHu#%3JE4>?zFf=lMOkqBYr*Ke4#j z>eF33p>)<1p^uH6i zqLi9fT|UvDo9Zd@XO<837LWCdT#1cxnhbsO&a9lxs2hcADNt; z+232@35K;E1&rWZXWF8J1%O60sV~_&hMD=?SzDJuA~e>bl`^U8-R#9kbVFhz+EIor zsGt*FlttM*dxp_)%4vi+Y=JHLCmP~7pNyc9{4*-VEW;ub)kTp*T6e%soL?{zJ7N-z zrZJRTkDaj#cBRLul*)NR-H9C7>3$w#>_K_d`3ydYz3?IIjeWp^eeq#@1e38J_QwHy zR|+2w!c?`&zAsQhUK&sNTuT|1P%(}Bzihz4P1%4Mlu8A6GodJiDP{%jWmDmwm|{*4 zQ}X;BLZ2;gr|CI`x!j${;TGn@%^#^_yTNvoMjv7RPM- zwKJJHn8G3KL=E%;htWhE`TG)Wrnl)x5Yk*x8R{wEL>8hf^;pXr>Wx-yM>YGAz}g2R zkxJugSqEBA3C#;htaET-5Ea+5F4XwfLU(pgQ#hL??`IG2>z(zW4YjN%y~q!`IyQP# zpCGAPi3JNV)U>c<)}Lz(&egI3wCQfp*+9-oA;D%&V(Dxsk0cw+hOo!zWqO6S)Uynh z$+BoGy+&_v1CF?P0XCA2=K9zu+E&ZP(5sX%aJ%RR?p@j*Dri>3inG%*lKE{`$BO>E^_iEA4{9ApJ8D@8?fgHjTxSJ)g86dNBQlaE zG*R;F>ZUe*^uf-D`PoGEi8i(bKB$CA!_zaGSi3m_RcvxlcQw6P%bujSczN)hB_U4( ztT5=|_D4&1@1X6?8>X|TL4J|VU^Ce)Hk-|1bJ;vLpDmzwXcz6Kcj-NPpZ3sR+V>*F zvqd~37O^F4DO1_AoUbjkpXb7UIzR{M5FO^@m{-TdyD<>TcGehbJU|alh=D}t0uMlM zcnq?6Lq7&@^4`|SB6(XI&pPl{wiD~l?qwWm-nv$CXO{71bro+^*YPH`jyI@{Y$tnz zy~W;PyV-ke58KBMutV$%b~P+IEG2Aw*ru?9VPA!v*XT6inn0u`R?|k)R?|+?LDNyw zNz+BsNAs{IS<_$hsAixhRgS6-AUb=z0G~Hm`P+g`jTbHYwqMNFt@O9xYgr5jMtp~kFU#6d_U!Y&1U#(xO zuhG}(8}vu@AL_rB0~pN5~V3gmR%m@Ci=}^MxhC zD&Ym;ZDE)2uJFFFSJ*Ec6b=hVh2z2rA#g#sD10k?FI*C?2-k$`!q395!cE~;M9YXl z5oHn85zj~Lj5ro?HR6uJW{5U)Hw-YO7zP>A4B3W!L$P6kVXk4mq292=u+Q+R;i}PS zG#f>uWQ;V*M#Y$J9A+G09AzA19B-^NPBBh3l5x6mp>dh-r}?PEtQrk zOTaSCGQ%>>~~pE5zC2DzQ=AEAAH$iigFc;&JhW zcvAd8{7AeaUK6j2Ka0PLH^p04jWygVSPfRi>as>zTUy&$yIOl&2Us(#S=M3J9BaOH zthLm-+c}IJx9y4c9`*mjq_F}u&KHgqxpKPCIpJAV6pJQKSf5yJl{;YkOy}`cKzSn-t{)zny`&af0 z_KWt*_N(?E?LXOnvEPx6vPm|}qAba+sQRR`$tH%5&w#0eOk6%FoGbk*~?u z<)7tW<(u*?`L;uLC=Qn+%F)sh?TB&2IpQ4&j`og3N0OtnV~}I4W1?e?W3S^Q#}A62 zv{1S${gu(mIHgo6SA2?JsZ?etvz58Z0%f7HSXrY4)++0i8fCMxMcJmjrtDPSP!1@k zlyk~w$`{I4%2nmM^0RV7`AzxVsdeg|5l*Ai>5O*9IOCl0&W_Gb&MwYw&U>8M&PC3h z&b`h9&O^?l&g0G#&U4NS&P&c;oHtzP(z{G9m#c-Vl`F;-=ZbfAb@g=ha}99Kb}bIL zYF!&#TV1fzuiAYRsaA1 diff --git a/sentence_api/DEPLOYMENT.md b/sentence_api/DEPLOYMENT.md index d83b81e..0a757b4 100644 --- a/sentence_api/DEPLOYMENT.md +++ b/sentence_api/DEPLOYMENT.md @@ -150,6 +150,14 @@ curl -X POST http://127.0.0.1:8001/v1/audio/transcriptions \ 响应应包含 `text`；开启 `verbose_json` 后应包含带 `start`、`end`、`text` 的 `segments`。 +### 断句规则 + +`sentence_api` 不以 Whisper 返回的 segment 直接作为句子，而是把每个 segment 的文本 +按句号（`.`）拆成一句句，句号即句子结束。若转写服务支持词级时间戳 +（`timestamp_granularities[]=word`，Whisper/Speaches 支持），句子的结束时间用 +句号所在单词的时间戳；不支持时自动按文字长度在 segment 内估算，请求失败会回退为 +不带词级时间戳的普通请求。MOSS 不支持该参数时同样自动回退，不影响转写。 + ## 3. 部署 API ### Docker Compose 方式 diff --git a/sentence_api/processing.py b/sentence_api/processing.py index 2bfee26..b92f57f 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -9,10 +9,10 @@ from .config import Settings from .generate_boundaries import ALGORITHM_VERSION, make_entry from .models import SentenceBoundary, SentenceBoundaryDocument from .repository import VideoRepository -from .transcription import Transcript, Transcriber +from .transcription import Transcript, Transcriber, split_segment_by_periods -MOSS_ALGORITHM_VERSION = "moss-timestamp-v1" +MOSS_ALGORITHM_VERSION = "moss-period-v2" class VideoProcessor: @@ -108,28 +108,29 @@ def document_from_transcript( sentences: List[SentenceBoundary] = [] previous_end = 0 for segment in sorted(transcript.segments, key=lambda item: (item.start_seconds, item.end_seconds)): - start_ms = max(previous_end, int(round(segment.start_seconds * 1000))) - end_ms = min(duration_ms, int(round(segment.end_seconds * 1000))) - if not segment.text.strip() or end_ms <= start_ms: - continue - start_sample = max(0, int(start_ms / 1000 * sample_rate)) - end_sample = min(samples.size, int(end_ms / 1000 * sample_rate)) - try: - metrics = analyze_samples(samples[start_sample:end_sample], sample_rate) - speech_duration_ms = metrics.speech_duration_ms - except AudioAnalysisError: - speech_duration_ms = end_ms - start_ms - sentences.append( - SentenceBoundary( - index=len(sentences), - start_ms=start_ms, - end_ms=end_ms, - text=segment.text.strip(), - language=language, - reference_speech_duration_ms=max(1, speech_duration_ms), + for sentence_segment in split_segment_by_periods(segment): + start_ms = max(previous_end, int(round(sentence_segment.start_seconds * 1000))) + end_ms = min(duration_ms, int(round(sentence_segment.end_seconds * 1000))) + if not sentence_segment.text.strip() or end_ms <= start_ms: + continue + start_sample = max(0, int(start_ms / 1000 * sample_rate)) + end_sample = min(samples.size, int(end_ms / 1000 * sample_rate)) + try: + metrics = analyze_samples(samples[start_sample:end_sample], sample_rate) + speech_duration_ms = metrics.speech_duration_ms + except AudioAnalysisError: + speech_duration_ms = end_ms - start_ms + sentences.append( + SentenceBoundary( + index=len(sentences), + start_ms=start_ms, + end_ms=end_ms, + text=sentence_segment.text.strip(), + language=language, + reference_speech_duration_ms=max(1, speech_duration_ms), + ) ) - ) - previous_end = end_ms + previous_end = end_ms return SentenceBoundaryDocument( video_hash=video_hash, duration_ms=duration_ms, diff --git a/sentence_api/tests/test_processing.py b/sentence_api/tests/test_processing.py new file mode 100644 index 0000000..01aadb2 --- /dev/null +++ b/sentence_api/tests/test_processing.py @@ -0,0 +1,84 @@ +import numpy as np + +from sentence_api.processing import MOSS_ALGORITHM_VERSION, document_from_transcript +from sentence_api.transcription import ( + Transcript, + TranscriptionSegment, + WordTimestamp, + _wav_bytes, +) + + +SAMPLE_RATE = 16000 +VIDEO_HASH = "a" * 64 + + +def _speech(seconds: float) -> np.ndarray: + t = np.arange(int(seconds * SAMPLE_RATE)) / SAMPLE_RATE + return 0.25 * np.sin(2 * np.pi * 220 * t) + + +def _silence(seconds: float) -> np.ndarray: + return np.zeros(int(seconds * SAMPLE_RATE)) + + +def test_document_from_transcript_splits_sentences_at_periods(tmp_path): + samples = np.concatenate( + [ + _speech(2.0), + _silence(0.2), + _speech(2.0), + _silence(0.2), + _speech(1.6), + ] + ).astype(np.float32) + wav = tmp_path / "audio.wav" + wav.write_bytes(_wav_bytes(samples, SAMPLE_RATE)) + transcript = Transcript( + text="Hello world. Good day everyone. Nice to meet you.", + segments=[ + TranscriptionSegment( + start_seconds=0.0, + end_seconds=2.0, + text="Hello world.", + words=[ + WordTimestamp(0.0, 0.6, "Hello"), + WordTimestamp(0.7, 1.5, "world."), + ], + ), + TranscriptionSegment( + start_seconds=2.2, + end_seconds=5.5, + text="Good day everyone. Nice to meet you.", + words=[ + WordTimestamp(2.2, 2.8, "Good"), + WordTimestamp(2.9, 3.5, "day"), + WordTimestamp(3.6, 4.2, "everyone."), + WordTimestamp(4.4, 4.9, "Nice"), + WordTimestamp(5.0, 5.5, "you."), + ], + ), + ], + ) + + document = document_from_transcript( + video_hash=VIDEO_HASH, + duration_ms=6000, + transcript=transcript, + language="en", + audio_path=wav, + ) + + assert document.algorithm_version == MOSS_ALGORITHM_VERSION + assert [sentence.text for sentence in document.sentences] == [ + "Hello world.", + "Good day everyone.", + "Nice to meet you.", + ] + assert document.sentences[0].start_ms == 0 + assert document.sentences[0].end_ms == 1500 + assert document.sentences[1].start_ms == 2200 + assert document.sentences[1].end_ms == 4200 + assert document.sentences[2].start_ms == 4200 + assert document.sentences[2].end_ms == 5500 + assert all(sentence.reference_speech_duration_ms > 0 for sentence in document.sentences) diff --git a/sentence_api/tests/test_transcription.py b/sentence_api/tests/test_transcription.py index 7db9cdd..ce1270d 100644 --- a/sentence_api/tests/test_transcription.py +++ b/sentence_api/tests/test_transcription.py @@ -6,8 +6,12 @@ import pytest from sentence_api.transcription import ( MossTranscriber, + TranscriptionSegment, + WordTimestamp, _parse_json_segments, _plan_chunks, + _parse_word_timestamps, + split_segment_by_periods, _wav_bytes, _wav_duration_seconds, ) @@ -107,7 +111,16 @@ def test_chunked_transcribe_offsets_timestamps(monkeypatch): ] elif call_count["n"] == 2: segments = [ - {"start": 1.0, "end": 4.0, "text": "third part", "compression_ratio": 1.4}, + { + "start": 1.0, + "end": 4.0, + "text": "third part", + "compression_ratio": 1.4, + "words": [ + {"word": "third", "start": 1.0, "end": 2.5}, + {"word": " part", "start": 2.6, "end": 4.0}, + ], + }, {"start": 27.0, "end": 30.0, "text": "fourth part", "compression_ratio": 1.5}, ] else: @@ -115,6 +128,8 @@ def test_chunked_transcribe_offsets_timestamps(monkeypatch): assert abs(duration - 30.0) < 1.0 or abs(duration - 5.0) < 1.0 class Response: + status_code = 200 + def raise_for_status(self): pass @@ -134,5 +149,162 @@ def test_chunked_transcribe_offsets_timestamps(monkeypatch): (60.5, 63.5, "tail part"), ] assert result.text == "first part second part third part fourth part tail part" + assert [(w.start_seconds, w.end_seconds, w.text) for w in result.segments[2].words] == [ + (31.0, 32.5, "third"), + (32.6, 34.0, "part"), + ] + assert result.segments[0].words is None assert len(requests) == 3 assert all(request.get("condition_on_previous_text") == "false" for request in requests) + assert all(request.get("timestamp_granularities[]") == "word" for request in requests) + + +def test_parse_json_segments_parses_word_timestamps(): + raw = [ + { + "start": 0.0, + "end": 4.0, + "text": "Hello world. Good day.", + "words": [ + {"word": "Hello", "start": 0.0, "end": 0.6}, + {"word": " world.", "start": 0.7, "end": 1.5}, + {"word": " Good", "start": 1.8, "end": 2.4}, + {"word": " day.", "start": 2.5, "end": 3.2}, + ], + }, + {"start": 4.5, "end": 6.0, "text": "no words"}, + ] + segments = _parse_json_segments(raw) + assert [word.text for word in segments[0].words] == ["Hello", "world.", "Good", "day."] + assert segments[1].words is None + + +def test_parse_word_timestamps_ignores_invalid_entries(): + words = _parse_word_timestamps( + [ + {"word": "ok", "start": 0.0, "end": 0.5}, + {"word": "bad"}, + {"word": "", "start": 1.0, "end": 1.5}, + {"word": "flat", "start": 2.0, "end": 2.0}, + ] + ) + assert [(word.text, word.start_seconds, word.end_seconds) for word in words] == [ + ("ok", 0.0, 0.5) + ] + assert _parse_word_timestamps(None) is None + assert _parse_word_timestamps("nope") is None + + +def test_split_segment_by_periods_uses_word_timestamps(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=4.0, + text="Hello world. Good day. Nice to meet you.", + words=[ + WordTimestamp(0.0, 0.6, "Hello"), + WordTimestamp(0.7, 1.5, "world."), + WordTimestamp(1.8, 2.4, "Good"), + WordTimestamp(2.5, 3.2, "day."), + WordTimestamp(3.3, 3.8, "Nice"), + WordTimestamp(3.9, 4.0, "you."), + ], + ) + sentences = split_segment_by_periods(segment) + assert [(s.text, s.start_seconds, s.end_seconds) for s in sentences] == [ + ("Hello world.", 0.0, 1.5), + ("Good day.", 1.5, 3.2), + ("Nice to meet you.", 3.2, 4.0), + ] + + +def test_split_segment_by_periods_falls_back_to_proportional(): + segment = TranscriptionSegment( + start_seconds=10.0, + end_seconds=20.0, + text="First sentence. Second sentence. Third.", + ) + sentences = split_segment_by_periods(segment) + assert [s.text for s in sentences] == [ + "First sentence.", + "Second sentence.", + "Third.", + ] + assert sentences[0].start_seconds == 10.0 + assert sentences[1].start_seconds == sentences[0].end_seconds + assert sentences[2].end_seconds == 20.0 + assert sentences[0].end_seconds > 10.0 + assert sentences[1].end_seconds < 20.0 + assert sentences[0].end_seconds < sentences[1].end_seconds + + +def test_split_segment_by_periods_falls_back_when_word_ends_are_invalid(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=2.0, + text="One. Two.", + words=[ + WordTimestamp(0.0, 2.5, "One."), + WordTimestamp(2.6, 3.0, "Two."), + ], + ) + sentences = split_segment_by_periods(segment) + assert [s.text for s in sentences] == ["One.", "Two."] + assert sentences[0].end_seconds == sentences[1].start_seconds + assert sentences[1].end_seconds == 2.0 + + +def test_split_segment_by_periods_keeps_segment_without_period(): + segment = TranscriptionSegment( + start_seconds=1.0, end_seconds=2.0, text="no period here" + ) + assert split_segment_by_periods(segment) == [segment] + + +def test_split_segment_by_periods_handles_ellipsis_and_dots_only(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=2.0, + text="Wait... What? ...", + ) + sentences = split_segment_by_periods(segment) + assert [s.text for s in sentences] == ["Wait...", "What? ..."] + assert sentences[0].start_seconds == 0.0 + assert sentences[0].end_seconds == sentences[1].start_seconds + assert sentences[1].end_seconds == 2.0 + + +def test_post_audio_requests_word_timestamps_and_falls_back(monkeypatch): + calls = [] + + class RejectedResponse: + status_code = 400 + + def raise_for_status(self): + pass + + def json(self): + return {"error": "unknown parameter"} + + class OkResponse: + status_code = 200 + + def raise_for_status(self): + pass + + def json(self): + return {"text": "hi.", "segments": []} + + def fake_post(endpoint, data=None, files=None, timeout=None): + calls.append(dict(data)) + if "timestamp_granularities[]" in data: + return RejectedResponse() + return OkResponse() + + monkeypatch.setattr("sentence_api.transcription.httpx.post", fake_post) + transcriber = MossTranscriber(endpoint="http://whisper:9000", model="whisper") + result = transcriber._post_audio( + io.BytesIO(b"fake-audio"), "clip.wav", "audio/wav", "en" + ) + assert result["text"] == "hi." + assert calls[0]["timestamp_granularities[]"] == "word" + assert "timestamp_granularities[]" not in calls[1] diff --git a/sentence_api/transcription.py b/sentence_api/transcription.py index e2801ec..642205c 100644 --- a/sentence_api/transcription.py +++ b/sentence_api/transcription.py @@ -31,6 +31,14 @@ class TranscriptionSegment: end_seconds: float text: str speaker: Optional[str] = None + words: Optional[List["WordTimestamp"]] = None + + +@dataclass(frozen=True) +class WordTimestamp: + start_seconds: float + end_seconds: float + text: str @dataclass(frozen=True) @@ -114,12 +122,23 @@ class MossTranscriber: if not chunk_segments: chunk_segments = _parse_compact_segments(str(payload.get("text") or "").strip()) for segment in chunk_segments: + words = None + if segment.words: + words = [ + WordTimestamp( + start_seconds=word.start_seconds + start_seconds, + end_seconds=word.end_seconds + start_seconds, + text=word.text, + ) + for word in segment.words + ] segments.append( TranscriptionSegment( start_seconds=segment.start_seconds + start_seconds, end_seconds=segment.end_seconds + start_seconds, text=segment.text, speaker=segment.speaker, + words=words, ) ) segments.sort(key=lambda item: (item.start_seconds, item.end_seconds)) @@ -144,17 +163,27 @@ class MossTranscriber: data["language"] = language try: - response = httpx.post( - self.endpoint, - data=data, - files={"file": (filename, audio_file, content_type)}, - timeout=httpx.Timeout(self.timeout_seconds, connect=30), + word_payload = dict(data) + word_payload["timestamp_granularities[]"] = "word" + response = self._post_once( + word_payload, audio_file, filename, content_type ) + if response.status_code in (400, 422): + audio_file.seek(0) + response = self._post_once(data, audio_file, filename, content_type) response.raise_for_status() return response.json() except (httpx.HTTPError, ValueError) as exc: raise RuntimeError(f"MOSS transcription request failed: {exc}") from exc + def _post_once(self, data: dict, audio_file, filename: str, content_type: str): + return httpx.post( + self.endpoint, + data=data, + files={"file": (filename, audio_file, content_type)}, + timeout=httpx.Timeout(self.timeout_seconds, connect=30), + ) + def _parse_json_segments( raw_segments, compression_limit: Optional[float] = None @@ -186,11 +215,126 @@ def _parse_json_segments( end_seconds=end, text=text, speaker=str(speaker) if speaker is not None else None, + words=_parse_word_timestamps(item.get("words")), ) ) return segments +def _parse_word_timestamps(raw_words) -> Optional[List[WordTimestamp]]: + if not isinstance(raw_words, list): + return None + words = [] + for item in raw_words: + if not isinstance(item, dict): + continue + try: + start = float(item["start"]) + end = float(item["end"]) + except (KeyError, TypeError, ValueError): + continue + text = str(item.get("word") or "").strip() + if text and end > start: + words.append(WordTimestamp(start_seconds=start, end_seconds=end, text=text)) + return words or None + + +_PERIOD_BOUNDARY = re.compile(r"\.+") + + +def split_segment_by_periods( + segment: TranscriptionSegment, +) -> List[TranscriptionSegment]: + """Split one whisper segment into sentences at periods (".").""" + text = segment.text.strip() + if not text: + return [] + pieces: List[str] = [] + cursor = 0 + for match in _PERIOD_BOUNDARY.finditer(text): + piece = text[cursor:match.end()].strip() + if piece.strip(".").strip(): + pieces.append(piece) + cursor = match.end() + tail = text[cursor:].strip() + if tail: + pieces.append(tail) + if len(pieces) <= 1: + return [segment] if pieces else [] + ends = _sentence_end_seconds(segment, pieces) + sub_segments: List[TranscriptionSegment] = [] + start = segment.start_seconds + for piece, end in zip(pieces, ends): + if end > start: + sub_segments.append( + TranscriptionSegment( + start_seconds=start, + end_seconds=end, + text=piece, + speaker=segment.speaker, + ) + ) + start = end + return sub_segments + + +def _sentence_end_seconds( + segment: TranscriptionSegment, pieces: List[str] +) -> List[float]: + lengths = [len(piece) for piece in pieces] + total = sum(lengths) + if segment.words and len(segment.words) >= len(pieces): + word_ends = _word_boundary_ends(segment, lengths) + if _valid_boundaries(segment, word_ends): + return word_ends + [segment.end_seconds] + return _proportional_ends(segment, lengths) + [segment.end_seconds] + + +def _word_boundary_ends( + segment: TranscriptionSegment, lengths: List[int] +) -> List[float]: + targets = [sum(lengths[:count]) for count in range(1, len(lengths))] + buffer = "" + word_indexes: List[int] = [] + target_index = 0 + for word_index, word in enumerate(segment.words): + buffer += (" " if buffer else "") + word.text + while target_index < len(targets) and len(buffer) >= targets[target_index]: + word_indexes.append(word_index) + target_index += 1 + if len(word_indexes) != len(targets): + return [] + return [segment.words[index].end_seconds for index in word_indexes] + + +def _valid_boundaries( + segment: TranscriptionSegment, ends: List[float] +) -> bool: + previous = segment.start_seconds + for end in ends: + if not segment.start_seconds < end < segment.end_seconds: + return False + if end <= previous: + return False + previous = end + return True + + +def _proportional_ends( + segment: TranscriptionSegment, lengths: List[int] +) -> List[float]: + total = sum(lengths) + ends = [] + cumulative = 0 + for length in lengths[:-1]: + cumulative += length + ends.append( + segment.start_seconds + + (cumulative / total) * (segment.end_seconds - segment.start_seconds) + ) + return ends + + def _wav_duration_seconds(path: Path) -> Optional[float]: try: with wave.open(str(path), "rb") as wav: