+ IV-j ^RIHt]'gR]9d^RIHtM^RIt^RItRt Rt Rt Rt !R R ]4t!R R ]4t]P$!]4!R R]4t]P(!]4!RR]4t]P,!]4!RR]4t]P0!]4RtRt!RR]4t]P8!]4!RR]4t]P<!]4Rt^RI t ^RI!t!^RIt^RI"t"^RI#t$^RI%H&t&^RI%H't'Rt(Rt)]PTt+]PTt,]-!]R]P\4]-!]R]P\4]P^]n0]Pb]n2R$F t3])!]]34K ](!]4](!]4](!]4]t4]t5^RI6H7t7]!]"PpPs]:!]$PvPyR 44R!44!R"R#]4t=R# ] d ^RItELi;i)%) version_info.)_sentencepieceNcRVPP4,pRVPP : RVPP : RV: R2# \PdRpLPi;i)z proxy of )this__repr__ __builtin__ Exception __class__ __module____name__)selfstrthiss& g/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/sentencepiece/__init__.py _swig_reprrs_ 2 2 44"^^668O8OQX ZZ  s!AA43A4caV3RlpV#)c<VR8Xd S!WV4R#VR8XdVPPV4R#\W4'd5\\ \ V4V4\ 4'd S!WV4R#\RV,4h)r thisownz(You cannot add instance attributes to %sN)r ownhasattr isinstancegetattrtypepropertyAttributeError)rnamevaluesets&&&rset_instance_attrE_swig_setattr_nondynamic_instance_variable..set_instance_attrsi 6> E " Y  IIMM% T ZT D0I8%T%T E " !Kd!RS S)r r!sf r*_swig_setattr_nondynamic_instance_variabler%sT r#caV3RlpV#)c<\W4'd+\\W4\4'g S!WV4R#\ RV,4h)z%You cannot add class attributes to %sN)rrrrr)clsrrr s&&&rset_class_attr?_swig_setattr_nondynamic_class_variable..set_class_attr)s: 3  j1CX&N&N 5 ! !H3!NO Or#r$)r r)sf r'_swig_setattr_nondynamic_class_variabler+(sP r#caV3RlpV#)zlClass decorator for adding a metaclass to a SWIG wrapped class - a slimmed down version of six.add_metaclasscp<S!VPVPVPP44#N)r __bases____dict__copy)r( metaclasss&rwrapper$_swig_add_metaclass..wrapper3s's}}cll6G6G6IJJr#r$)r2r3sf r_swig_add_metaclassr51sK Nr#c>]tRt^8tRt]!]P4tRtR#)_SwigNonDynamicMetazKMeta class to enforce nondynamic attributes (no new attributes) for a classr$N) rr __qualname____firstlineno____doc__r+r __setattr____static_attributes__r$r#rr7r78sU9$:J:JKKr#r7c a]tRt^=to]!RRRR7t]tRt] Pt Rt Rt RtR tR tR tR t]!] 4t]!]4t]!] 4t]!]4t]!]4t]!]4t]!]4tR tRtRt]tRtVtR#)1ImmutableSentencePieceText_ImmutableSentencePiecec6VPP4#r.r rxs&r:ImmutableSentencePieceText_ImmutableSentencePiece.>r#c8VPPV4#r.r@rBvs&&rrCrD>AFFJJqMr#The membership flagdoccZ\P!V\P!44R#r.)r:ImmutableSentencePieceText_ImmutableSentencePiece_swiginit5new_ImmutableSentencePieceText_ImmutableSentencePiecers&r__init__:ImmutableSentencePieceText_ImmutableSentencePiece.__init__As,QQRVXfY]Y]Y_ `r#c.\P!V4#r.)r8ImmutableSentencePieceText_ImmutableSentencePiece__piecerPs&r_piece8ImmutableSentencePieceText_ImmutableSentencePiece._pieceEVVW[\\r#c.\P!V4#r.)r:ImmutableSentencePieceText_ImmutableSentencePiece__surfacerPs&r_surface:ImmutableSentencePieceText_ImmutableSentencePiece._surfaceHsXXY]^^r#c.\P!V4#r.)r5ImmutableSentencePieceText_ImmutableSentencePiece__idrPs&r_id5ImmutableSentencePieceText_ImmutableSentencePiece._idKsSSTXYYr#c.\P!V4#r.)r8ImmutableSentencePieceText_ImmutableSentencePiece__beginrPs&r_begin8ImmutableSentencePieceText_ImmutableSentencePiece._beginNrWr#c.\P!V4#r.)r6ImmutableSentencePieceText_ImmutableSentencePiece__endrPs&r_end6ImmutableSentencePieceText_ImmutableSentencePiece._endQsTTUYZZr#c.\P!V4#r.)rCImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytesrPs&r_surface_as_bytesCImmutableSentencePieceText_ImmutableSentencePiece._surface_as_bytesTsaabfggr#c.\P!V4#r.)rAImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytesrPs&r_piece_as_bytesAImmutableSentencePieceText_ImmutableSentencePiece._piece_as_bytesWs__`deer#cRPVPVPVPVPVP 4#)z3piece: "{}" id: {} surface: "{}" begin: {} end: {} )formatpieceidsurfacebeginendrPs&r__str__9ImmutableSentencePieceText_ImmutableSentencePiece.__str__bs7"6$**dggt||"&**dhh8 8r#c<VPVP8H;'d}VPVP8H;'d\VPVP8H;'d;VPVP8H;'dVPVP8H#r.)rrrsrtrurvrothers&&r__eq__8ImmutableSentencePieceText_ImmutableSentencePiece.__eq__js ZZ5;; &YY477ehh+>YY4<r>=s-/IOdeGH`%^^]_Z][hf V Eo.Nx G 12 #B V E 4.C8YHr#r>ca]tRt^uto]!RRRR7t]tRt] Pt Rt Rt RtR tR tR t]!]4t]!]4t]!]4t!R R 4t]R4tRtRtRt]tRtVtR#)ImmutableSentencePieceTextc6VPP4#r.r@rAs&rrC#ImmutableSentencePieceText.vrEr#c8VPPV4#r.r@rGs&&rrCrvrIr#rJrKcZ\P!V\P!44R#r.)r#ImmutableSentencePieceText_swiginitnew_ImmutableSentencePieceTextrPs&rrQ#ImmutableSentencePieceText.__init__ys::4AnAnApqr#c.\P!V4#r.)r'ImmutableSentencePieceText__pieces_sizerPs&r _pieces_size'ImmutableSentencePieceText._pieces_size}sEEdKKr#c.\P!W4#r.)r"ImmutableSentencePieceText__piecesrindexs&&r_pieces"ImmutableSentencePieceText._piecess@@MMr#c.\P!V4#r.)r ImmutableSentencePieceText__textrPs&r_text ImmutableSentencePieceText._texts>>tDDr#c.\P!V4#r.)r!ImmutableSentencePieceText__scorerPs&r_score!ImmutableSentencePieceText._scores??EEr#c.\P!V4#r.)r,ImmutableSentencePieceText_SerializeAsStringrPs&rSerializeAsString,ImmutableSentencePieceText.SerializeAsStringJJ4PPr#c.\P!V4#r.)r)ImmutableSentencePieceText__text_as_bytesrPs&r_text_as_bytes)ImmutableSentencePieceText._text_as_bytessGGMMr#c<a]tRt^toRtRtRtRt]tRt Vt R#)9ImmutableSentencePieceText.ImmutableSentencePieceIteratorcPWnVPP4VnR#r.)protorlenrrs&&rrQBImmutableSentencePieceText.ImmutableSentencePieceIterator.__init__ ::**,r#cVP#r.rrPs&r__len__AImmutableSentencePieceText.ImmutableSentencePieceIterator.__len__ xxr#c\V\4'df\VP4Uu.uFq PP V4NK upVP VPVP1,#V^8dWP,pV^8gWP8d \R4hVPP V4#uupi)rzpiece index is out of range) rslicerangerrrstartstopstep IndexErrorrris&& r __getitem__EImmutableSentencePieceText.ImmutableSentencePieceIterator.__getitem__s eU # #16txxAA**$$Q'A%++ejjY^YcYcBcd d 19(("% 19)89 9zz!!%(( B#Cc ~RPVUu.uFpRP\V44NK up4#uupi) pieces {{ {}}}joinrqrrrBs& rrwAImmutableSentencePieceText.ImmutableSentencePieceIterator.__str__3yyDIDq+223q6:DIJJI#:rrN rrr8r9rQrrrwr r<rrs@rImmutableSentencePieceIteratorr!-)Khr#rc,\PV4#r.)rrrPs&rpieces!ImmutableSentencePieceText.piecess ' F Ft LLr#cDVP4VP48H#r.rrzs&&rr|!ImmutableSentencePieceText.__eq__  # # %)@)@)B BBr#c4\VP44#r.rrrPs&rr#ImmutableSentencePieceText.__hash__ $((* ++r#cRPVPVPRPVPUu.uFpRP\ V44NK up44#uupi)ztext: "{}" score: {} {}rr)rqtextscorerrrrs& rrw"ImmutableSentencePieceText.__str__s^F499djj99PTP[P[%\P[1&7&>&>s1v&FP[%\]__&]s#A)r$N)rrr8r9rrrr rQr!delete_ImmutableSentencePieceTextrrrrrrrr text_as_bytesrrrr|rrwr<rrs@rrrus-/IOdeGHr%GGLNEFQN E?D^,M V E,MMC,_ Hr#rca]tRt^to]!RRRR7t]tRt] Pt Rt Rt Rt!R R 4t]R 4tR tR tRt]tRtVtR#)ImmutableNBestSentencePieceTextc6VPP4#r.r@rAs&rrC(ImmutableNBestSentencePieceText.rEr#c8VPPV4#r.r@rGs&&rrCrrIr#rJrKcZ\P!V\P!44R#r.)r(ImmutableNBestSentencePieceText_swiginit#new_ImmutableNBestSentencePieceTextrPs&rrQ(ImmutableNBestSentencePieceText.__init__s??nFxFxFz{r#c.\P!V4#r.)r,ImmutableNBestSentencePieceText__nbests_sizerPs&r _nbests_size,ImmutableNBestSentencePieceText._nbests_sizerr#c.\P!W4#r.)r'ImmutableNBestSentencePieceText__nbestsrs&&r_nbests'ImmutableNBestSentencePieceText._nbestssEEdRRr#c.\P!V4#r.)r1ImmutableNBestSentencePieceText_SerializeAsStringrPs&rr1ImmutableNBestSentencePieceText.SerializeAsStringsOOPTUUr#c<a]tRt^toRtRtRtRt]tRt Vt R#)BImmutableNBestSentencePieceText.ImmutableSentencePieceTextIteratorcPWnVPP4VnR#r.)rrrrs&&rrQKImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__init__rr#cVP#r.rrPs&rrJImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__len__rr#c\V\4'df\VP4Uu.uFq PP V4NK upVP VPVP1,#V^8dWP,pV^8gWP8d \R4hVPP V4#uupi)rznbests index is out of range) rrrrrrrrrrrs&& rrNImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__getitem__s eU # #16txxAA**$$Q'A%++ejjY^YcYcBcd d 19(("% 19)9: :zz!!%(( Brc ~RPVUu.uFpRP\V44NK up4#uupirznbests {{ {}}}rrs& rrwJImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__str__rrrNrrs@r"ImmutableSentencePieceTextIteratorrrr#rc,\PV4#r.)rrrPs&rnbests&ImmutableNBestSentencePieceText.nbestss , O OPT UUr#cDVP4VP48H#r.rrzs&&rr|&ImmutableNBestSentencePieceText.__eq__rr#c4\VP44#r.rrPs&rr(ImmutableNBestSentencePieceText.__hash__rr#c RPVPUu.uFpRP\V44NK up4#uupir)rrrqrrs& rrw'ImmutableNBestSentencePieceText.__str__s7 YY$++N+Q)00Q8+N OONs#Ar$N)rrr8r9rrrr rQr&delete_ImmutableNBestSentencePieceTextrrrrrrr|rrwr<rrs@rrrsw-/IOdeGH|%LLQSV,VVC,PHr#rc a]tRt^to]!RRRR7t]tRt] Pt Rt Rt RtR tR tR tR tR tRtRtRtRtRtRtRtRtRtRtRtRtRt Rt!Rt"Rt#Rt$Rt%R t&R!t'R"t(R#t)R$t*R%t+R&t,R't-R(t.R)t/R*t0R+t1R,t2R-t3R.t4R/t5R0t6R1t7R2t8R3t9R4t:R5t;R6tR9t?R:t@R;tAR<tBR=tCR>R>]DR?R?R?R?R?RgR@Rg3 RAltERhRBltFRCtGRDtHREtIRFtJRiRGltKRiRHltLRiRIltMRiRJltNRjRKltORkRLltPRkRMltQRkRNltRRkROltSRhRPltTRiRQltURiRRltVRiRSltWRiRTltX]YR>3RUltZ]Y3RVlt[]Y3RWlt\RlRXlt]RlRYlt^RmRZlt_RmR[lt`RkR\ltRkR]ltaR^tbR_tcR`tdRateRbtfRctgRdthRiReltiRftjVtkR>#)nSentencePieceProcessorc6VPP4#r.r@rAs&rrCSentencePieceProcessor.rEr#c8VPPV4#r.r@rGs&&rrCrrIr#rJrKcZ\P!V\P!44R#r.)rSentencePieceProcessor_swiginitnew_SentencePieceProcessorrPs&rrQSentencePieceProcessor.__init__s66t^=f=f=hir#c.\P!W4#r.)r.SentencePieceProcessor_LoadFromSerializedProtor serializeds&&rLoadFromSerializedProto.SentencePieceProcessor.LoadFromSerializedProtosLLT^^r#c.\P!W4#r.)r,SentencePieceProcessor_SetEncodeExtraOptionsr extra_options&&rSetEncodeExtraOptions,SentencePieceProcessor.SetEncodeExtraOptionsJJ4^^r#c.\P!W4#r.)r,SentencePieceProcessor_SetDecodeExtraOptionsr s&&rSetDecodeExtraOptions,SentencePieceProcessor.SetDecodeExtraOptionsr$r#c.\P!W4#r.)r$SentencePieceProcessor_SetVocabulary)r valid_vocabs&&r SetVocabulary$SentencePieceProcessor.SetVocabularysBB4UUr#c.\P!V4#r.)r&SentencePieceProcessor_ResetVocabularyrPs&rResetVocabulary&SentencePieceProcessor.ResetVocabulary sDDTJJr#c0\P!WV4#r.)r%SentencePieceProcessor_LoadVocabulary)rfilename thresholds&&&rLoadVocabulary%SentencePieceProcessor.LoadVocabularysCCDT]^^r#c2\P!V.VO5!#r.)r'SentencePieceProcessor_CalculateEntropyrargss&*rCalculateEntropy'SentencePieceProcessor.CalculateEntropysEEdRTRRr#c.\P!V4#r.)r#SentencePieceProcessor_GetPieceSizerPs&r GetPieceSize#SentencePieceProcessor.GetPieceSizesAA$GGr#c.\P!W4#r.)r SentencePieceProcessor_PieceToIdrrrs&&r PieceToId SentencePieceProcessor.PieceToIds>>tKKr#c.\P!W4#r.)r SentencePieceProcessor_IdToPiecerrss&&r IdToPiece SentencePieceProcessor.IdToPiece>>tHHr#c.\P!W4#r.)rSentencePieceProcessor_GetScorerIs&&rGetScoreSentencePieceProcessor.GetScore==dGGr#c.\P!W4#r.)r SentencePieceProcessor_IsUnknownrIs&&r IsUnknown SentencePieceProcessor.IsUnknown rLr#c.\P!W4#r.)r SentencePieceProcessor_IsControlrIs&&r IsControl SentencePieceProcessor.IsControl#rLr#c.\P!W4#r.)rSentencePieceProcessor_IsUnusedrIs&&rIsUnusedSentencePieceProcessor.IsUnused&rQr#c.\P!W4#r.)rSentencePieceProcessor_IsByterIs&&rIsByteSentencePieceProcessor.IsByte)s;;DEEr#c.\P!V4#r.)rSentencePieceProcessor_unk_idrPs&runk_idSentencePieceProcessor.unk_id,;;DAAr#c.\P!V4#r.)rSentencePieceProcessor_bos_idrPs&rbos_idSentencePieceProcessor.bos_id/rfr#c.\P!V4#r.)rSentencePieceProcessor_eos_idrPs&reos_idSentencePieceProcessor.eos_id2rfr#c.\P!V4#r.)rSentencePieceProcessor_pad_idrPs&rpad_idSentencePieceProcessor.pad_id5rfr#c.\P!V4#r.)r-SentencePieceProcessor_serialized_model_protorPs&rserialized_model_proto-SentencePieceProcessor.serialized_model_proto8sKKDQQr#c.\P!W4#r.)r#SentencePieceProcessor_LoadFromFilerargs&&r LoadFromFile#SentencePieceProcessor.LoadFromFile;sAA$LLr#c 6\P!WW#WEWgV4 #r.)r#SentencePieceProcessor__EncodeAsIds rrenable_sampling nbest_sizealphaadd_bosadd_eosreverseemit_unk_pieces &&&&&&&&&r _EncodeAsIds#SentencePieceProcessor._EncodeAsIds>s,AA$okp|CN\] ]r#c 6\P!WW#WEWgV4 #r.)r&SentencePieceProcessor__EncodeAsPiecesrs &&&&&&&&&r_EncodeAsPieces&SentencePieceProcessor._EncodeAsPiecesAs-DDTQ`nsFQ_` `r#c 6\P!WW#WEWgV4 #r.)r/SentencePieceProcessor__EncodeAsSerializedProtors &&&&&&&&&r_EncodeAsSerializedProto/SentencePieceProcessor._EncodeAsSerializedProtoDs-MMdZiw|HOZhi ir#c 6\P!WW#WEWgV4 #r.)r.SentencePieceProcessor__EncodeAsImmutableProtors &&&&&&&&&r_EncodeAsImmutableProto.SentencePieceProcessor._EncodeAsImmutableProtoGs-LLTYhv{GNYgh hr#c 6\P!WW#WEWgW4 #r.)r(SentencePieceProcessor__EncodeAsIdsBatch rins num_threadsrrrrrrrs &&&&&&&&&&r_EncodeAsIdsBatch(SentencePieceProcessor._EncodeAsIdsBatchJs-FFtR]pzDKV]n nr#c 6\P!WW#WEWgW4 #r.)r+SentencePieceProcessor__EncodeAsPiecesBatchrs &&&&&&&&&&r_EncodeAsPiecesBatch+SentencePieceProcessor._EncodeAsPiecesBatchMs-II$U`s}GNY`q qr#c 6\P!WW#WEWgW4 #r.)r4SentencePieceProcessor__EncodeAsSerializedProtoBatchrs &&&&&&&&&&r_EncodeAsSerializedProtoBatch4SentencePieceProcessor._EncodeAsSerializedProtoBatchPs2RRSW^i}GPWbiz zr#c 6\P!WW#WEWgW4 #r.)r3SentencePieceProcessor__EncodeAsImmutableProtoBatchrs &&&&&&&&&&r_EncodeAsImmutableProtoBatch3SentencePieceProcessor._EncodeAsImmutableProtoBatchSs2QQRV]h|FOVahy yr#c.\P!W4#r.)r!SentencePieceProcessor__DecodeIdsridss&&r _DecodeIds!SentencePieceProcessor._DecodeIdsVs??JJr#c.\P!W4#r.)r(SentencePieceProcessor__DecodeIdsAsBytesrs&&r_DecodeIdsAsBytes(SentencePieceProcessor._DecodeIdsAsBytesYsFFtQQr#c.\P!W4#r.)r$SentencePieceProcessor__DecodePiecesrrs&&r _DecodePieces$SentencePieceProcessor._DecodePieces\sBB4PPr#c.\P!W4#r.)r2SentencePieceProcessor__DecodeIdsAsSerializedProtors&&r_DecodeIdsAsSerializedProto2SentencePieceProcessor._DecodeIdsAsSerializedProto_sPPQU[[r#c.\P!W4#r.)r5SentencePieceProcessor__DecodePiecesAsSerializedProtors&&r_DecodePiecesAsSerializedProto5SentencePieceProcessor._DecodePiecesAsSerializedProtobsSSTXaar#c.\P!W4#r.)r1SentencePieceProcessor__DecodeIdsAsImmutableProtors&&r_DecodeIdsAsImmutableProto1SentencePieceProcessor._DecodeIdsAsImmutableProtoesOOPTZZr#c.\P!W4#r.)r4SentencePieceProcessor__DecodePiecesAsImmutableProtors&&r_DecodePiecesAsImmutableProto4SentencePieceProcessor._DecodePiecesAsImmutableProtohsRRSW``r#c0\P!WV4#r.)r&SentencePieceProcessor__DecodeIdsBatchrrrs&&&r_DecodeIdsBatch&SentencePieceProcessor._DecodeIdsBatchksDDTP[\\r#c0\P!WV4#r.)r-SentencePieceProcessor__DecodeIdsAsBytesBatchrs&&&r_DecodeIdsAsBytesBatch-SentencePieceProcessor._DecodeIdsAsBytesBatchnsKKDWbccr#c0\P!WV4#r.)r7SentencePieceProcessor__DecodeIdsAsSerializedProtoBatchrs&&&r _DecodeIdsAsSerializedProtoBatch7SentencePieceProcessor._DecodeIdsAsSerializedProtoBatchqsUUVZalmmr#c0\P!WV4#r.)r6SentencePieceProcessor__DecodeIdsAsImmutableProtoBatchrs&&&r_DecodeIdsAsImmutableProtoBatch6SentencePieceProcessor._DecodeIdsAsImmutableProtoBatchtsTTUY`kllr#c0\P!WV4#r.)r)SentencePieceProcessor__DecodePiecesBatchrs&&&r_DecodePiecesBatch)SentencePieceProcessor._DecodePiecesBatchwsGGS^__r#c0\P!WV4#r.)r:SentencePieceProcessor__DecodePiecesAsSerializedProtoBatchrs&&&r#_DecodePiecesAsSerializedProtoBatch:SentencePieceProcessor._DecodePiecesAsSerializedProtoBatchzsXXY]doppr#c0\P!WV4#r.)r9SentencePieceProcessor__DecodePiecesAsImmutableProtoBatchrs&&&r"_DecodePiecesAsImmutableProtoBatch9SentencePieceProcessor._DecodePiecesAsImmutableProtoBatch}sWWX\cnoor#c 4\P!WW#WEV4#r.)r(SentencePieceProcessor__NBestEncodeAsIdsrrrrrrrs&&&&&&&r_NBestEncodeAsIds(SentencePieceProcessor._NBestEncodeAsIdss&FFtS]ho{IJ Jr#c 4\P!WW#WEV4#r.)r+SentencePieceProcessor__NBestEncodeAsPiecesrs&&&&&&&r_NBestEncodeAsPieces+SentencePieceProcessor._NBestEncodeAsPiecess&II$V`kr~LM Mr#c 4\P!WW#WEV4#r.)r4SentencePieceProcessor__NBestEncodeAsSerializedProtors&&&&&&&r_NBestEncodeAsSerializedProto4SentencePieceProcessor._NBestEncodeAsSerializedProtos'RRSW_it{GUV Vr#c 4\P!WW#WEV4#r.)r3SentencePieceProcessor__NBestEncodeAsImmutableProtors&&&&&&&r_NBestEncodeAsImmutableProto3SentencePieceProcessor._NBestEncodeAsImmutableProtos'QQRV^hszFTU Ur#c 6\P!WW#WEWgW4 #r.)r1SentencePieceProcessor__SampleEncodeAndScoreAsIds rr num_samplesrwor include_bestrrrrs &&&&&&&&&&r_SampleEncodeAndScoreAsIds1SentencePieceProcessor._SampleEncodeAndScoreAsIdss.OOPT\gpsDKV]n nr#c 6\P!WW#WEWgW4 #r.)r4SentencePieceProcessor__SampleEncodeAndScoreAsPiecesrs &&&&&&&&&&r_SampleEncodeAndScoreAsPieces4SentencePieceProcessor._SampleEncodeAndScoreAsPiecess.RRSW_jsvGNY`q qr#c 6\P!WW#WEWgW4 #r.)r=SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProtors &&&&&&&&&&r&_SampleEncodeAndScoreAsSerializedProto=SentencePieceProcessor._SampleEncodeAndScoreAsSerializedProtos.[[\`hs|PWbiz zr#c 6\P!WW#WEWgW4 #r.)r to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout. nbest_size = {0,1}: No sampling is performed. nbest_size > 1: samples from the nbest_size results. nbest_size < 0: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling algorithm. alpha: Soothing parameter for unigram sampling, and dropout probability of merge operations for BPE-dropout. num_threads: number of threads in batch processing (Default = -1, auto-detected) ) model_file model_protoN) $_sentencepiece_processor_init_native _out_type_add_bos_add_eos_reverse_emit_unk_piece_enable_sampling _nbest_size_alpha _num_threadsLoad) rrr out_typerrrrrrrrs &&&&&&&&&&&&rInitSentencePieceProcessor.InitsVD+40nmmm+-#k% { Z A#r#c  Vf VPpVf VPpVf VPpVf VPpVf VPpVf VP pVf VP pV f VPp V f VPp VR8Xd"VeV^8Xg V^8XgV f \R4hV e\V 4\Jd \R4h\V4\Jd~V\JdVPWWxWWEV4 #V\JdVPWWxWWEV4 #VR8XgVR8XdVP!WWxWWEV4 #VR8XdVP#WWxWWEV4 #V\JdVP%WVWWEV4#V\JdVP'WVWWEV4#VR8XgVR8XdVP)WVWWEV4#VR8XdVP+WVWWEV4#\RP-V44h)aEncode text input to segmented ids or tokens. Args: input: input string. accepsts list of string. out_type: output type. int or str. add_bos: Add to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout. nbest_size = {0,1}: No sampling is performed. nbest_size > 1: samples from the nbest_size results. nbest_size < 0: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling algorithm. alpha: Soothing parameter for unigram sampling, and merge probability for BPE-dropout (probablity 'p' in BPE-dropout paper). num_threads: the number of threads used in the batch processing (Default = -1). TaWhen enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. when "nbest_size = -1" , this method samples from all candidates on the lattice instead of nbest segmentations.num_threads must be intserialized_protorimmutable_protozunknown out_type={})r"r#r$r%r&r'r(r)r* RuntimeErrorrintlistrrrrrrrrrrq) rinputr,rrrrrrrrs &&&&&&&&&&&rEncodeSentencePieceProcessor.EncodesW@  >> -- -- --  --  //  %%    '' D j&8J!O&0Ao .    [ 1 <455 e  s?''O(-.Z Z s?**5+07^] ] ) )X-@33E49GVdf f ( (225387Uce e S  !&>S S S##EJ$)GnV V ' '8w+>,,UZ-2W~_ _ & &++EJ,1Gn^ ^ .55h? @@r#c 6VP!RRVR\/VB#r6r,r$r7rrr6kwargss&&,rEncodeAsPieces%SentencePieceProcessor.EncodeAsPieces5 [[ =u =s =f ==r#c 6VP!RRVR\/VB#r:r7r4r<s&&,r EncodeAsIds"SentencePieceProcessor.EncodeAsIds9r@r#c .VP!RRVRR/VB#)r6r,r1r$r7r<s&&,rEncodeAsSerializedProto.SentencePieceProcessor.EncodeAsSerializedProto=s [[ Lu L/A LV LLr#c .VP!RRVRR/VB#)r6r,r2r$rFr<s&&,rEncodeAsImmutableProto-SentencePieceProcessor.EncodeAsImmutableProtoAs [[ Ku K/@ KF KKr#c BVP!RRVRVRVR\RR/VB#r6rrr,rTr$r;rr6rrr=s&&&&,rSampleEncodeAsPieces+SentencePieceProcessor.SampleEncodeAsPiecesEH [[GuGG5G"%G7;G?EGGr#c BVP!RRVRVRVR\RR/VB#rMrBrNs&&&&,rSampleEncodeAsIds(SentencePieceProcessor.SampleEncodeAsIdsJrQr#c :VP!RRVRVRVRRRR/VB#)r6rrr,r1rTr$rFrNs&&&&,rSampleEncodeAsSerializedProto4SentencePieceProcessor.SampleEncodeAsSerializedProtoOsH [[VuVV5V"4VFJVNTVVr#c :VP!RRVRVRVRRRR/VB#)r6rrr,r2rTr$rFrNs&&&&,rSampleEncodeAsImmutableProto3SentencePieceProcessor.SampleEncodeAsImmutableProtoTsH [[UuUU5U"3UEIUMSUUr#c haaaaaaaSf SPoSf SPoSf SPoSf SPoSf SPoSf SP oS^8:d^oVVVVVVV3Rlp\ V4\JdVU u.uF q!V 4NK up #V!V4#uup i)aNBestEncode text input to segmented ids or tokens. Args: input: input string. accepsts list of string. out_type: output type. int or str. add_bos: Add to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) nbest_size: nbest size c<S\JdSPVSSSSS4#S\JdSPVSSSSS4#SR8XgSR8XdSP VSSSSS4#SR8XdSP VSSSSS4#\ R4h)r1rr2zunknown out_type)r4rrrrrr3)rrrrrr,rrs&r_encode3SentencePieceProcessor.NBestEncode.._encode}s s?''j(/'>S S s?**4+2GWnV V ) )X-@33D*4;Wg~_ _ ( (2243:GWn^ ^-..r#)r"r#r$r%r&r(rr5) rr6r,rrrrrr]ns f&ffffff r NBestEncode"SentencePieceProcessor.NBestEncodeYs(  >> -- -- --  --  %% q // e $)*Eq E** U^+sB/c :VP!RRVRVR\/VB#r6rr,r$)r`rrr6rr=s&&&,rNBestEncodeAsPieces*SentencePieceProcessor.NBestEncodeAsPieces3   6E6j6'*6.466r#c :VP!RRVRVR\/VB#rc)r`r4rds&&&,rNBestEncodeAsIds'SentencePieceProcessor.NBestEncodeAsIdsrgr#c 2VP!RRVRVRR/VB#)r6rr,r1r$r`rds&&&,rNBestEncodeAsSerializedProto3SentencePieceProcessor.NBestEncodeAsSerializedProtos9   EEEjE'9E=CEEr#c 2VP!RRVRVRR/VB#)r6rr,r2r$rlrds&&&,rNBestEncodeAsImmutableProto2SentencePieceProcessor.NBestEncodeAsImmutableProtos9   DEDjD'8D to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) num_samples: How many samples to return (Default = 1) alpha: inverse temperature for sampling wor: whether to sample without replacement (Default = false) include_best: whether to include the best tokenization, requires wor=True (Default = false) g?Fznum_examples must be positivez8When include_best is True, We must specify "wor = True".c 4< S\JdS PVSSS SSSSS4 #S\JdS PVSSS SSSSS4 #SR8XgSR8XdS P VSSS SSSSS4 #SR8XdS P VSSS SSSSS4 #\ R4h)r1rr2zunknown output type)r4rrrrrr3) rrrrrrrr,rrrs &rr]._encodes s?00{E3P\18'7N\ \ s?33D+ucS_4;Wg~_ _ ) )X-@<> -- -- --  --        :;; cUVV22& e $)*Eq E** U^+sC!c >VP!RRVRVRVR\/VB#r6rrr,r$)rurrr6rrr=s&&&&,rSampleEncodeAndScoreAsPieces3SentencePieceProcessor.SampleEncodeAndScoreAsPieces;  & &?U? ?SX?03?7=??r#c >VP!RRVRVRVR\/VB#rx)rur4rys&&&&,rSampleEncodeAndScoreAsIds0SentencePieceProcessor.SampleEncodeAndScoreAsIdsr|r#c 6VP!RRVRVRVRR/VB#)r6rrr,r1r$rurys&&&&,r%SampleEncodeAndScoreAsSerializedProto67 7**5EE  # #E 11r#caaVV3Rlp\V4\JdVUu.uF qC!V4NK up#V!V4#uupi)cX<S'dSPV4#SPV4#r.rr rr with_offsetss&r _normalize4SentencePieceProcessor.Normalize.._normalize& ++D1 1t$$r#rr5rr6rrrBsf&f r Normalize SentencePieceProcessor.Normalize?% e ',-u! 1 u--  .<c t/pVP4Fwr4\V4W#&K VPV4#r.)itemsrr)rr= new_kwargskeyrs&, rOverrideNormalizerSpec-SentencePieceProcessor.OverrideNormalizerSpecs5j *#e* '  ) )* 55r#c"VP4#r.r@rPs&r piece_size!SentencePieceProcessor.piece_size     r#c"VP4#r.rrPs&r vocab_size!SentencePieceProcessor.vocab_sizerr#c"VP4#r.rurPs&r __getstate__#SentencePieceProcessor.__getstate__  ( ( **r#cHVP4VPV4R#r.rQrrrus&&r __setstate__#SentencePieceProcessor.__setstate__ mmo ""#9:r#c"VP4#r.rrPs&rrSentencePieceProcessor.__len__rr#c$VPV4#r.)rErDs&&rr"SentencePieceProcessor.__getitem__s ^^E ""r#c V'dV'd \R4hV'dVPV4#VPV4#)zOverwride SentencePieceProcessor.Load to support both model_file and model_proto. Args: model_file: The sentencepiece model file path. model_proto: The sentencepiece model serialized proto. Either `model_file` or `model_proto` must be set. z-model_file and model_proto must be exclusive.)r3rr{)rrr s&&&rr+SentencePieceProcessor.Loads:  JKK ++K88   z **r#) r#r$r)r&r'r(r*r"r%) NNNNNNNNNNN)NNNNNNr.)r1)r2)lrrr8r9rrrr rQrdelete_SentencePieceProcessorrrr"r'r,r0r6r<r@rErJrOrTrXr\r`rdrirmrqrur{rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rrrrr4r-r7r>rCrGrJrOrSrVrYr`rerirmrprurzr~rrrrrrrrrrrrrrrrrrr+r<rrs@rrrs-/IOdeGHj%CC___VK_SHLIHIIHFBBBBRM]`ihnqzyKRQ\b[a]dnm`qpJMVUnqzyLWZkY!"-B`[|>>MLG G V U 7t6 6 E D JZ? ? N M &)d\~,/C),CCCCC 26!!+; !# + +r#rc.\P!V4#r.)rSetRandomGeneratorSeed)seeds&rrrs  0 0 66r#c.\P!V4#r.)rSetMinLogLevel)rHs&rrrs  ( ( ++r#ca]tRtRto]!RRRR7tRt]t] R4t ] R4t ] R 4t ] R 4t ] R 4t] RR l4t] RRl4tRtVtR #)SentencePieceTraineric6VPP4#r.r@rAs&rrCSentencePieceTrainer.rEr#c8VPPV4#r.r@rGs&&rrCrrIr#rJrKc\R4h)zNo constructor defined)r)rr;r=s&*,rrQSentencePieceTrainer.__init__s566r#c.\P!V4#r.)r%SentencePieceTrainer__TrainFromString)rzs&r_TrainFromString%SentencePieceTrainer._TrainFromStringsCCCHHr#c.\P!V4#r.)r"SentencePieceTrainer__TrainFromMapr;s&r _TrainFromMap"SentencePieceTrainer._TrainFromMaps@@FFr#c.\P!W4#r.)r#SentencePieceTrainer__TrainFromMap2r;iters&&r_TrainFromMap2#SentencePieceTrainer._TrainFromMap2AA$MMr#c.\P!V4#r.)r#SentencePieceTrainer__TrainFromMap3rs&r_TrainFromMap3#SentencePieceTrainer._TrainFromMap3sAA$GGr#c.\P!W4#r.)r#SentencePieceTrainer__TrainFromMap4rs&&r_TrainFromMap4#SentencePieceTrainer._TrainFromMap4rr#Nc Ve)\V4\Jd\PV4#RpRpRp/pVP 4F%wrgVR9dTpKVR9dTpKV!V4WV&K' V'dHV'd\P VV4pM\P V4pVPV4R#V'd\PWS4#\PV4#)zDTrain Sentencepiece model. Accept both kwargs and legacy string arg.NcN\V4\Jd\P^,^8Xd \ 4pM \ 4p\ P!VRR7pTPVUu.uFp\V4NK up4VP4#\V4#uupi)zEncode value to CSV..r)lineterminator) rr5sysrStringIOBytesIOcsvwriterwriterowrgetvalue)rfrrHs& rr],SentencePieceTrainer._Train.._encodesy ;$    a A % A A::a3& //515a3q651 2 U 2s,B")sentence_iteratorsentence_reader) model_writer) rrrrrrrwriterr) rzr=r]rrrrrr s &, r_TrainSentencePieceTrainer._Trains T#Y#-#44S99 lj *# : :#  $ $,#EN*/ '  ,;;J;LN+-;;JG+;' %44ZS S%33J? ?r#c \VR7;_uu_4\P!RRV/VBRRR4R# +'giR#;i))ostreamrzNr$) _LogStreamrr)rz logstreamr=s&&,rTrainSentencePieceTrainer.Trains2 i ( (##66v6 ) ( ( (s 8 A r$r.r)rrr8r9rrrQrr staticmethodrrrrrrr r<rrs@rrrs-/IOdeG7HIIGGNNHHNN**X77r#rca]tRtRto]!RRRR7t]tRt] Pt Rt Rt R tR tR tR tR tRtRRltRRltRtRtRtVtR#)SentencePieceNormalizeric6VPP4#r.r@rAs&rrC SentencePieceNormalizer.rEr#c8VPPV4#r.r@rGs&&rrCrrIr#rJrKcZ\P!V\P!44R#r.)r SentencePieceNormalizer_swiginitnew_SentencePieceNormalizerrPs&rrQ SentencePieceNormalizer.__init__ s77n>h>h>jkr#c.\P!W4#r.)r/SentencePieceNormalizer_LoadFromSerializedProtors&&rr/SentencePieceNormalizer.LoadFromSerializedProto$sMMd__r#c.\P!W4#r.)r'SentencePieceNormalizer_LoadFromRuleTSV)rr4s&&rLoadFromRuleTSV'SentencePieceNormalizer.LoadFromRuleTSV'sEEdUUr#c.\P!W4#r.)r(SentencePieceNormalizer_LoadFromRuleName)rrs&&rLoadFromRuleName(SentencePieceNormalizer.LoadFromRuleName*sFFtRRr#c.\P!V4#r.)r.SentencePieceNormalizer_serialized_model_protorPs&rru.SentencePieceNormalizer.serialized_model_proto-sLLTRRr#c.\P!W4#r.)r$SentencePieceNormalizer_LoadFromFilerys&&rr{$SentencePieceNormalizer.LoadFromFile0sBB4MMr#c.\P!W4#r.)r"SentencePieceNormalizer__Normalizer s&&rr "SentencePieceNormalizer._Normalize3s@@LLr#c.\P!W4#r.)r-SentencePieceNormalizer__NormalizeWithOffsetsr s&&rr-SentencePieceNormalizer._NormalizeWithOffsets6sKKDWWr#c0\P!WV4#r.)r&SentencePieceNormalizer__SetProtoField)rrrs&&&r_SetProtoField&SentencePieceNormalizer._SetProtoField9sDDTQVWWr#Nc \V4V'dVPV4pMYV'dVPV4pM?V'dVPV4pM%V'dVP V4pM \ R4hV'd9VP RV4VP RV4VP RV4R#R#)aInitialzie sentencePieceNormalizer. Args: model_file: The sentencepiece model file path. model_proto: The sentencepiece model serialized proto. rule_tsv: The normalization rule file in TSV format. rule_name: Pre-defined normalization name. add_dummy_prefix: add dummy prefix. escape_whitespaces: escape whitespaces. remove_extra_whitespaces: remove extra whitespaces. zno model is specifiedadd_dummy_prefixescape_whitespacesremove_extra_whitespacesN)%_sentencepiece_normalizer_init_nativer{rrr r3r0) rrr rule_tsv rule_namer3r4r5statuss &&&&&&&& rr-SentencePieceNormalizer.Init<s(,D1 "":. --k: %%h/ &&y1233  .0@A 02DE 68PQ r#caaVV3Rlp\V4\JdVUu.uF qC!V4NK up#V!V4#uupi)cX<S'dSPV4#SPV4#r.rrs&rr5SentencePieceNormalizer.Normalize.._normalizecrr#rrsf&f rr!SentencePieceNormalizer.Normalizebrrc"VP4#r.rrPs&rr$SentencePieceNormalizer.__getstate__mrr#cHVP4VPV4R#r.rrs&&rr$SentencePieceNormalizer.__setstate__qrr#r$)NNNNFFFr.)rrr8r9rrrr rQrdelete_SentencePieceNormalizerrrrr rur{r rr0r-rrrr<rrs@rrrsx-/IOdeGHl%DD`VSSNMXX$RL+;;r#rc.\P!V4#r.)r SetDataDir)data_dirs&rrErEys  $ $X ..r#)r)rcL/pVPP4F^wr#\P!RV4'gK$\P!RRV4P 4P RR4pW1V&K` VP4Fwr#\WV4K R#)z1Added snake_cased method from CammelCased method.z^[A-Z]+z(?._funcs5 Aw#~1q5A$7 2 33 :r#ct<\V4\JdVUu.uF pS!W4NK up#S!W4#uupir.r)rrzr_rYs&& r _batched_func!_batchnize.._batched_funcs8 CyD&) *ceDnc ** 4 +s5)rrP)rQrr\rYrXs&& @@r _batchnizer^s% $ '$   )=)r#rQ) __version__ sentencepiece package_datac6a]tRtRtoRRltRtRtRtVtR#)r iNcxWnVPe&\PP4VnR#R#r.)rrstderrfilenoorig_stream_fileno)rrs&&rrQ_LogStream.__init__s+L || # 1 1 3d r#cVPea\P!VP4Vn\P !VPP 4VP4R#R#r.)rosduprforig_stream_dupdup2rerPs&r __enter___LogStream.__enter__sI ||VVD$;$; r#r )rErJrOrTrXr\r`)>rr_swig_python_version_info __package__rrrbuiltinsr ImportErrorrr%r+r5rr7objectr>>ImmutableSentencePieceText_ImmutableSentencePiece_swigregisterr'ImmutableSentencePieceText_swigregisterr,ImmutableNBestSentencePieceText_swigregisterr#SentencePieceProcessor_swigregisterrrr!SentencePieceTrainer_swigregisterr$SentencePieceNormalizer_swigregisterrErKrriimportlib.resources importlibiorrrUr^rQr!r6rPr-r7Tokenizer Detokenizemset_random_generator_seedset_min_log_level_versionr_pathrr resourcesfilesr r$r#rrs&:#/ "[ L$L 33nMMNADDP667QR4f4p;;<[\J +VJ +\223IJ7,K76K7^001EFW;fW;v334KL/  *"(>'F'F$(?(H(H% ,B,G,GH-D-I-IJ"8"?"?$:$A$A! A #Q'   &'$%'(2"! 277<<I//55oFG XYY%sG>> H  H