+
    QV-j¹#  ã                   ó~  € ^ RI t ^ RIt^ RIHtHt ^ RIHt ^ RIt^ RIH	t	 ^ RI
Ht ^RIHt ^RIHt ^RIHtHt ^R	IHtHtHtHt ]P0                  ! ]4      t]! ]P8                  ! 4       4      t];QJ d    . R
 ] 4       F  NK  	  5M! R
 ] 4       4      t] ! R R4      4       t  ! R R]4      t! ! R R]4      t"R# )é    N)Ú	dataclassÚfield)ÚEnum)ÚFileLock)ÚDataset)Ú$MODEL_FOR_QUESTION_ANSWERING_MAPPING)ÚPreTrainedTokenizer)Úcheck_torch_load_is_safeÚlogging)ÚSquadFeaturesÚSquadV1ProcessorÚSquadV2ProcessorÚ"squad_convert_examples_to_featuresc              #   ó8   "  € T F  qP                   x € K  	  R # 5i©N)Ú
model_type)Ú.0Úconfs   & Úq/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/data/datasets/squad.pyÚ	<genexpr>r   !   s   é € ÐEÑ0D¨—O–OÓ0Dùs   ‚c                   ó|  a € ] tR t^$t o Rt]! RRRRP                  ]4      ,           /R7      t]! RRR/R7      t	]! ^€RR/R7      t
]! ^€RR	/R7      t]! ^@RR
/R7      t]! ^RR/R7      t]! RRR/R7      t]! RRR/R7      t]! RRR/R7      t]! ^RR/R7      t]! ^ RR/R7      t]! ^RR/R7      tV 3R ltRtV tR# )ÚSquadDataTrainingArgumentszZ
Arguments pertaining to what data we are going to input our model for training and eval.
NÚhelpz!Model type selected in the list: z, )ÚdefaultÚmetadatazFThe input data dir. Should contain the .json files for the SQuAD task.z‹The maximum total input sequence length after tokenization. Sequences longer than this will be truncated, sequences shorter will be padded.zVWhen splitting up a long document into chunks, how much stride to take between chunks.zkThe maximum number of tokens for the question. Questions longer than this will be truncated to this length.z�The maximum length of an answer that can be generated. This is needed because the start and end predictions are not conditioned on one another.Fz1Overwrite the cached training and evaluation setszDIf true, the SQuAD examples contain some that do not have an answer.g        zIIf null_score - best_non_null is greater than the threshold predict null.zjlanguage id of input for language-specific xlm models (see tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)z3multiple threads for converting example to featuresc                óª   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   # )é   r   Údata_dirÚmax_seq_lengthÚ
doc_strideÚmax_query_lengthÚmax_answer_lengthÚoverwrite_cacheÚversion_2_with_negativeÚnull_score_diff_thresholdÚn_best_sizeÚlang_idÚthreads)ÚstrÚintÚboolÚfloat)ÚformatÚ__classdict__s   "€r   Ú__annotate__Ú'SquadDataTrainingArguments.__annotate__$   sÅ   ø‡ ‚ ñ ñ ñ ñ ñ ñ ñ ñ ñ ñ* ñ ñ+ ñ2 ñ ñ3 ñD ñ ñE ñV ñ ñW ñ\ "ñ ñ] ñb  %ñ ñc ñh ñ ñi ñn ñ ño ñ@ ÑmòA ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   ÚjoinÚMODEL_TYPESr   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r.   s   @r   r   r   $   sN  ø‡ € ññ Ø Ð(KÈdÏiÉiÐXcÓNdÕ(dÐeô€Jñ Ø Ð(pÐqô€Hñ  ØàðQð
ô€Nñ ØØÐrÐsô€Jñ "Øàð/ð
ôÐñ #ØàðJð
ôÐñ "Ø Ð)\Ð ]ô€Oñ %*Ø Ð)oÐ pô%Ðñ (-Ø˜vÐ'rÐsô(Ðñ Ø˜fÐ&qÐrô€Kñ ØàðCð
ô€Gñ  ¨fÐ6kÐ-lÔm€G÷A ƒ r1   r   c                   ó   € ] tR t^gtRtRtRtR# )ÚSplitÚtrainÚdevr2   N)r3   r4   r5   r6   r@   rA   r;   r2   r1   r   r?   r?   g   s   † Ø€EØ
„Cr1   r?   c                   óx   a € ] tR t^lt o R]P
                  RRR3V 3R lR lltR tV 3R lR ltV 3R	 lt	R
t
V tR# )ÚSquadDatasetNFÚptc                óp   <€ V ^8„  d   QhRS[ RS[RS[R,          RS[S[,          RS[RS[R,          RS[/# )	r   ÚargsÚ	tokenizerÚlimit_lengthNÚmodeÚis_language_sensitiveÚ	cache_dirÚdataset_format)r   r	   r*   r)   r?   r+   )r-   r.   s   "€r   r/   ÚSquadDataset.__annotate__r   sf   ø€ ÷ Jñ Já(ðJñ 'ðJñ ˜D•jð	Jñ
 ‘E�kðJñ  $ðJñ ˜•:ðJñ ñJr1   c                ó~  € Wn         WPn        VP                  '       d   \        4       M	\	        4       V n        \        V\        4      '       d    \        V,          pW@n
        VP                  '       d   RMRp\        P                  P                  Ve   TMVP                  RVP                   RVP                   P"                   RVP$                   RV 24      p	V	R,           p
\'        V
4      ;_uu_ 4        \        P                  P)                  V	4      '       Ed   VP*                  '       Eg   \,        P,                  ! 4       p\/        4        \0        P2                  ! V	RR7      V n        V P4                  R	,          V n        V P4                  P9                  R
R4      V n        V P4                  P9                  RR4      V n        \>        PA                  RV	 R2\,        P,                  ! 4       V,
          4       V P:                  e   V P<                  f   \>        PC                  RV	 R24       EMTV\        PD                  8X  d,   V P
                  PG                  VP                  4      V n        M*V P
                  PI                  VP                  4      V n        \K        V P<                  VVP$                  VPL                  VPN                  V\        PP                  8H  VPR                  VR7      w  V n        V n        \,        P,                  ! 4       p\0        PT                  ! R	V P6                  R
V P:                  RV P<                  /V	4       \>        PA                  RV	 R\,        P,                  ! 4       V,
          R R24       RRR4       R#   \         d    \        R 4      hi ; i  + '       g   i     R# ; i)zmode is not a valid split nameÚv2Úv1NÚcached_Ú_z.lockT)Úweights_onlyÚfeaturesÚdatasetÚexamplesz"Loading features from cached file z [took %.3f s]zDeleting cached file z; will allow dataset and examples to be cached in future run)rV   rG   r   r    r!   Úis_trainingr(   Úreturn_datasetz!Saving features into cached file z [took z.3fz s])+rF   rJ   r$   r   r   Ú	processorÚ
isinstancer)   r?   ÚKeyErrorrI   ÚosÚpathr8   r   ÚvalueÚ	__class__r3   r   r   Úexistsr#   Útimer
   ÚtorchÚloadÚold_featuresrT   ÚgetrU   rV   ÚloggerÚinfoÚwarningrA   Úget_dev_examplesÚget_train_examplesr   r    r!   r@   r(   Úsave)ÚselfrF   rG   rH   rI   rJ   rK   rL   Úversion_tagÚcached_features_fileÚ	lock_pathÚstarts   &&&&&&&&    r   Ú__init__ÚSquadDataset.__init__r   sò  € ð Œ	Ø%:Ô"Ø/3×/K×/KÐ/KÔ)Ô+ÔQaÓQcˆŒÜ�dœC× Ò ðAÜ˜T•{�ð Œ	à"×:×:Ð:‘dÀˆÜ!Ÿw™wŸ|™|Ø"Ò.‰I°D·M±MØ�d—j‘j�\  9×#6Ñ#6×#?Ñ#?Ð"@ÀÀ$×BUÑBUÐAVÐVWÐXcÐWdÐeó 
Ðð )¨7Õ2ˆ	Ü�i× Õ Ü�w‰w�~‰~Ð2×3Ó3¸D×<P×<PÑ<PÜŸ	š	›�Ü(Ô*Ü$)§J¢JÐ/CÐRVÔ$W�Ô!ð !%× 1Ñ 1°*Õ =�”Ø#×0Ñ0×4Ñ4°YÀÓE�”Ø $× 1Ñ 1× 5Ñ 5°jÀ$Ó G�”Ü—‘Ø8Ð9MÐ8NÈnÐ]Ô_c×_hÒ_hÓ_jÐmrÕ_rôð —<‘<Ò'¨4¯=©=Ò+@Ü—N‘NØ/Ð0DÐ/Eð F&ð &ôùð
 œ5Ÿ9™9Ô$Ø$(§N¡N×$CÑ$CÀDÇMÁMÓ$R�D•Mà$(§N¡N×$EÑ$EÀdÇmÁmÓ$T�D”Mä.PØ!Ÿ]™]Ø'Ø#'×#6Ñ#6Ø#Ÿ™Ø%)×%:Ñ%:Ø $¬¯©Ñ 3Ø ŸL™LØ#1ô	/Ñ+�”˜tœ|ô Ÿ	š	›�Ü—
’
Ø §¡°	¸4¿<¹<ÈÐUY×UbÑUbÐcØ(ôô
 —‘Ø7Ð8LÐ7MÈWÔUY×U^ÒU^ÓU`ÐchÕUhÐilÐTmÐmpÐqô÷W !Ñ øô ô AÜÐ?Ó@Ð@ðAú÷ !× Ð ús   ÁN Ã89N+Ä2IN+ÎN(Î+N<	c                ó,   € \        V P                  4      # r   )ÚlenrT   )rl   s   &r   Ú__len__ÚSquadDataset.__len__¾   s   € Ü�4—=‘=Ó!Ð!r1   c                óJ   <€ V ^8„  d   QhRS[ S[S[P                  3,          /# )r   Úreturn)Údictr)   rb   ÚTensor)r-   r.   s   "€r   r/   rM   Á   s"   ø€ ÷  ñ  ¡¡S©%¯,©,Ð%6Õ 7ñ  r1   c                óz  € V P                   V,          p\        P                  ! VP                  \        P                  R 7      p\        P                  ! VP
                  \        P                  R 7      p\        P                  ! VP                  \        P                  R 7      p\        P                  ! VP                  \        P                  R 7      p\        P                  ! VP                  \        P                  R 7      p\        P                  ! VP                  \        P                  R 7      pRVRVRV/p	V P                  P                  R
9   d   V	R V P                  P                  R9   d³   V	P                  RVRV/4       V P                  P                  '       d   V	P                  RV/4       V P                  '       d]   V	P                  R\        P                   ! VP"                  \        P$                  R 7      V P                  P&                  ,          /4       V P(                  \*        P,                  8X  dv   \        P                  ! VP.                  \        P                  R 7      p
\        P                  ! VP0                  \        P                  R 7      pV	P                  RV
R	V/4       V	# ))ÚdtypeÚ	input_idsÚattention_maskÚtoken_type_idsÚ	cls_indexÚp_maskÚis_impossibleÚlangsÚstart_positionsÚend_positions)ÚxlmÚrobertaÚ
distilbertÚ	camembert)Úxlnetr†   )rT   rb   Útensorr}   Úlongr~   r   r€   r�   r,   r‚   rF   r   Úupdater$   rJ   ÚonesÚshapeÚint64r'   rI   r?   r@   Ústart_positionÚend_position)rl   ÚiÚfeaturer}   r~   r   r€   r�   r‚   Úinputsr„   r…   s   &&          r   Ú__getitem__ÚSquadDataset.__getitem__Á   sË  € à—-‘- Õ"ˆä—L’L ×!2Ñ!2¼%¿*¹*ÔEˆ	ÜŸš g×&<Ñ&<ÄEÇJÁJÔOˆÜŸš g×&<Ñ&<ÄEÇJÁJÔOˆÜ—L’L ×!2Ñ!2¼%¿*¹*ÔEˆ	Ü—’˜gŸn™n´E·K±KÔ@ˆÜŸš W×%:Ñ%:Ä%Ç+Á+ÔNˆð ˜Ø˜nØ˜nð
ˆð �9‰9×ÑÐ#PÔPØÐ'Ð(à�9‰9×ÑÐ#3Ô3Ø�M‰M˜;¨	°8¸VÐDÔEØ�y‰y×0×0Ð0Ø—‘˜°Ð>Ô?Ø×)×)Ð)Ø—‘˜w¬¯ª°I·O±OÌ5Ï;É;Ô)WÐZ^×ZcÑZc×ZkÑZkÕ)kÐmÔnà�9‰9œŸ™Ô#Ü#Ÿlšl¨7×+AÑ+AÌÏÉÔTˆOÜ!ŸLšL¨×)=Ñ)=ÄUÇZÁZÔPˆMØ�M‰MÐ,¨o¸ÐP]Ð^Ô_àˆr1   c                óZ   <€ V ^8„  d   Qh/ S[ ;R&   S[S[,          ;R&   S[;R&   S[;R&   # )r   rF   rT   rI   rJ   )r   Úlistr   r?   r+   )r-   r.   s   "€r   r/   rM   l   s9   ø‡ ‚ Ù
$Ñ$ñ á‘=Õ!Ñ!ñ ñ �Kñ ñ  Ñò	 r1   )rF   rU   rV   rT   rJ   rI   rd   rY   )r3   r4   r5   r6   r?   r@   rq   ru   r–   r:   r;   r<   r=   s   @r   rC   rC   l   s>   ø‡ € ð $(Ø!ŸK™KØ&+Ø $Ø"÷Jò JòX"÷ ð  ÷k ƒ r1   rC   )#r\   ra   Údataclassesr   r   Úenumr   rb   Úfilelockr   Útorch.utils.datar   Úmodels.auto.modeling_autor   Útokenization_pythonr	   Úutilsr
   r   Úprocessors.squadr   r   r   r   Ú
get_loggerr3   rf   r™   ÚkeysÚMODEL_CONFIG_CLASSESÚtupler9   r   r?   rC   r2   r1   r   Ú<module>r¦      s©   ðó 
Û ß (Ý ã Ý Ý $å MÝ 6ß 6ß tÓ tð 
×	Ò	˜HÓ	%€áÐ@×EÒEÓGÓHÐ ßˆeÑEÑ0DÓE�e‰eÑEÑ0DÓEÓE€ð ÷?nð ?nó ð?nôDˆDô ô
u�7ö ur1   