+
    TV-jt%  ã                   óâ   € ^ RI t ^ RIt^ RIHt ^ RIHtHtHt ^ RIH	t	  ! R R4      t
 ! R R4      t ! R	 R
4      t ! R R4      t ! R R4      tR R ltR R ltR R ltR R ltR R ltR# )é    N)ÚPath)ÚAnyÚDictÚList©ÚPreTrainedTokenizerc                   óX   a € ] tR t^t o RtR
V 3R lR lltR tV 3R lR ltR tRt	V t
R	# )ÚTextDatasetz)
Light-weight wrapper to hold a dataset.
c                óR   <€ V ^8„  d   QhRS[ S[S[S[3,          ,          RS[RS[/# )é   ÚdataÚ	tokenizerÚtext_key)r   r   Ústrr   )ÚformatÚ__classdict__s   "€Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/tuner/datasets.pyÚ__annotate__ÚTextDataset.__annotate__   s6   ø€ ÷ !ñ !á‘4™™S˜•>Õ"ð!ñ 'ð!ñ ñ	!ó    c                ó*   € Wn         W n        W0n        R # ©N)Ú_datar   r   )Úselfr   r   r   s   &&&&r   Ú__init__ÚTextDataset.__init__   s   € ð Œ
Ø"ŒØ Žr   c                óî   € V P                   P                  WP                  ,          4      pVR,          V P                   P                  8w  d&   VP	                  V P                   P                  4       V^ 3# )é   éÿÿÿÿ)r   Úencoder   Úeos_token_idÚappend©r   Úds   &&r   ÚprocessÚTextDataset.process   sR   € Ø�N‰N×!Ñ! !§M¡MÕ"2Ó3ˆØˆR�5�D—N‘N×/Ñ/Ô/Ø�H‰H�T—^‘^×0Ñ0Ô1Ø�1ˆvˆr   c                ó    <€ V ^8„  d   QhRS[ /# ©r   Úidx©Úint)r   r   s   "€r   r   r       ó   ø€ ÷ ñ ™sñ r   c                ó(   € V P                   V,          # r   ©r   ©r   r)   s   &&r   Ú__getitem__ÚTextDataset.__getitem__    ó   € Ø�z‰z˜#�Ðr   c                ó,   € \        V P                  4      # r   ©Úlenr   ©r   s   &r   Ú__len__ÚTextDataset.__len__#   ó   € Ü�4—:‘:‹Ðr   )r   r   r   N)Útext©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r%   r0   r7   Ú__static_attributes__Ú__classdictcell__©r   s   @r   r
   r
      s-   ø‡ € ñ÷!ò !ò÷ð ÷ð r   r
   c                   óX   a € ] tR t^'t o RtR
V 3R lR lltR tV 3R lR ltR tRt	V t
R	# )ÚChatDatasetz’
A dataset for chat data in the format of {"messages": [...]}
https://platform.openai.com/docs/guides/supervised-fine-tuning#formatting-your-data
c                óX   <€ V ^8„  d   QhRS[ S[S[S[3,          ,          RS[RS[RS[/# )r   r   r   Úchat_keyÚmask_prompt©r   r   r   r   Úbool)r   r   s   "€r   r   ÚChatDataset.__annotate__-   s@   ø€ ÷ 
#ñ 
#á‘4™™S˜•>Õ"ð
#ñ 'ð
#ñ ð	
#ñ
 ñ
#r   c                ó6   € Wn         W0n        W@n        W n        R # r   ©r   rG   rH   r   )r   r   r   rG   rH   s   &&&&&r   r   ÚChatDataset.__init__-   s   € ð Œ
Ø ŒØ&ÔØ"Žr   c           	     óD  € WP                   ,          pVP                  R R4      pV P                  P                  VVRR7      pV P                  '       dJ   VR,          P                  R4      R8H  p\        V P                  P                  VRR VVRR7      4      pWF3# V^ 3# )ÚtoolsNF©rP   Úreturn_dictÚroleÚ	assistant©rP   Úadd_generation_promptrR   r   )rG   Úgetr   Úapply_chat_templaterH   r5   )r   r$   ÚmessagesrP   ÚtokensrV   Úoffsets   &&     r   r%   ÚChatDataset.process9   s°   € Ø—]‘]Õ#ˆØ—‘�g˜tÓ$ˆØ—‘×3Ñ3ØØØð 4ó 
ˆð
 ××ÐØ$,¨R¥L×$4Ñ$4°VÓ$<ÀÑ$KÐ!ÜØ—‘×2Ñ2Ø˜S˜b�MØØ*?Ø %ð	 3ó óˆFð Ð#Ð#à˜A�;Ðr   c                ó    <€ V ^8„  d   QhRS[ /# r(   r*   )r   r   s   "€r   r   rK   O   r,   r   c                ó(   € V P                   V,          # r   r.   r/   s   &&r   r0   ÚChatDataset.__getitem__O   r2   r   c                ó,   € \        V P                  4      # r   r4   r6   s   &r   r7   ÚChatDataset.__len__R   r9   r   rM   N)rY   Fr;   rC   s   @r   rE   rE   '   s-   ø‡ € ñ÷

#ò 
#ò÷,ð ÷ð r   rE   c                   óT   a € ] tR t^Vt o RtV 3R lR ltR tV 3R lR ltR tRt	V t
R	# )
ÚCompletionsDatasetzÚ
A dataset for prompt-completion data in the format of {"prompt": ..., "completion": ...}
or using user-provided keys for prompt and completion values
https://platform.openai.com/docs/guides/fine-tuning/example-format
c          
      ó^   <€ V ^8„  d   QhRS[ S[S[S[3,          ,          RS[RS[RS[RS[/# )r   r   r   Ú
prompt_keyÚcompletion_keyrH   rI   )r   r   s   "€r   r   ÚCompletionsDataset.__annotate__]   sJ   ø€ ÷ #ñ #á‘4™™S˜•>Õ"ð#ñ 'ð#ñ ð	#ñ
 ð#ñ ñ#r   c                óB   € Wn         W0n        W@n        WPn        W n        R # r   )r   re   rf   rH   r   )r   r   r   re   rf   rH   s   &&&&&&r   r   ÚCompletionsDataset.__init__]   s   € ð Œ
Ø$ŒØ,ÔØ&ÔØ"Žr   c           	     ó@  € VP                  R R4      pRRRWP                  ,          /RRRWP                  ,          /.pV P                  P	                  W2RR7      pV P
                  '       d/   \        V P                  P	                  VRR
 VRRR	7      4      pWE3# V^ 3# )rP   NrS   ÚuserÚcontentrT   FrQ   TrU   r   )rW   re   rf   r   rX   rH   r5   )r   r$   rP   rY   rZ   r[   s   &&    r   r%   ÚCompletionsDataset.processk   s·   € Ø—‘�g˜tÓ$ˆà�V˜Y¨¯/©/Õ(:Ð;Ø�[ )¨Q×/BÑ/BÕ-CÐDð
ˆð —‘×3Ñ3Ø¨uð 4ó 
ˆð ××ÐÜØ—‘×2Ñ2Ø˜S˜b�MØØ*.Ø %ð	 3ó óˆFð Ð#Ð#à˜ˆ{Ðr   c                ó    <€ V ^8„  d   QhRS[ /# r(   r*   )r   r   s   "€r   r   rg   �   r,   r   c                ó(   € V P                   V,          # r   r.   r/   s   &&r   r0   ÚCompletionsDataset.__getitem__�   r2   r   c                ó,   € \        V P                  4      # r   r4   r6   s   &r   r7   ÚCompletionsDataset.__len__„   r9   r   )r   rf   rH   re   r   Nr;   rC   s   @r   rc   rc   V   s-   ø‡ € ñ÷#ð #ò÷,ð ÷ð r   rc   c                   óP   a € ] tR t^ˆt o V 3R lR ltV 3R lR ltR tR tRtV t	R# )	ÚConcatenatedDatasetc                ó0   <€ V ^8„  d   QhRS[ S[,          /# ©r   r   )r   r   )r   r   s   "€r   r   Ú ConcatenatedDataset.__annotate__‰   s   ø€ ÷ 4ñ 4™T¡#�Yñ 4r   c                óT   € Wn         \        R  V P                    4       4      V n        R# )c              3   ó8   "  € T F  p\        V4      x € K  	  R # 5ir   )r5   )Ú.0r$   s   & r   Ú	<genexpr>Ú/ConcatenatedDataset.__init__.<locals>.<genexpr>‹   s   é € Ð3©
 1œ˜AŸ˜«
ùs   ‚N)r   ÚsumÚ_len©r   r   s   &&r   r   ÚConcatenatedDataset.__init__‰   s   € ØŒ
ÜÑ3¨¯
ª
Ó3Ó3ˆŽ	r   c                ó    <€ V ^8„  d   QhRS[ /# r(   r*   )r   r   s   "€r   r   rw   �   s   ø€ ÷ ñ ™sñ r   c                ó˜   € \        V P                  4       F"  w  r#V\        V4      ,
          pV^ 8  d    MTpK$  	  XV,          pXVR&   V# )r   Ú_dataset)Ú	enumerater   r5   )r   r)   Údata_idxr   ÚjÚdatums   &&    r   r0   ÚConcatenatedDataset.__getitem__�   sM   € Ü'¨¯
©
Ö3‰NˆHØ”c˜$“i•ˆAØ�1ŒuÙØŠCñ	 4ð
 �S•	ˆØ$ˆˆjÑØˆr   c                óT   € V P                   VR ,          ,          P                  V4      # )rƒ   )r   r%   r#   s   &&r   r%   ÚConcatenatedDataset.process—   s    € Ø�z‰z˜!˜J�-Õ(×0Ñ0°Ó3Ð3r   c                ó   € V P                   # r   )r~   r6   s   &r   r7   ÚConcatenatedDataset.__len__š   s   € Ø�y‰yÐr   )r   r~   N)
r<   r=   r>   r?   r   r0   r%   r7   rA   rB   rC   s   @r   rt   rt   ˆ   s(   ø‡ € ÷4ð 4÷ð ò4÷ð r   rt   c                   ó\   a € ] tR t^žt o V 3R lR ltV 3R lR ltV 3R lR ltR tRtV t	R	# )
ÚCacheDatasetc                ó    <€ V ^8„  d   QhRS[ /# rv   )r   )r   r   s   "€r   r   ÚCacheDataset.__annotate__Ÿ   s   ø€ ÷ -ñ -™Sñ -r   c                óB   € Wn         R .\        V4      ,          V n        R # r   )r   r5   Ú
_proc_datar   s   &&r   r   ÚCacheDataset.__init__Ÿ   s   € ØŒ
Ø˜&¤3 t£9Õ,ˆŽr   c                ó    <€ V ^8„  d   QhRS[ /# r(   r*   )r   r   s   "€r   r   r�   £   s   ø€ ÷ $ñ $™3ñ $r   c                ó:   € \        V P                  V,          4      # r   r4   r/   s   &&r   ÚitemlenÚCacheDataset.itemlen£   s   € Ü�4—:‘:˜c•?Ó#Ð#r   c                ó    <€ V ^8„  d   QhRS[ /# r(   r*   )r   r   s   "€r   r   r�   ¦   s   ø€ ÷ $ñ $™sñ $r   c                óÄ   € V P                   V,          f:   V P                  P                  V P                  V,          4      V P                   V&   V P                   V,          # r   )r’   r   r%   r/   s   &&r   r0   ÚCacheDataset.__getitem__¦   sE   € Ø�?‰?˜3ÕÒ'Ø#'§:¡:×#5Ñ#5°d·j±jÀµoÓ#FˆD�O‰O˜CÑ Ø�‰˜sÕ#Ð#r   c                ó,   € \        V P                  4      # r   r4   r6   s   &r   r7   ÚCacheDataset.__len__«   r9   r   )r   r’   N)
r<   r=   r>   r?   r   r–   r0   r7   rA   rB   rC   s   @r   rŽ   rŽ   ž   s-   ø‡ € ÷-ð -÷$ð $÷$ð $÷
ð r   rŽ   c                ó$   € V ^8„  d   QhR\         /# ©r   r   r   )r   s   "r   r   r   ¯   s   € ÷ 
ñ 
ä"ñ
r   c                 óP  € \        VR R4      p\        VRR4      p\        VRR4      p\        VRR4      p\        VRR	4      pV ^ ,          pWH9   d   Wh9   d   \        WWFV4      # Wx9   d   \        WWsR
7      # WX9   d!   V'       d   \        R4      h\	        WVR7      # \        R4      h)rH   FÚprompt_featureÚpromptÚtext_featurer:   Úcompletion_featureÚ
completionÚchat_featurerY   )rG   rH   z.Prompt masking not supported for text dataset.)r   z~Unsupported data format, check the supported formats here:
https://github.com/ml-explore/mlx-lm/blob/main/mlx_lm/LORA.md#Data.)Úgetattrrc   rE   Ú
ValueErrorr
   )	r   r   ÚconfigrH   r    r¢   r£   r¥   Úsamples	   &&&      r   Úcreate_datasetrª   ¯   sÇ   € ô
 ˜& -°Ó7€KÜ˜VÐ%5°xÓ@€NÜ˜6 >°6Ó:€LÜ  Ð)=¸|ÓLÐÜ˜6 >°:Ó>€LØ�!�W€FØÔÐ$6Ô$@Ü!Ø˜^Àó
ð 	
ð 
Ô	ÜØ lô
ð 	
ð 
Ô	ßÜÐMÓNÐNÜ˜4°\ÔBÐBäðRó
ð 	
r   c                ó0   € V ^8„  d   QhR\         R\        /# )r   Ú	data_pathr   )r   r   )r   s   "r   r   r   Í   s   € ÷ ñ Üðä"ñr   c                 óp   aa€ VV3R  lpRpV Uu. uF  qS! W R2,          4      NK  	  upw  rgpWgV3# u upi )c                 ó  <€ V P                  4       '       g   . # \        V R 4      ;_uu_ 4       pV Uu. uF  p\        P                  ! V4      NK  	  ppRRR4       \	        XSS4      # u upi   + '       g   i     L"; i)ÚrN)ÚexistsÚopenÚjsonÚloadsrª   )ÚpathÚfidÚlr   r¨   r   s   &   €€r   Úload_subsetÚ'load_local_dataset.<locals>.load_subsetÒ   s]   ø€ Ø�{‰{�}Š}ØˆIÜ�$˜�_Œ_ Ù+.Ó/©3 a”D—J’J˜q–M©3ˆDÐ/÷ ä˜d I¨vÓ6Ð6ùò 0÷ �_ús   ®A.³A)ÁA.Á)A.Á.A>	z.jsonl©ÚtrainÚvalidÚtest© )	r¬   r   r¨   r·   ÚnamesÚnrº   r»   r¼   s	   &ff      r   Úload_local_datasetrÀ   Í   sG   ù€ ö
7ð '€EÙINÓOÉÀA˜+ i°C°v°,Õ&>Ö?ÉÑOÑ€E�$Ø˜ÐÐùò Ps   �3c                ó0   € V ^8„  d   QhR\         R\        /# )r   Údata_idr   )r   r   )r   s   "r   r   r   Þ   s   € ÷ ñ Üðä"ñr   c                 óø   € ^ RI HpHp  V! V 4      pRpV Uu. uF+  pWuP                  4       9   d   \	        WW,          W4      M. NK-  	  upw  r‰p
Y‰T
3# u upi   TP
                   d    \        RT  R24      hi ; i)r   )Ú
exceptionsÚload_datasetz Not found Hugging Face dataset: z .r¹   )ÚdatasetsrÄ   rÅ   Úkeysrª   ÚDatasetNotFoundErrorr§   )rÂ   r   r¨   rÄ   rÅ   Údatasetr¾   r¿   rº   r»   r¼   s   &&&        r   Úload_hf_datasetrÊ   Þ   s£   € ÷
 2ðIÙ˜wÓ'ˆà*ˆñ ó
ñ �ð Ÿ™›Ô&ô ˜w�z¨9Ô=àòñ ñ
Ñˆ�dð ˜ÐÐùò
øð ×*Ñ*ô IÜÐ;¸G¸9ÀBÐGÓHÐHðIús   ŠA ˜1AÁ	A ÁA Á!A9c                ó$   € V ^8„  d   QhR\         /# rž   r   )r   s   "r   r   r   ù   s   € ÷ 9=ñ 9=Ô,?ñ 9=r   c           	      ó¶  aa€ ^ RI oVV3R lpV P                  p\        V\        4      '       d   V.p. pV Fç  pVR,          p\	        RV R24       \        V RR4      VR&   \        P                  ! R/ VB pVP                  R/ 4      pV P                  '       d<   VP                  R	R
4      p	VP                  RR4      p
V! VVV	V4      pV! VVV
V4      pM. . rËV P                  '       d   VP                  R4      pV! VVVV4      pM. pVP                  W¼V34       Ké  	  \        V4      ^8X  d
   V^ ,          # \        \        \        \!        V!  4      4      # )r   Nc                 óH   <€ SP                   ! V 3R V/VB p\        VSV4      # )Úsplit)rÅ   rª   )Údataset_namer¨   rÎ   Ú	hf_configÚdsrÆ   r   s   &&&& €€r   Úcreate_hf_datasetÚ1load_custom_hf_dataset.<locals>.create_hf_datasetü   s9   ø€ Ø×"Ò"Øñ
àð
ð ñ
ˆô
 ˜b )¨VÓ4Ð4r   r´   úLoading Hugging Face dataset Ú.rH   Fr¨   Útrain_splitztrain[:80%]Úvalid_splitztrain[-10%:]Ú
test_splitr½   )rÆ   Ú
hf_datasetÚ
isinstanceÚdictÚprintr¦   ÚtypesÚSimpleNamespacerW   rº   r¼   r"   r5   ÚtupleÚmaprt   Úzip)Úargsr   rÒ   Údataset_collectionÚ
collectionrÑ   Úds_pathr¨   rÐ   rÖ   r×   rº   r»   rØ   r¼   rÆ   s   &f             @r   Úload_custom_hf_datasetræ   ù   si  ù€ Ûö5ð Ÿ™ÐÜÐ$¤d×+Ò+Ø0Ð1Ðà€JÛ ˆØ�V•*ˆÜÐ-¨g¨Y°aÐ8Ô9Ü# D¨-¸Ó?ˆˆ=ÑÜ×&Ò&Ñ,¨Ñ,ˆØ—F‘F˜8 RÓ(ˆ	Ø�:�:ˆ:ØŸ&™& °Ó>ˆKØŸ&™& °Ó?ˆKÙ%ØØØØó	ˆEñ &ØØØØó	‰Eð ˜r�5à�9�9ˆ9ØŸ™ Ó-ˆJÙ$ØØØØó	‰Dð ˆDà×Ñ˜5¨Ð.Ö/ñG !ôJ ˆ:ƒ˜!ÔØ˜!�}Ðô ”Ô(¬#¨zÑ*:Ó;Ó<Ð<r   c                ó$   € V ^8„  d   QhR\         /# rž   r   )r   s   "r   r   r   5  s   € ÷ ñ Ô"5ñ r   c                 ó6  € \        V R R4      '       d   \        W4      w  r#pMm\        V P                  4      pVP	                  4       '       d   \        WQV 4      w  r#pM2\        RV P                   R24       \        V P                  W4      w  r#pV P                  '       d   \        V4      ^ 8X  d   \        R4      hV P                  '       d   \        V4      ^ 8X  d   \        R4       V P                  '       d   \        V4      ^ 8X  d   \        R4      hW#V3# )rÙ   FrÔ   rÕ   zKTraining set not found or empty. Must provide training set for fine-tuning.zUWarning: Validation set not found or empty. Training will proceed without validation.zBTest set not found or empty. Must provide test set for evaluation.)r¦   ræ   r   r   r°   rÀ   rÜ   rÊ   rº   r5   r§   r¼   )râ   r   rº   r»   r¼   r¬   s   &&    r   rÅ   rÅ   5  sí   € Üˆt�\ 5×)Ò)Ü3°DÓDÑˆ‘dä˜Ÿ™“Oˆ	Ø×Ñ×ÒÜ!3°IÈ$Ó!OÑˆE™$äÐ1°$·)±)°¸AÐ>Ô?Ü!0°·±¸IÓ!LÑˆE˜$à‡z‡z€z”c˜%“j A”oÜØYó
ð 	
ð ‡z‡z€z”c˜%“j A”oÜØcô	
ð ‡y‡y€y”S˜“Y !”^ÜØPó
ð 	
ð ˜ÐÐr   )r²   rÝ   Úpathlibr   Útypingr   r   r   Útransformersr   r
   rE   rc   rt   rŽ   rª   rÀ   rÊ   ræ   rÅ   r½   r   r   Ú<module>rì      sg   ðó Û Ý ß "Ñ "å ,÷ñ ÷8,ñ ,÷^/ñ /÷dñ ÷,ñ õ"
õ<õ"õ69=÷xr   