+
    TV-j&1  ã                   ó  € ^ RI t ^ RIHtHt ^ RIHt ^ RIHt ^ RIH	t
 ^ RIHt ^ RIt^ RIHt ^ RIHtHt ^ RIHt ^RIHt ^R	IHt R
 R ltR t] ! R R4      4       tR tRR ltR]]^ 3R R lltR]! 4       ]]R3R R lltR# )é    N)Ú	dataclassÚfield)Úpartial)ÚPath)Úaverage_gradients)Útree_flattenÚtree_map)Útqdm)ÚTrainingCallback)ÚCacheDatasetc                ó$   € V ^8„  d   QhR\         /# )é   Ú	threshold)Úint)Úformats   "Úe/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/tuner/trainer.pyÚ__annotate__r      s   € ÷ ñ œCñ ó    c                 óh   € \         P                  ! 4       V 8”  d   \         P                  ! 4        R # R # ©N)ÚmxÚget_cache_memoryÚclear_cache)r   s   &r   Ú_clear_cacher      s"   € Ü	×ÒÓ˜yÔ(Ü
�ŠÖñ )r   c                ó^   a€ \        V 4      P                  oV3R lpV\        V 4      n        R# )zD
Update all instances of type(layer) to use gradient checkpointing.
c                 óp   <a € VV 3R  lp\         P                  ! V4      ! S P                  4       .VO5/ VB # )c                 ó>   <€ SP                  V 4       S! S.VO5/ VB # r   )Úupdate)ÚparamsÚargsÚkwargsÚfnÚmodels   &*,€€r   Úinner_fnÚ:grad_checkpoint.<locals>.checkpointed_fn.<locals>.inner_fn    s$   ø€ Ø�L‰L˜Ô Ù�eÐ-˜dÒ- fÑ-Ð-r   )r   Ú
checkpointÚtrainable_parameters)r#   r    r!   r$   r"   s   f*, €r   Úcheckpointed_fnÚ(grad_checkpoint.<locals>.checkpointed_fn   s1   ù€ ö	.ô �}Š}˜XÔ& u×'AÑ'AÓ'CÐUÀdÒUÈfÑUÐUr   N)ÚtypeÚ__call__)Úlayerr(   r"   s   & @r   Úgrad_checkpointr-      s(   ø€ ô 
ˆe‹×	Ñ	€BõVð +„DˆƒKÖr   c                   ó4  a € ] tR t^)t o ]! ^RR/R7      t]! ^dRR/R7      t]! ^RR/R7      t]! ^
RR/R7      t]! ^ÈRR/R7      t	]! ^dRR/R7      t
]! R	RR
/R7      t]! RRR/R7      t]! RRR/R7      t]! ^RR/R7      t]! ^ RR/R7      tV 3R ltRtV tR# )ÚTrainingArgsÚhelpzMinibatch size.)ÚdefaultÚmetadatazIterations to train for.z@Number of validation batches, -1 uses the entire validation set.z0Number of training steps between loss reporting.z-Number of training steps between validations.z!Save the model every number stepsé   zMaximum sequence length.zadapters.safetensorsz/Save/load path for the trained adapter weights.Fz0Use gradient checkpointing to reduce memory use.zLNumber of steps to accumulate gradients before applying an optimizer update.z>Clear the allocator cache between steps if it grows too large.c                óž   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[;R	&   S[ ;R
&   S[ ;R&   # )r   Ú
batch_sizeÚitersÚval_batchesÚsteps_per_reportÚsteps_per_evalÚsteps_per_saveÚmax_seq_lengthÚadapter_filer-   Úgrad_accumulation_stepsÚclear_cache_threshold)r   ÚstrÚbool)r   Ú__classdict__s   "€r   r   ÚTrainingArgs.__annotate__)   s¦   ø‡ ‚ áÑLñ ñ ÑRñ ñ ñ ñ	 ñ ñ ñ ñ ñ ñ ñ" ñ ñ# ñ( ñ ñ) ñ. ñ ñ/ ñ6 ñ ñ7 ñ> !ñ ñ? ñJ ñ òK r   © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r5   r6   r7   r8   r9   r:   r;   r<   r-   r=   r>   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)rA   s   @r   r/   r/   )   s  ø‡ € á A°Ð9JÐ0KÔL€JÙ˜s¨fÐ6PÐ-QÔR€EÙØàÐVð
ô€Kñ "ØØÐLÐMôÐñ  Ø˜vÐ'VÐWô€Nñ  Ø˜vÐ'JÐKô€Nñ  Ø Ð(BÐCô€Nñ Ø&ØÐKÐLô€Lñ "ØØÐLÐMô€Oñ $)ØàÐbð
ô$Ðñ "'ØàÐTð
ô"Ð÷K ƒ r   r/   c                 óÈ  € VR RR13,          pVR,          pV ! V4      p\         P                  ! ^VP                  ^,          ^,           4      p\         P                  ! WbR,          8¬  WbR,          8*  4      p\        P
                  P                  WT4      V,          pVP                  4       p	VP                  \         P                  4      P                  4       V	,          pW‰3# )ºNNNNéÿÿÿÿ)rL   :é   NN)rL   :r   rN   N)
r   ÚarangeÚshapeÚlogical_andÚnnÚlossesÚcross_entropyÚsumÚastypeÚfloat32)
r#   ÚbatchÚlengthsÚinputsÚtargetsÚlogitsÚstepsÚmaskÚceÚntokss
   &&&       r   Údefault_lossra   V   s­   € Ø�1�c�r�c�6�]€FØ�E�l€Gá�6‹]€Fä�IŠI�a˜Ÿ™ qÕ)¨AÕ-Ó.€EÜ�>Š>˜%¨6¥?Ñ2°EÀU½^Ñ4KÓL€Dä	�‰×	 Ñ	  Ó	1°DÕ	8€BØ�H‰H‹J€EØ	�‰”2—:‘:Ó	×	"Ñ	"Ó	$ uÕ	,€Bàˆ9Ðr   c           
   #   ó
  a "  € \        S \        4      '       d   V 3R  lpMV 3R lp\        \        \	        S 4      4      VR7      p\	        S 4      V8  d   \        RV R\	        S 4       R24      hVe"   VP                  4       pVP                  4       p	M^ p^p	W,          ^ 8w  d   \        R4      h\        ^ \	        V4      V,
          ^,           V4       U
u. uF"  p
WzV,           W¨,           V,           V	1,          NK$  	  pp
V'       d    \        P                  P                  V4        \        P                  P                  \	        V4      4      pV EFŽ  p
Wº,           Uu. uF  pS V,          NK  	  pp\	        V^ ,          4      ^8X  d   \        V!  w  rïM^ .\	        V4      ,          pV Uu. uF  p\	        V4      NK  	  pp\        V4      V8”  d   \        RV R	\        V4       R
V R24       ^ p^V\        V4      V,           ^,
          V,          ,          ,           p\        VV4      p\        P                   ! W,          V3\        P"                  4      p\        W,          4       F-  p\        VV,          V4      pWí,          RV VVRV13&   VVV&   K/  	  \$        P&                  ! V4      pV\$        P&                  ! \)        \        VV4      4      4      3x € EK‘  	  V'       d   EKÉ  R# u up
i u upi u upi 5i)c                 ó&   <€ SP                  V 4      # r   )Úitemlen©ÚidxÚdatasets   &€r   Ú<lambda>Ú!iterate_batches.<locals>.<lambda>p   s   ø€ ˜WŸ_™_¨SÔ1r   c                 ó6   <€ \        SV ,          ^ ,          4      # )r   )Úlenre   s   &€r   rh   ri   r   s   ø€ œS ¨¥¨a¥Ô1r   )Úkeyz&Dataset must have at least batch_size=z examples but only has Ú.Nz9The batch size must be divisible by the number of workersz)[WARNING] Some sequences are longer than z tokens. The longest sentence z will be truncated to z2. Consider pre-splitting your data to save memory.)Ú
isinstancer   ÚsortedÚrangerk   Ú
ValueErrorÚrankÚsizeÚnpÚrandomÚseedÚpermutationÚzipÚmaxÚprintÚminÚzerosÚint32r   ÚarrayÚlist)rg   r5   r;   Úlooprv   Ú
comm_groupÚlen_fnrf   ÚoffsetÚstepÚiÚ	batch_idxÚindicesÚjrX   ÚoffsetsÚxrY   Úpad_toÚmax_length_in_batchÚ	batch_arrÚtruncated_lengths   f&&&&&                r   Úiterate_batchesr�   f   sÃ  øé € ô �'œ<×(Ò(Ü1‰ä1ˆÜ
””s˜7“|Ó$¨&Ô
1€CÜ
ˆ7ƒ|�jÔ ÜØ4°Z°LØ%¤c¨'£l ^°1ð6ó
ð 	
ð ÒØ—‘Ó"ˆØ�‰Ó ‰àˆØˆØÕ˜AÔÜÐTÓUÐUô
 �qœ#˜c›( ZÕ/°!Õ3°ZÔ@óá@ˆAð 	��J˜� jÕ0°4Ð7×8Ð8Ù@ð ð ÷ Ü
�	‰	�‰�tÔØ
Ü—)‘)×'Ñ'¬¨I«Ó7ˆÜˆAØ)2®Ó6© A�W˜Q—Z�Z©ˆEÐ6Ü�5˜•8‹} Ô!Ü!$ e¡‘��wà˜#¤ E£
Õ*�Ù',Ó-¡u !”s˜1–v¡uˆGÐ-Ü�7‹|˜nÔ,ÜØ?ÀÐ?Oð P,Ü,/°«L¨>Ð9OÐP^ÐO_ð `GðGôð ˆFØ"# f´°W³ÀÕ1FÈÕ1JÈvÕ0UÕ&VÕ"VÐÜ"%Ð&9¸>Ó"JÐäŸš *Õ"4Ð6IÐ!JÌBÏHÉHÓUˆIä˜:Õ-Ö.�Ü#& w¨q¥z°>Ó#BÐ Ø27µ(Ð;LÐ<LÐ2M�	˜!Ð.Ð.Ð.Ð.Ñ/à$ð ˜“
ñ /ô —H’H˜YÓ'ˆEØœŸš¤$¤s¨7°GÓ'<Ó"=Ó>Ð>Õ>ñ9 ÷< ŠtÙùòOùò 7ùò
 .ùs2   ƒCLÃ(K4Ä A"LÅ"K9Å4<LÆ0K>ÇD(LË2Lr3   c                ó<   € V ^8„  d   QhR\         R\         R\        /# )r   Úlossr�   r>   )Úcallabler   )r   s   "r   r   r   °   s*   € ÷ 'ñ 'ô ð'ô ð'ô ñ'r   c                 ó  € V P                  4        \        P                  ! R 4      p\        P                  ! ^ 4      p	VR8w  d   \        \	        V4      4      M\        \
        ^4      p
\        \        V
V! VVV\        P                  P                  4       R7      4      R\        \        V4      V,          V4      R7       FI  w  r¼V! V .VO5!  w  rÞW�V,          ,          pWž,          p	\        P                   ! W‰4       \        V4       KK  	  \        P                  P                  V\        P                  R7      p\        P                  P                  V	\        P                  R7      p	W‰,          P                  4       pV# )g        )rg   r5   r;   r�   zCalculating loss...)ÚdescÚtotal©ÚstreamrM   )Úevalr   r~   Úiterrp   r   r
   rx   ÚdistributedÚinitr{   rk   r   Úall_sumÚcpuÚitem)r#   rg   r5   Únum_batchesr;   r‘   r�   r>   Ú
all_lossesÚntokensÚindex_iteratorÚ_rX   rS   ÚtoksÚavg_losss   &&&&&&&&        r   Úevaluater¦   °   s(  € ð 
‡J�J„LÜ—’˜#“€JÜ�hŠh�q‹k€Gà1<ÀÔ1B”Tœ% Ó,Ô-ÌÌSÐRSË€NäÜØÙØØ%Ø-ÜŸ>™>×.Ñ.Ó0ô	ó	
ð #Ü”#�g“, *Õ,¨kÓ:÷‰ˆñ ˜EÐ* EÓ*‰ˆØ˜t•mÕ#ˆ
Ø�ˆÜ
�Š�
Ô$ÜÐ*Ö+ñ#ô& —‘×'Ñ'¨
¼2¿6¹6Ð'ÓB€JÜ�n‰n×$Ñ$ W´R·V±VÐ$Ó<€GØÕ$×*Ñ*Ó,€Hà€Or   c                óH   € V ^8„  d   QhR\         R\        R\        R\        /# )r   r    r‘   r�   Útraining_callback)r/   r’   r   )r   s   "r   r   r   Ú   s:   € ÷ i>ñ i>ô
 ði>ô ði>ô ði>ô (ñi>r   c                 óŽ  a aa#a$€ \         P                  P                  4       '       d1   \         P                  ! \         P                  ! 4       R ,          4       \        RVP                   24       \         P                  P                  4       pVP                  4       p	VP                  4       p
V	^8”  d   \        RV
 RV	 24       VP                  '       d   \        S P                  ^ ,          4       \        P                  ! S V4      o$VP                  o#S#^8  d   \!        R4      hS P"                  SP"                  \         P$                  P"                  .p\'        \         P(                  W»R7      V#V$V V3R l4       pS P+                  4        ^ p^ p^ p^ p^ pRp\-        \/        ^VP                  ^,           4      V! VVP0                  VP2                  RVR	7      4       EFî  w  pp\4        P6                  ! 4       pV'       dû   V^8X  g*   VVP8                  ,          ^ 8X  g   VVP                  8X  dË   \4        P6                  ! 4       p\;        S VVVP0                  VP<                  VP2                  VR
7      pS P+                  4        \4        P6                  ! 4       V,
          pV
^ 8X  d   \        RV RVR RVR R2RR7       Ve!   RV^,
          RVRV/pVP?                  V4       \4        P6                  ! 4       pV! VVVS#,          ^ 8H  4      w  pppVV,          pVV,          pV^,          p\         P@                  ! W½VV4       \C        VPD                  4       V\4        P6                  ! 4       V,
          ,          pVVPF                  ,          ^ 8X  g   VVP                  8X  EdJ   \         P                  PI                  V\         PJ                  R7      PM                  4       pVWù,          ,          p\         P                  PI                  V\         PJ                  R7      PM                  4       pSPN                  PM                  4       pVPF                  V,          p\Q        V4      V,          pVV,          p\         PR                  ! 4       R,          pV
^ 8X  d)   \        RV RVR RVR RVR RVR RV RVR R2RR7       Ve"   RVRVRVR VR!VR"VR#V/p VPU                  V 4       ^ p^ p^ p^ pVVPV                  ,          ^ 8X  g   EK/  V
^ 8X  g   EK9  \Y        \[        S P]                  4       4      4      p!\         P^                  ! \a        VPb                  4      V!4       \e        VPb                  4      Pf                  VR$ R%2,          p"\         P^                  ! \a        V"4      V!4       \        RV R&VPb                   R'V" R(24       EKñ  	  V
^ 8X  dh   \Y        \[        S P]                  4       4      4      p!\         P^                  ! \a        VPb                  4      V!4       \        R)VPb                   R(24       R# R# )*Ú max_recommended_working_set_sizezStarting training..., iters: zNode z of z*grad_accumulation_steps must be at least 1)rZ   Úoutputsc                 óÈ   <€ S! S.V O5!  w  w  r4pVe   \        R WQ4      pV'       d7   \        V4      pS^8”  d   \        V3R lV4      pS	P                  SV4       R pW4V3# )Nc                 ó   € W,           # r   rC   )rŠ   Úys   &&r   rh   Ú%train.<locals>.step.<locals>.<lambda>ý   s   € ¨®r   c                 ó   <€ V S,          # r   rC   )rŠ   Úgrad_accum_stepss   &€r   rh   r¯     s   ø€ ¨!Ð.>Ö*>r   )r	   r   r   )
rX   Ú	prev_gradÚ	do_updateÚlvaluer¤   Úgradr±   Úloss_value_and_gradr#   Ú	optimizers
   &&&   €€€€r   r„   Útrain.<locals>.stepø   sn   ø€ á2°5ÐA¸5ÓAÑ‰ˆ˜àÒ ÜÑ.°Ó@ˆDçÜ$ TÓ*ˆDØ !Ô#ÜÔ >ÀÓE�Ø×Ñ˜U DÔ)ØˆDà˜TÐ!Ð!r   NT)rg   r5   r;   r€   r�   )r#   rg   r‘   r5   rŸ   r;   r�   zIter z: Val loss z.3fz, Val took Ús)ÚflushÚ	iterationÚval_lossÚval_timer–   g    eÍÍAz: Train loss z, Learning Rate z.3ez	, It/sec z, Tokens/sec z, Trained Tokens z, Peak mem z GBÚ
train_lossÚlearning_rateÚiterations_per_secondÚtokens_per_secondÚtrained_tokensÚpeak_memoryÚ07dz_adapters.safetensorsz: Saved adapter weights to z and rm   zSaved final weights to )4r   ÚmetalÚis_availableÚset_wired_limitÚdevice_inforz   r6   rš   r›   rs   rr   r-   ÚlayersrR   Úvalue_and_gradr=   rq   Ústateru   r   ÚcompileÚtrainrx   rp   r5   r;   ÚtimeÚperf_counterr9   r¦   r7   Úon_val_loss_reportr˜   r   r>   r8   rœ   r�   rž   r¿   ÚfloatÚget_peak_memoryÚon_train_loss_reportr:   Údictr   r'   Úsave_safetensorsr?   r<   r   Úparent)%r#   r·   Útrain_datasetÚval_datasetr    r‘   r�   r¨   ÚworldÚ
world_sizerr   rË   r„   rS   Ún_tokensr]   rÂ   Ú
train_timeÚ
grad_accumÚitrX   Úticr¼   r½   Úval_infor´   r¤   r¾   r¿   Úit_secÚ
tokens_secÚpeak_memÚ
train_infoÚadapter_weightsr&   r±   r¶   s%   ff&&&&&&                           @@r   rÍ   rÍ   Ú   s¥  û€ ô 
‡x�x×Ñ×ÒÜ
×Òœ2Ÿ>š>Ó+Ð,NÕOÔPÜ	Ð)¨$¯*©*¨Ð
6Ô7Ü�N‰N×ÑÓ!€EØ—‘“€JØ�:‰:‹<€DØ�A„~Ü��d�V˜4 
˜|Ð,Ô-à××ÐÜ˜Ÿ™ Q�Ô(ä×+Ò+¨E°4Ó8Ðà×3Ñ3ÐØ˜!ÔÜÐEÓFÐFà�[‰[˜)Ÿ/™/¬2¯9©9¯?©?Ð;€EäŒR�Z‰Z Ô5÷"ó 6ð"ð 
‡K�K„MØ€FØ€HØ€EØ€NØ€JØ€Jô Üˆa�—‘˜a•Ó ÙØ!Ø—‘Ø×.Ñ.ØØô	
÷	‰	ˆˆEô ×ÒÓ!ˆ÷ Ø�!ŒG�r˜D×/Ñ/Õ/°1Ô4¸¸d¿j¹jÔ8Hä×#Ò#Ó%ˆCÜØØ#ØØŸ?™?Ø ×,Ñ,Ø#×2Ñ2Ø /ôˆHð �K‰KŒMÜ×(Ò(Ó*¨SÕ0ˆHØ�qŒyÜØ˜B˜4ð   Ø (¨˜~ð . Ø (¨˜~¨Qð0ð õ	ð !Ò,à  a¥Ø Ø ð�ð
 "×4Ñ4°XÔ>ä×#Ò#Ó%ˆCá#'ØØØÐ!Õ! QÑ&ó$
Ñ ˆ��jð 	�&ÕˆØ�DÕˆØ��
ˆÜ
�Š�˜x¨Ô4Ü�T×/Ñ/Ô0Ø”d×'Ò'Ó)¨CÕ/Õ/ˆ
ð �×%Ñ%Õ%¨Ô*¨b°D·J±JÕ.>ÜŸ™×/Ñ/°¼r¿v¹vÐ/ÓF×KÑKÓMˆJØ˜%Õ,Õ,ˆJÜ—~‘~×-Ñ-¨h¼r¿v¹vÐ-ÓF×KÑKÓMˆHØ%×3Ñ3×8Ñ8Ó:ˆMØ×*Ñ*¨ZÕ7ˆFÜ˜x›¨:Õ5ˆJØ˜hÕ&ˆNÜ×)Ò)Ó+¨cÕ1ˆHØ�qŒyÜØ˜B˜4˜}¨Z¸Ð,<ð =%Ø%2°3Ð$7ð 8Ø$ S˜\ð *"Ø",¨SÐ!1ð 2&Ø&4Ð%5ð 6 Ø (¨˜~¨Sð2ð õð !Ò,à Ø  *Ø# ]Ø+¨VØ'¨Ø$ nØ! 8ð�
ð "×6Ñ6°zÔBàˆFØˆHØˆEØˆJð �×#Ñ#Õ# q×(¨T°Q¯YÜ"¤<°×0JÑ0JÓ0LÓ#MÓNˆOÜ×Ò¤ D×$5Ñ$5Ó 6¸ÔHä�T×&Ñ&Ó'×.Ñ.°B°s°8Ð;PÐ1QÕQð ô ×Ò¤ J£°ÔAÜØ˜�tÐ6Ø×$Ñ$Ð% U¨:¨,°að9÷ñS	ð^ ˆq„yÜœ|¨E×,FÑ,FÓ,HÓIÓJˆÜ
×ÒœC × 1Ñ 1Ó2°OÔDÜÐ'¨×(9Ñ(9Ð':¸!Ð<Ö=ñ r   )FNN) rÎ   Údataclassesr   r   Ú	functoolsr   Úpathlibr   Úmlx.coreÚcorer   Úmlx.nnrR   Únumpyrt   Úmlx.nn.utilsr   Ú	mlx.utilsr   r	   r
   Ú	callbacksr   Údatasetsr   r   r-   r/   ra   r�   r¦   rÍ   rC   r   r   Ú<module>rñ      sŽ   ðó ß (Ý Ý å Ý Û Ý *ß ,Ý å 'Ý "õò
+ð  ÷)ð )ó ð)òXô Gð^ Ø!Ø /Ø!"÷'ð\ Ù%›Ø!Ø /Ø*.÷i>ñ i>r   