+
    TV-jf0  ã                   ó  € ^ RI t ^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RI	H
t
 ^ RIHt ^ RIt^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIHtHt ^ RIHt ^ R	IHtHtHtHtH t  R
 t!]PD                  RR3R R llt#RR R llt$R t%R# )é    N)ÚPath)Útree_map)Útqdm)Úload_dataset)Úkl_div_loss)Úgrad_checkpointÚiterate_batches)Úprint_trainable_parameters)ÚloadÚload_tokenizerÚpipeline_loadÚquantize_modelÚsavec                 ó    a aaaa€ \         P                  P                  4       P                  4       oVVV VV3R  lpV! W1R4       V! W!R4       R# )c                 óX  <€ S^ 8X  d   W,          pVP                  RRR7       \        \        \        V S
SSR7      4      \	        V 4      S
,          RV 2S^ 8g  R7      ;p FÅ  w  pw  rVVRRR13,          pS! V4      p\
        P                  ! V\
        P                  R7      p\
        P                  ! V4       S^ 8X  g   Kb  \
        P                  ! VRRR	7      R
RR13,          p\
        P                  ! WxRR7      pWR R2,          p	\
        P                  ! V	RVRV/4       KÇ  	  R# )r   T)ÚparentsÚexist_ok©ÚseedzComputing targets for )ÚtotalÚdescÚdisableºNNNN©Ústream)ÚkthÚaxis.©r   Ú010dú.safetensorsÚlogitsÚindiceséÿÿÿÿi üÿÿ)Úmkdirr   Ú	enumerater	   ÚlenÚmxÚstop_gradientÚcpuÚevalÚargpartitionÚtake_along_axisÚsave_safetensors)ÚdataÚpathÚsplitÚpbarÚiÚbatchÚ_r!   ÚidxÚfileÚ
batch_sizeÚmax_seq_lengthÚmodelÚrankr   s   &&&       €€€€€Úa/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/quant/dwq.pyÚ_compute_targetsÚ-compute_dwq_targets.<locals>._compute_targets(   s  ø€ à�1Œ9Ø•<ˆDØ�J‰J˜t¨dˆJÔ3äÜœ/¨$°
¸NÐQUÔVÓWÜ˜$“i :Õ-Ø-¨e¨WÐ5Ø ™	ô	ð ˆDò ñ ˆA‰z�ð ˜!˜S˜b˜S˜&•MˆEÙ˜5“\ˆFä×%Ò% f´R·V±VÔ<ˆFÜ�GŠG�FŒOØ�qŽyÜ—o’o f°%¸bÔAÀ#ÀuÁvÀ+ÕN�Ü×+Ò+¨F¸bÔA�à 4 ¨Ð5Õ5�Ü×#Ò# D¨8°V¸YÈÐ*LÖMó#ó    ÚvalidÚtrainN)r'   ÚdistributedÚinitr:   )	r9   Úsave_dirÚ
train_dataÚ
valid_datar7   r8   r   r<   r:   s	   f&&&fff @r;   Úcompute_dwq_targetsrF      sB   ü€ ô �>‰>×ÑÓ ×%Ñ%Ó'€D÷Nñ Nñ2 �Z¨7Ô3Ù�Z¨7Ö3r>   Fg       @c                óP   € V ^8„  d   QhR\         P                  R\        R\        /# )é   ÚdtypeÚgradient_checkpointÚtemperature)r'   ÚDtypeÚboolÚfloat)Úformats   "r;   Ú__annotate__rP   E   s3   € ÷ L>ñ L>ô �8‰8ðL>ô ðL>ô ñL>r>   c                 ó¶  a aaaaaaaa a!a"a#a$€ \         P                  P                  4       pVP                  4       o$VP	                  4       o!V!3R  lo"R pS P                  4        S P                  V4       \        S 4       V	'       d   \        S P                  ^ ,          4       ^V
,          o#VV V#3R lo V V3R lpVV VV"VVVV$3R lp\        R S P                  4       4      pRp^ p^ p\        P                  ! 4       pV! V^ R7      ;pp\        \        \        VSSSR7      4      \!        V4      S,          R	7      ;p EFÒ  w  pw  ppVR
RR13,          pS! VVRR7      p\         P"                  ! V4       V! VVVV4      w  ppp\         P"                  ! VV4       \         P                  P%                  V\         P&                  R7      P)                  4       S$,          p\         P                  P%                  V\         P&                  R7      P)                  4       pVV,          pVVV,          ,          pS!^ 8X  d²   VP+                  RVR 2R7       V^,           ^,          ^ 8X  d‡   V\        P                  ! 4       V,
          ,          p\         P,                  ! 4       R,          pVV,          pVV,          pS"! RV: RVR RV: RVR RVR 2
4       \        P                  ! 4       p^ p^ pV^,           ^È,          ^ 8X  g   EKÈ  V! VVR7      pEKÕ  	  V! VXR7      pVV8  d   S"! RVR RVR R24       S P/                  \        V3R lV4      4       R# )c                  óF   <€ S^ 8X  d   \         P                  ! V / VB  R# R# )r   N)r   Úwrite)ÚargsÚkwargsr:   s   *,€r;   ÚrprintÚdwq_quantize.<locals>.rprintV   s    ø€ Ø�1Œ9Ü�JŠJ˜Ð' Ô'ñ r>   c                 óÌ   € \        VR 4      '       dR   \        VR4      '       d>   VP                  R8X  d+   VP                  ^8  d   VP                  RR.RR7       R# R# R# R# R# )ÚbitsÚ
group_sizeÚaffineÚscalesÚbiasesF)ÚkeysÚrecurseN)ÚhasattrÚmoderY   Úunfreeze)r4   Úms   &&r;   rb   Údwq_quantize.<locals>.unfreezeZ   s\   € ä�A�v×ÒÜ˜˜<×(Ò(Ø—‘˜(Ô"Ø—‘˜”
à�J‰J˜X xÐ0¸%ˆJÖ@ñ ñ #ñ )ñ r>   c                 ó¼  <€ SP                  \        V
3R  lV 4      4       S! V4      p\        V\        4      '       d   Vw  r%\        P
                  ! WERR7      p\        SV,          SV,          4      p\        P                  ! ^^VP                  ^,          ,           4      VR,          8  pVP                  4       pWv,          P                  4       V,          p	W˜3# )c                 ó&   <€ V P                  S4      # ©N©Úastype©ÚxrI   s   &€r;   Ú<lambda>Ú/dwq_quantize.<locals>.loss_fn.<locals>.<lambda>m   s   ø€ ¨¯©°¬r>   r   r#   )r   :é   NN)
Úupdater   Ú
isinstanceÚtupler'   r,   r   ÚarangeÚshapeÚsum)Úparamsrk   ÚtargetsÚlengthsr!   ÚidsÚlossesÚmaskÚntoksÚlossrI   r9   Úscales   &&&&      €€€r;   Úloss_fnÚdwq_quantize.<locals>.loss_fnl   s©   ø€ Ø�‰”XÔ7¸Ó@ÔAÙ�q“ˆÜ�gœu×%Ò%Ø"‰LˆGÜ×'Ò'¨¸"Ô=ˆFÜ˜U V�^¨U°W­_Ó=ˆÜ�yŠy˜˜A §¡¨aÕ 0Õ0Ó1°G¸EµNÑBˆØ—‘“
ˆØ•×"Ñ"Ó$ uÕ,ˆØˆ{Ðr>   c                 óž   <€ \         P                  ! S4      ! W0W4      w  w  rEp\        P                  ! V4      pSP	                  Wc4      pWEV3# rg   )r'   Úvalue_and_gradÚnnÚaverage_gradientsÚapply_gradients)	Úinputsrv   rw   ru   r|   r{   Úgradsr~   Úopts	   &&&&   €€r;   ÚstepÚdwq_quantize.<locals>.stepx   sQ   ø€ Ü!×0Ò0°Ô9Ø˜Gó 
Ñ‰ˆ�uô ×$Ò$ UÓ+ˆØ×$Ñ$ UÓ3ˆØ˜FÐ"Ð"r>   c                 óŽ  <€ R p^ p\        \        \        SS
SSR7      4      \        S4      S
,          RRR7       Fî  w  pw  rVVRRR13,          pS! WTRR7      p\        P
                  ! V4       S! WWv4      w  r‰\        P
                  ! W‰4       \        P                  P                  V\        P                  R	7      P                  4       S,          p\        P                  P                  V	\        P                  R	7      P                  4       p	W9,          pW(V	,          ,          pKð  	  W#,          pS! R
V: RVR 24       V# )ç        r   zComputing validation lossF)r   r   Úleaver   Nr?   ©r0   r   zValidation: it=z, loss=ú.3fr#   )
r   r%   r	   r&   r'   r*   rA   Úall_sumr)   Úitem)ru   ÚitÚv_lossÚv_tokensr2   r3   rw   rv   r|   r{   r7   r~   r8   rV   r   Ú	target_fnrE   Ú
world_sizes   &&        €€€€€€€€r;   ÚvalidateÚdwq_quantize.<locals>.validate€   s  ø€ ØˆØˆÜ#'ÜÜ 
¨J¸ÈTÔRóô �j“/ ZÕ/Ø,Ø÷$
ÑˆAÑ�ð ˜!˜S˜b˜S˜&•MˆEÙ °Ô8ˆGÜ�GŠG�GÔÙ! &°ÓB‰KˆDÜ�GŠG�DÔ Ü—>‘>×)Ñ)¨$´r·v±vÐ)Ó>×CÑCÓEÈ
ÕRˆDÜ—N‘N×*Ñ*¨5¼¿¹Ð*Ó@×EÑEÓGˆEØÕˆHØ˜U•lÕ"ŠFñ!$
ð" Õ ˆÙÐ!˜b™U ( T¨3 KÐ0Ô1Øˆr>   c                 ó@   € V P                  \        P                  4      # rg   )ri   r'   Úfloat32)rk   s   &r;   rl   Údwq_quantize.<locals>.<lambda>š   s   € �!—(‘(œ2Ÿ:™:Ô&r>   r‹   )r‘   r   )r   r   Nr@   r�   r   zloss=z.4f)r   g    eÍÍAzit=z, avg_loss=z, total_tokens=z, toks_per_sec=rŽ   z, peak_memory_gb=u*   â�Œâ�Œâ�Œ
[WARNING] Final validation loss z' is worse than initial validation loss u2   . Model quality will likely be degraded.
â�Œâ�Œâ�Œc                 ó&   <€ V P                  S4      # rg   rh   rj   s   &€r;   rl   rš   Ñ   s   ø€  A§H¡H¨U¤Or>   r#   )r'   rA   rB   Úsizer:   r@   Úapply_to_modulesr
   r   Úlayersr   Útrainable_parametersÚtimer   r%   r	   r&   r*   r�   r)   r�   Úset_descriptionÚget_peak_memoryro   )%r9   r”   r‡   rD   rE   r7   r8   r   rI   rJ   rK   Úgrouprb   rˆ   r–   ru   Ú
total_lossÚtotal_tokensÚtokensÚticÚinitial_valid_lossÚ
valid_lossr1   r‘   r3   rw   rv   r|   r{   Útoks_per_secÚpeak_memory_gbÚavg_lossr~   r:   rV   r}   r•   s%   fff&fffff&&                     @@@@@r;   Údwq_quantizer­   E   s  ÿü€ ô �N‰N×ÑÓ!€EØ—‘“€JØ�:‰:‹<€Dõ(òAð 
‡K�K„MØ	×Ñ˜8Ô$Ü˜uÔ%çÜ˜Ÿ™ Q�Ô(à��O€E÷
ö#÷ô ô2 Ù&Ø×"Ñ"Ó$ó€Fð
 €JØ€LØ€Fä
�)Š)‹+€Cñ '/¨v¸!Ô&<Ð<Ð˜ô ÜÜ 
¨J¸ÈTÔRóô �j“/ ZÕ/ô	
ð 	
ˆó 	
ñ 	ˆÑˆU�Gð �a˜˜"˜�f•ˆÙ˜E 2¨WÔ5ˆÜ
�Š�ÔÙ" 5¨'°7¸FÓCÑˆˆe�VÜ
�Š��fÔÜ�~‰~×%Ñ% d´2·6±6Ð%Ó:×?Ñ?ÓAÀJÕNˆÜ—‘×&Ñ& u´R·V±VÐ&Ó<×AÑAÓCˆØ�%�ˆØ�d˜U•lÕ"ˆ
Ø�1Œ9Ø× Ñ ¨¨¨s¨ nÐ Ô5Ø�Q•˜"�} Ô!Ø%¬¯ª«°sÕ):Õ;�Ü!#×!3Ò!3Ó!5¸Õ!;�Ø%¨Õ.�Ø Õ&�ÙØ�r‘e˜<˜h¨˜_Ð,<¨|©oð >&Ø$ cÐ*Ð*<¨^¸SÐ,AðCôô —i’i“k�Ø�Ø�
Ø��F�c�>˜Q×Ù! &¨RÔ0‹Jñ?	
ñB ˜& RÔ(€JØ˜JÔ&ÙØ9¸*ÀSÐ9Ið J2Ø2DÀSÐ1Ið JAðAô	
ð 
‡L�L”Ô3°VÓ<Ö=r>   c                óH   € V ^8„  d   QhR\         R\        R\        R\        /# )rH   Ú	data_pathÚnum_samplesr8   Únum_valid_samples)ÚstrÚint)rO   s   "r;   rP   rP   Ô   s2   € ÷ ñ äðô ðô ð	ô
 ñr>   c                 óŽ  aa€ \         P                  ! R VRRRR/RRR7      p\        WP4      ^ ,          o\        P                  P                  \        S4      4      pVRV P                  4       pWbW$,            P                  4       pVV3R	 lp	V U
u. uF
  q©! V
4      NK  	  pp
V U
u. uF
  q©! V
4      NK  	  pp
W¼3# u up
i u up
i )
r/   Útrain_splitr@   Úvalid_splitz	train[:1]TF)Ú
hf_datasetr@   ÚtestNc                 óF   <€ SP                  SV ,          4      w  rVR S V3# rg   )Úprocess)r5   r¦   ÚoffsetÚdatasetr8   s   &  €€r;   rº   Úload_data.<locals>.processé   s)   ø€ Ø Ÿ™¨°­Ó6‰ˆØ�˜Ð'¨Ð0Ð0r>   )ÚtypesÚSimpleNamespacer   ÚnpÚrandomÚpermutationr&   Útolist)Ú	tokenizerr¯   r°   r8   r±   rT   ÚpermÚ
train_permÚ
valid_permrº   r2   r@   r?   r¼   s   &&&f&        @r;   Ú	load_datarÈ   Ô   sÌ   ù€ ô × Ò à�IØ˜7Ø˜;ð
ð
 Øô€Dô ˜4Ó+¨AÕ.€GÜ�9‰9× Ñ ¤ W£Ó.€DØ�l�{Ð#×*Ñ*Ó,€JØ KÕ$CÐD×KÑKÓM€Jö1ñ ",Ó,¡˜AˆW�QŽZ¡€EÐ,Ù!+Ó,¡˜AˆW�QŽZ¡€EÐ,Øˆ<Ðùò -ùÚ,s   ÂB=Â(Cc                  óâ
  aa€ \         P                  ! 4       p V P                  R RR\        RR7       V P                  R\        RRR7       V P                  R	R
RR7       V P                  R\        ^RR7       V P                  R\        ^@RR7       V P                  R\        RRR7       V P                  R\        RR7       V P                  R\        ^{R7       V P                  R\
        RR7       V P                  R\        ^R7       V P                  R\        RRR7       V P                  RRR R!7       V P                  R"\        RR#R7       V P                  R$RR%R!7       V P                  R&RR'R!7       V P                  4       p\        P                  P                  4       pVP                  pVP                  '       gM   W2P                  4       ,          ^ 8”  d2   W2P                  4       W2P                  4       ,          ,
          ,          p\        P                  P                  VP                  4       \        P                  P                  VP                  4       VP                   ey   \#        VP                   4      oSP%                  4       ;'       dK    \'        SR(,          P)                  R)4      4      ;'       d"    \'        SR*,          P)                  R)4      4      pMR+pRo\+        VP,                  4      p\/        WQP0                  VP                  VP2                  4      w  rgV'       d   VP4                  f_   VP                  '       d1   VP                  4       ^8”  d   \7        VP,                  RR,7      w  or‰M\9        VP,                  RRR-7      w  or‰MRoV'       g7   Se3   \;        SSVVVP<                  VP2                  VP                  R.7       RpVP>                  '       d   \A        ^ 4       V'       d   V3R/ lp
MV3R0 lp
VP4                  e/   \9        VP4                  RRR17      w  rµp	R2V	9  d   \C        R34      hM;\D        PF                  ! S4      p\I        VX	VPJ                  VPL                  R47      w  r‰V'       d   Se   >\        PN                  PQ                  4       '       d3   \        PR                  ! 4       R5,          p\        PT                  ! V4       \V        PX                  ! VPZ                  RR67      p\]        VV
VVVVP<                  VP2                  VP                  VP^                  R77	       \a        VPb                  VP,                  VVV	4       R# )8z--modelz-mz—A model to distill from for DWQ. If `quantized-model` is not given the student model will be this model quantized according to `bits` and `group-size`.T)ÚhelpÚtypeÚrequiredz--quantized-modelNzCAn already quantized model (the student model) to improve with DWQ.)rË   ÚdefaultrÊ   z
--mlx-pathÚ	mlx_modelz!Path to save the quantized model.)rÍ   rÊ   z--bitsz!Bits per weight for quantization.z--group-sizezGroup size for quantization.z--num-samplesi   z&Number of samples to use for training.z--max-seq-lengthi  )rË   rÍ   z--seedz--learning-rateg�íµ ÷Æ°>z--batch-sizez--data-pathzallenai/tulu-3-sft-mixturezIA Hugging Face dataset which is compatible with an mlx-lm dataset format.z--grad-checkpointÚ
store_truez0Use gradient checkpointing to reduce memory use.)ÚactionrÊ   z--target-dirzDirectory to save/load targets.z--targets-onlyzCompute the targets and exit.z
--pipelinez/Use pipeline parallel instead of data parallel.r@   z*.safetensorsr?   F)Úreturn_config)rÑ   Úlazy)r7   r8   r   c                 óx   <€ \         P                  ! SV,          VR  R2,          4      pVR,          VR,          3# )r   r    r!   r"   )r'   r   )r4   r5   r0   rv   Ú
target_dirs   &&& €r;   r”   Úmain.<locals>.target_fng  s9   ø€ Ü—g’g˜j¨5Õ0°c¸$°Z¸|Ð3LÕLÓMˆGØ˜8Õ$ g¨iÕ&8Ð8Ð8r>   c                 ó   <€ S! V 4      # rg   © )r3   r5   r0   r9   s   &&&€r;   r”   rÕ   m  s   ø€ Ù˜“<Ðr>   )rÒ   rÑ   Úquantizationz*Quantized model must already be quantized.)rZ   rY   Ú max_recommended_working_set_size)Úlearning_rateÚbias_correction)r7   r8   r   rJ   )2ÚargparseÚArgumentParserÚadd_argumentr²   r³   rN   Ú
parse_argsr'   rA   rB   r°   Úpipelinerœ   rÀ   rÁ   r   rÔ   r   Úis_dirÚanyÚglobr   r9   rÈ   r¯   r8   Úquantized_modelr   r   rF   r7   Útargets_onlyÚexitÚ
ValueErrorÚcopyÚdeepcopyr   rZ   rY   ÚmetalÚis_availableÚdevice_infoÚset_wired_limitÚ
optimizersÚAdamrÚ   r­   r   r   Úmlx_path)ÚparserrT   r£   r°   Úhas_targetsrÄ   rD   rE   r4   Úconfigr”   Úq_modelÚmax_rec_sizer‡   r9   rÔ   s                 @@r;   Úmainrö   ò   sÚ  ù€ Ü×$Ò$Ó&€FØ
×ÑØØð'ô Øð ô ð ×ÑØÜØØRð	 ô ð ×ÑØ˜kÐ0Sð ô ð ×ÑØÜØØ0ð	 ô ð ×ÑØœS¨"Ð3Qð ô ð ×ÑØÜØØ5ð	 ô ð ×ÑÐ*´¸dÐÔCØ
×Ñ˜¤s°CÐÔ8Ø
×ÑÐ)´¸tÐÔDØ
×Ñ˜¬S¸!ÐÔ<Ø
×ÑØÜØ,ØXð	 ô ð ×ÑØØØ?ð ô ð
 ×ÑØœS¨$Ð5Vð ô ð ×ÑØ Ð4Sð ô ð ×ÑØØØ>ð ô ð ×ÑÓ€Dä�N‰N×ÑÓ!€Eà×"Ñ"€KØ�=�=ˆ=˜[¯:©:«<Õ7¸!Ô;Ø—z‘z“| k·J±J³LÕ&@Õ@Õ@ˆä‡I�I‡N�N�4—9‘9ÔÜ‡I�I‡N�N�4—9‘9Ôà‡�Ò"Ü˜$Ÿ/™/Ó*ˆ
à×ÑÓ÷ Bð BÜ�Z 'Õ)×/Ñ/°Ó@ÓA÷Bð Bä�Z 'Õ)×/Ñ/°Ó@ÓAñ 	ð ˆØˆ
ä˜tŸz™zÓ*€Iä&Ø—>‘> 4×#3Ñ#3°T×5HÑ5HóÑ€J÷
 ˜$×.Ñ.Ò6Ø�=�=ˆ=˜UŸZ™Z›\¨AÔ-Ü,¨T¯Z©ZÀtÔLÑˆE�1�fä# D§J¡J¸dÈÔNÑˆE�1�fàˆ÷ ˜:Ò1ÜØØØØØ—‘Ø×.Ñ.Ø—‘õ	
ð ˆà××ÐÜˆQŒçö	9õ	 ð ×ÑÒ'Ü%)Ø× Ñ ØØô&
Ñ"ˆ˜Fð
  Ô'ÜÐIÓJÐJð (ô —-’- Ó&ˆÜ"ØØØ—‘Ø—‘ô	
‰	ˆ÷ �uÒ(Øä	‡x�x×Ñ×ÒÜ—~’~Ó'Ð(JÕKˆÜ
×Ò˜<Ô(ä
�/Š/¨×(:Ñ(:ÈDÔ
Q€CÜØØØØØØ—?‘?Ø×*Ñ*Ø�Y‰YØ ×0Ñ0õ
ô 	Ø�‰Ø�
‰
ØØØör>   )é    )&rÜ   rè   r    r¾   Úpathlibr   Úmlx.coreÚcorer'   Úmlx.nnr‚   Úmlx.optimizersrî   ÚnumpyrÀ   Ú	mlx.utilsr   r   Úmlx_lm.tuner.datasetsr   Úmlx_lm.tuner.lossesr   Úmlx_lm.tuner.trainerr   r	   Úmlx_lm.tuner.utilsr
   Úmlx_lm.utilsr   r   r   r   r   rF   Úbfloat16r­   rÈ   rö   r×   r>   r;   Ú<module>r     se   ðó Û Û Û Ý å Ý Ý #Û Ý Ý å .Ý +ß AÝ 9÷õ ò%4ðb —k‘kØ %Ø÷L>÷^ô<ir>   