
    ir/              	          d dl Z d dlmZmZ d dlmZ d dlmZ d dlm	Z
 d dlmZ d dlZd dlmZ d dlmZmZ d dlmZ dd	lmZ dd
lmZ d Ze G d d             Zd Z	 	 	 ddZdeefdedefdZd e       eedfdedededefdZy)    N)	dataclassfield)partial)Path)average_gradients)tree_flattentree_map)tqdm   )TrainingCallback)CacheDatasetc                 Z    t        |       j                  fd}|t        |       _        y)zL
    Update all instances of type(layer) to use gradient checkpointing.
    c                 l      fd} t        j                  |       j                         g|i |S )Nc                 >    j                  |         g|i |S N)update)paramsargskwargsfnmodels      i/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/tuner/trainer.pyinner_fnz:grad_checkpoint.<locals>.checkpointed_fn.<locals>.inner_fn   s$    LL e-d-f--    )mx
checkpointtrainable_parameters)r   r   r   r   r   s   `   r   checkpointed_fnz(grad_checkpoint.<locals>.checkpointed_fn   s3    	. 'r}}X&u'A'A'CUdUfUUr   N)type__call__)layerr   r   s     @r   grad_checkpointr"      s(     
e		BV +DKr   c                   b   e Zd ZU  edddi      Zeed<    edddi      Zeed<    ed	dd
i      Zeed<    edddi      Z	eed<    edddi      Z
eed<    edddi      Zeed<    edddi      Zeed<    edddi      Zeed<    edddi      Zeed<    edddi      Zeed<   y )!TrainingArgs   helpzMinibatch size.)defaultmetadata
batch_sized   zIterations to train for.iters   z@Number of validation batches, -1 uses the entire validation set.val_batches
   z0Number of training steps between loss reporting.steps_per_report   z-Number of training steps between validations.steps_per_evalz!Save the model every number stepssteps_per_save   zMaximum sequence length.max_seq_lengthzadapters.safetensorsz/Save/load path for the trained adapter weights.adapter_fileFz0Use gradient checkpointing to reduce memory use.r"   r   zLNumber of steps to accumulate gradients before applying an optimizer update.grad_accumulation_stepsN)__name__
__module____qualname__r   r)   int__annotations__r+   r-   r/   r1   r2   r4   r5   strr"   boolr6    r   r   r$   r$   $   s)   A9J0KLJLsf6P-QRE3RV
K  "LMc   v'VWNC   v'JKNC   (BCNC  &KLL#  "LMOT  $)b
$S r   r$   c                    |d d d df   }|d d dd f   } | |      }t        j                  d|j                  d   dz         }t        j                  ||d d ddf   k\  ||d d dd f   k        }t        j
                  j                  ||      |z  }|j                         }	|j                  t         j                        j                         |	z  }||	fS )Nr   r   )
r   arangeshapelogical_andnnlossescross_entropysumastypefloat32)
r   batchlengthsinputstargetslogitsstepsmaskcentokss
             r   default_lossrS   K   s    1crc6]FAqrElG6]FIIaq)A-.E>>%71ac6?2EWQU^4KLD		 	 	1D	8BHHJE	2::		"	"	$u	,Bu9r   c           
   #      K   t         t              r fd}n fd}t        t        t	                     |      }t	               |k  rt        d| dt	                d      |!|j                         }|j                         }	nd}d}	||	z  dk7  rt        d	      t        dt	        |      |z
  dz   |      D 
cg c]  }
||
|z   |
|z   |z   |	    }}
|rt        j                  j                  |       	 t        j                  j                  t	        |            }|D ]P  }
||
   D cg c]  } |   	 }}t	        |d         d
k(  rt        | \  }}ndgt	        |      z  }|D cg c]  }t	        |       }}t        |      |kD  rt        d| dt        |       d| d       d}d|t        |      |z   dz
  |z  z  z   }t        ||      }t        j                   ||	z  |ft        j"                        }t        ||	z        D ]%  }t        ||   |      }||   d | ||d |f<   |||<   ' t%        j&                  |      }|t%        j&                  t)        t        ||                  f S |sy c c}
w c c}w c c}w w)Nc                 &    j                  |       S r   )itemlenidxdatasets    r   <lambda>z!iterate_batches.<locals>.<lambda>e   s    W__S1r   c                 &    t        |    d         S )Nr   )lenrW   s    r   rZ   z!iterate_batches.<locals>.<lambda>g   s    Sa1r   )keyz&Dataset must have at least batch_size=z examples but only has .r   r   z9The batch size must be divisible by the number of workers   z)[WARNING] Some sequences are longer than z tokens. The longest sentence z will be truncated to z2. Consider pre-splitting your data to save memory.    )
isinstancer   sortedranger\   
ValueErrorranksizenprandomseedpermutationzipmaxprintminzerosint32r   arraylist)rY   r)   r4   loopri   
comm_grouplen_fnrX   offsetstepi	batch_idxindicesjrJ   offsetsxrK   pad_tomax_length_in_batch	batch_arrtruncated_lengths   `                     r   iterate_batchesr   [   s     '<(11
s7|$&
1C
7|j 4ZL%c'l^16
 	
 " DATUU
 q#c(Z/!3Z@@A 	AJVj0478@   
		t
))''I7A)216AWQZE658}!!$ew#E
*',-u!s1vuG-7|n,??O P,,/L>9OP^O_ `GG F"#fW1F1Jv0U&V"V"%&9>"J*"46I!JBHHUI:-.#&wqz>#B 27(;L<L2M	!..../$ 
 / HHY'E$s7G'<"=>>>9 < C  7
 .s,   B<J?I>AJ0J<2J.J DJr3   lossr   c                    | j                          t        j                  d      }t        j                  d      }|dk7  rt        t	        |            nt        t
        d      }	t        t        |	 ||||t        j                  j                                     dt        t        |      |z  |            D ]4  \  }
} || g| \  }}|||z  z  }||z  }t        j                   ||       6 t        j                  j                  |t        j                        }t        j                  j                  |t        j                        }||z  j                         S )	Ng        r   r@   r   )rY   r)   r4   rt   zCalculating loss...)desctotalstream)evalr   rq   iterrc   r:   r
   rk   distributedinitrn   r\   all_sumcpuitem)r   rY   r)   num_batchesr4   r   r   
all_lossesntokensindex_iterator_rJ   rE   tokss                 r   evaluater      s'    
JJL#JhhqkG1<1BT%,-SRSN%->>..0		
 ##g,*,k:5 E*E*ftm#
4

G$!$ ''
266'BJnn$$WRVV$<G &&((r   r   training_callbackc                 .   #$ t         j                  j                         r*t        j                  t        j                         d          t        d|j                          t         j                  j                         }|j                         }	|j                         }
|	dkD  rt        d|
 d|	        |j                  rt         j                  d          t        j                   |      $|j                  ##dk  rt!        d       j"                  j"                  t         j$                  j"                  g}t'        t         j(                  ||      #$ fd	       } j+                          d}d}d}d}d}d }t-        t/        d|j                  dz          |||j0                  |j2                  d
|            D ]c  \  }}t5        j6                         }|r|dk(  s!||j8                  z  dk(  s||j                  k(  rt5        j6                         }t;         |||j0                  |j<                  |j2                  |      } j+                          t5        j6                         |z
  }|
dk(  rt        d| d|dd|ddd
       ||dz
  ||d}|j?                  |       t5        j6                         } ||||#z  dk(        \  }}}||z  }||z  }|dz  }t        j@                  ||||       |t5        j6                         |z
  z  }||jB                  z  dk(  s||j                  k(  r't         j                  jE                  |t         jF                        jI                         }|||	z  z  }t         j                  jE                  |t         jF                        jI                         }jJ                  jI                         }|jB                  |z  }tM        |      |z  }||z  }t        jN                         dz  }|
dk(  r(t        d| d|dd|dd|dd|dd| d|ddd
       ||||||||d} |jQ                  |        d}d}d}d}||jR                  z  dk(  s|
dk(  stU        tW         jY                                     }!t        jZ                  t]        |j^                        |!       ta        |j^                        jb                  |dd z  }"t        jZ                  t]        |"      |!       t        d| d!|j^                   d"|" d#       f |
dk(  retU        tW         jY                                     }!t        jZ                  t]        |j^                        |!       t        d$|j^                   d#       y y )%N max_recommended_working_set_sizezStarting training..., iters: r   zNode z of r   z*grad_accumulation_steps must be at least 1)rL   outputsc                      g|  \  \  }}}|t        d ||      }|r3t        |      }dkD  rt        fd|      }	j                  |       d }|||fS )Nc                     | |z   S r   r>   )r}   ys     r   rZ   z%train.<locals>.step.<locals>.<lambda>   s    Qr   r   c                     | z  S r   r>   )r}   grad_accum_stepss    r   rZ   z%train.<locals>.step.<locals>.<lambda>   s    !.>*>r   )r	   r   r   )
rJ   	prev_grad	do_updatelvaluer   gradr   loss_value_and_gradr   	optimizers
         r   rw   ztrain.<locals>.step   st    25A5A .i@D$T*D!# >EUD)DtT!!r   T)rY   r)   r4   rs   rt   )r   rY   r   r)   r   r4   r   zIter z: Val loss z.3fz, Val took s)flush)	iterationval_lossval_timer   g    eAz: Train loss z, Learning Rate z.3ez	, It/sec z, Tokens/sec z, Trained Tokens z, Peak mem z GB)r   
train_losslearning_rateiterations_per_secondtokens_per_secondtrained_tokenspeak_memory07dz_adapters.safetensorsz: Saved adapter weights to z and r^   zSaved final weights to )2r   metalis_availableset_wired_limitdevice_inform   r+   r   r   rf   re   r"   layersrD   value_and_gradr6   rd   staterh   r   compiletrainrk   rc   r)   r4   timeperf_counterr1   r   r-   on_val_loss_reportr   r/   r   r   r   r   floatget_peak_memoryon_train_loss_reportr2   dictr   r   save_safetensorsr<   r5   r   parent)%r   r   train_datasetval_datasetr   r   r   r   world
world_sizere   r   rw   rE   n_tokensrO   r   
train_time
grad_accumitrJ   ticr   r   val_infor   r   r   r   it_sec
tokens_secpeak_mem
train_infoadapter_weightsr   r   r   s%   ``                                 @@r   r   r      s    
xx
2>>+,NOP	)$**
67NN!EJ::<DA~dV4
|,-Q(++E4833!EFF[[)//299??;ERZZu5" 6" 
KKMFHENJJ aa !..	
		E ! !GrD///14djj8H##%C#?? ,,#22 /H KKM((*S0HqyB4    (~ .  (~Q0 	 !,!#a ( (
 "44X>##%C#'!!Q&$
 j 	&D

vx4d'')C//
 %%%*bDJJ.>//rvv/FKKMJ%*,,J~~--hrvv-FKKMH%3388:M**Z7Fx:5Jh&N))+c1HqyB4}Z,< =%%23$7 8$S\ *"",S!1 2&&4%5 6  (~S2  !,!#",%2-3)3&4#+
 "66zBFHEJ ###q(TQY"<0J0J0L#MNOD$5$5 6HT&&'..Bs8;P1QQ  JAt6$$%U:,a9Q	\ qy|E,F,F,HIJ
C 1 12OD'(9(9':!<= r   )FNN) r   dataclassesr   r   	functoolsr   pathlibr   mlx.corecorer   mlx.nnrD   numpyrg   mlx.nn.utilsr   	mlx.utilsr   r	   r
   	callbacksr   datasetsr   r"   r$   rS   r   callabler   r   r>   r   r   <module>r      s     (      * ,  ' "+  # # #L( 
	G^ ! /$) $) $)V %! /*.h>
 h> h> h> (h>r   