
      i                         d dl Z d dlZd dlZd dlmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ erd dlmZ  G d d      Z G d d	e      Zy)
    N)TYPE_CHECKINGAnyLiteralOptionalUnion)!_TORCHMETRICS_GREATER_EQUAL_1_0_0)_PATH)Fabricc                       e Zd ZdZ	 	 	 	 	 	 	 d ded   dededee   dee   d	ee   d
e	f fdZ
 ej                         dddej                  dededdf
d       Zdej                  de	fdZdddeddfdZdeeej                  f   deeej                  f   de	fdZdeeej                  f   deeej                  f   de	fdZdej                  de	fdZdej                  ddfdZdeeef   fdZdeeef   ddfdZ xZS )!SpikeDetectiona  Spike Detection Callback.

    Terminates training with a ``TrainingSpikeException`` when a loss-spike was detected and
    saves the batches to skip when resuming to a file.

    We skip the current and the previous batch since it is unclear whether the previous batch
    altered the weights in a way that it causes the spike or just the current batch is corrupted somehow.

    Args:
        mode: Whether to minimize or maximize the tracked metric
        window: A running mean of metrics with ``window`` size. Serves as reference value for spikes.
        warmup: After how many batches spike-tracking should start
        atol: An absolute tolerance.  Every diff between the running mean and the current value,
            that's not an improvement and above ``atol`` will be considered a spike
        rtol: A relative tolerance. Every diff between the running mean and the current value,
            that's higher than ``rtol * running_mean`` is considered a spike
        exclude_batches_path: Where to save the file that contains the batches to exclude.
            Will default to current directory.
        finite_only: If set to ``False``, consider non-finite values like NaN, inf and -inf a spike as well.

    Nmode)minmaxwindowwarmupatolrtolexclude_batches_pathfinite_onlyc                 @   t         rddlm} ddlm}	 nt        d      t        
|           d| _         |	 |dd      |      | _	        d| j                  _
        d| j                  _        || _        || _        || _        || _        g | _        || _        || _        y )	Nr   )
MeanMetric)RunningzJSpikeDetection requires `torchmetrics>=1.0.0` Please upgrade your version.        F)dist_sync_on_stepsync_on_compute)r   )r   torchmetrics.aggregationr   torchmetrics.wrappersr   RuntimeErrorsuper__init__last_valrunning_meanr   r   r   r   r   r   bad_batchesr   r   )selfr   r   r   r   r   r   r   r   r   	__class__s             u/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning_fabric/utilities/spike.pyr    zSpikeDetection.__init__&   s     -;5kll47#JX]$^gmn.3+,1)			&($8!&    fabricr
   lossbatch	batch_idxreturnc                    |dk(  r/| j                   j                  |j                  j                         | j                  t        j                         | _        t        | j                        j                  d      s/t
        j                  j                  | j                  d      | _        t        || j                  k\  xr | j                  |            }|j                  j                          |j                  j                  |d      }|r| j!                  ||       y| j"                  xsG |j                  j                  t        t%        j&                  |      j)                               d      }|r| j+                  |       yy)z)Checks if we currently have a loss-spike.r   Nz.jsonzskip_batches.jsonF)allT)r"   tostrategyroot_devicer   osgetcwdstrendswithpathjoinboolr   	_is_spikebarrierreduce_boolean_decision_handle_spiker   torchisfiniter.   _update_stats)r$   r(   r)   r*   r+   is_spikeis_spike_globalis_finite_alls           r&   on_train_batch_endz!SpikeDetection.on_train_batch_endF   s5    >  !<!<=$$,(*		D%4,,-66w?(*T5N5NPc(dD%	T[[0IT^^D5IJ! !//AA(PUAVvy1 ,, 0W0WU^^D)--/0d 1X 1M ""4( r'   c                    t        j                         5  t        j                  d       | j                  j	                         }d d d        || j
                  z
  }| j                  rt        j                  |      sy| j                  |      ry| j                  |      xr | j                  ||      S # 1 sw Y   rxY w)NignoreTF)warningscatch_warningssimplefilterr"   computer!   r   r=   r>   
_is_better_check_atol_check_rtol)r$   r)   running_val	curr_diffs       r&   r9   zSpikeDetection._is_spikea   s     $$&!!(+++335K ' 4==(	ENN4$8??9%k2Zt7G7Gk7ZZ '&s   0B66B?c                    | j                   j                  |dz
  |g       |j                  dk(  r| j                  J t	        j
                  t        j                  j                  | j                        d       t        | j                  d      5 }t        j                  | j                   |d       d d d        t        |      # 1 sw Y   xY w)	N   r   T)exist_okw   )indent)r+   )r#   extendglobal_rankr   r2   makedirsr6   dirnameopenjsondumpTrainingSpikeException)r$   r(   r+   fs       r&   r<   zSpikeDetection._handle_spikeq   s     	Q	 :;",,888KK(A(ABTRd//5		$**Aa8 6 %y99 65s   #C		Cval_aval_bc                 |    | j                   d u xs- t        t        ||z
        t        | j                         k\        S N)r   r8   absr$   r^   r_   s      r&   rK   zSpikeDetection._check_atol   s1    		T!Pd3uu}+=TYY+O&PPr'   c                     | j                   d u xs0 t        t        ||z
        t        | j                   |z        k\        S ra   )r   r8   rb   rc   s      r&   rL   zSpikeDetection._check_rtol   s8    		T!Xd3uu}+=TYYQVEVAW+W&XXr'   diff_valc                     | j                   dk(  rt        |dk  j                               S | j                   dk(  rt        |dk\  j                               S t        d| j                          )Nr   r   r   r   z*Invalid mode. Has to be min or max, found )r   r8   r.   
ValueError)r$   re   s     r&   rJ   zSpikeDetection._is_better   sa    99S--/0099Q++-..Edii[QRRr'   valc                 H    | j                   j                  |       || _        y ra   )r"   updater!   )r$   rh   s     r&   r?   zSpikeDetection._update_stats   s      %r'   c           
         t        | j                  t        j                        r| j                  j	                         n| j                  | j
                  | j                  | j                  | j                  | j                  | j                  | j                  j                         | j                  j                  j                         d	S )N)	r!   r   r   r   r   r#   bad_batches_pathrunningmean)
isinstancer!   r=   Tensoritemr   r   r   r   r#   r   r"   
state_dictbase_metric)r$   s    r&   rr   zSpikeDetection.state_dict   s    0:4==%,,0W**,]a]j]jIIkkIIII++ $ 9 9((335%%11<<>

 
	
r'   rr   c                    |j                  d      | _        |j                  d      | _        |j                  d      | _        |j                  d      | _        |j                  d      | _        |j                  d      | _        |j                  d      | _        | j                  j                  |j                  d             | j                  j                  j                  |j                  d	             y )
Nr!   r   r   r   r   r#   rl   rm   rn   )popr!   r   r   r   r   r#   r   rm   load_state_dictr"   rs   )r$   rr   s     r&   rv   zSpikeDetection.load_state_dict   s    "z2NN6*	 nnX.NN6*	NN6*	%>>-8$.NN3E$F!$$Z^^I%>?%%55jnnV6LMr'   )r   
   rP   Ng       @NT)__name__
__module____qualname____doc__r   intr   floatr	   r8   r    r=   no_gradrp   r   rC   r9   r<   r   rK   rL   rJ   r?   dictr4   rr   rv   __classcell__r%   s   @r&   r   r      s   0 ', $ #04 'l#' ' 	'
 uo' uo' 'uo' '@ U]]_) ) )c )^a )fj ) )4[ell [t [ :H : : :Quell':!; QE%QVQ]Q]J]D^ Qcg QYuell':!; YE%QVQ]Q]J]D^ Ycg YS5<< SD S $ 

DcN 
	N$sCx. 	NT 	Nr'   r   c                   0     e Zd ZdZdededef fdZ xZS )r\   z,Exception to be raised with Training Spikes.r+   argskwargsc                 2    t        |   d| g|i | y )Nz!Training spike detected in batch )r   r    )r$   r+   r   r   r%   s       r&   r    zTrainingSpikeException.__init__   s#    <YKHZ4ZSYZr'   )rx   ry   rz   r{   r|   r   r    r   r   s   @r&   r\   r\      s*    6[# [c [S [ [r'   r\   )rZ   r2   rF   typingr   r   r   r   r   r=   "lightning_fabric.utilities.importsr    lightning_fabric.utilities.typesr	   lightning_fabric.fabricr
   r   r   r\    r'   r&   <module>r      s@     	  ? ?  P 2.ZN ZNz[\ [r'   