
      ihZ                     T   d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
mZmZmZ ddlmc mc mZ ddlmZ ddlmZmZ dd	lmZmZ ddlZdd
lmZ ddlmZ ddl m!Z! ddl"m#Z#m$Z$  ejJ                  e&      Z'ejP                  ejR                  ejT                  ejV                  dZ,ejZ                  ej\                  ej^                  ej`                  dZ1e2ejf                  e4f   Z5ee5   Z6eejn                  ejp                  ejr                  fZ: G d de      Z; G d de      Z<y)z
ModelPruning
^^^^^^^^^^^^
    N)Sequence)deepcopypartial)AnyCallableOptionalUnion)apply_to_collection)Tensornn)	TypedDictoverride)Callback)LightningModule)MisconfigurationException)rank_zero_debugrank_zero_only)ln_structuredl1_unstructuredrandom_structuredrandom_unstructuredc                   F    e Zd ZU ej                  ed<   eeee	f      ed<   y)	_LayerRefdatanamesN)
__name__
__module____qualname__r   Module__annotations__listtupleintstr     x/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/pytorch_lightning/callbacks/pruning.pyr   r   :   s    
))Oc3h  r'   r   c                      e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 d5deeef   dedee	e      de
deeeeegeeef   f   f   dee
eege
f   f   d	e
d
ee
eege
f   f   de
dee   dee   dede
ddfdZd6dedefdZdededeeej"                  f   fdZedededefd       Zdej,                  ddfdZedej,                  dej,                  deddfd       Zd7dZdeddfdZdedeeef   fdZdeddfdZedej,                  dedeeef   fd        Zdeeef   ddfd!Z e!	 d8d"e	eeef      d#e	eeef      deeef   ddfd$       Z"e#d%d&d'e$d(eddfd)       Z%d*eddfd+Z&e#d%d&d'e$ddfd,       Z'e#d%d&d'e$ddfd-       Z(e#d%d&d'e$ddfd.       Z)d'e$deeef   fd/Z*e#d%d&d'e$d0eeef   ddfd1       Z+e	 d9d'e$dede,e   defd2       Z-ed3ede
fd4       Z.y):ModelPruning)weightbiasN
pruning_fnparameters_to_pruneparameter_namesuse_global_unstructuredamountapply_pruningmake_pruning_permanentuse_lottery_ticket_hypothesisresample_parameterspruning_dimpruning_normverboseprune_on_train_epoch_endreturnc           	         || _         || _        || _        |	| _        || _        |xs | j
                  | _        i | _        d| _        d| _	        | j                  D ]+  }|| j
                  vst        d| d| j
                          t        |t              ri }|j                         }|t        vr-t        d| dt        t        j!                                d      |j#                  d      r)|
t        d      |d	k(  r|t        d
      ||d<   |
|d<    | j$                  |fi |}nK| j'                  |      r|s8t        d      t        dt        t        j!                                d| d      |r+|j(                  dk7  rt        d| d|j(                   d      || _        || _        || _        t        |t0        t2        f      st5        |      st        d      || _        |dvrt        d      || _        y)a  Model pruning Callback, using PyTorch's prune utilities. This callback is responsible of pruning networks
        parameters during training.

        To learn more about pruning with PyTorch, please take a look at
        `this tutorial <https://pytorch.org/tutorials/intermediate/pruning_tutorial.html>`_.

        .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

        .. code-block:: python

            parameters_to_prune = [(model.mlp_1, "weight"), (model.mlp_2, "weight")]

            trainer = Trainer(
                callbacks=[
                    ModelPruning(
                        pruning_fn="l1_unstructured",
                        parameters_to_prune=parameters_to_prune,
                        amount=0.01,
                        use_global_unstructured=True,
                    )
                ]
            )

        When ``parameters_to_prune`` is ``None``, ``parameters_to_prune`` will contain all parameters from the model.
        The user can override ``filter_parameters_to_prune`` to filter any ``nn.Module`` to be pruned.

        Args:

            pruning_fn: Function from torch.nn.utils.prune module or your own PyTorch ``BasePruningMethod`` subclass.
                Can also be string e.g. `"l1_unstructured"`. See pytorch docs for more details.

            parameters_to_prune: List of tuples ``(nn.Module, "parameter_name_string")``.

            parameter_names: List of parameter names to be pruned from the nn.Module.
                Can either be ``"weight"`` or ``"bias"``.

            use_global_unstructured: Whether to apply pruning globally on the model.
                If ``parameters_to_prune`` is provided, global unstructured will be restricted on them.

            amount: Quantity of parameters to prune:

                - ``float``. Between 0.0 and 1.0. Represents the fraction of parameters to prune.
                - ``int``. Represents the absolute number of parameters to prune.
                - ``Callable``. For dynamic values. Will be called every epoch. Should return a value.

            apply_pruning: Whether to apply pruning.

                - ``bool``. Always apply it or not.
                - ``Callable[[epoch], bool]``. For dynamic values. Will be called every epoch.

            make_pruning_permanent: Whether to remove all reparameterization pre-hooks and apply masks
                when training ends or the model is saved.

            use_lottery_ticket_hypothesis: See `The lottery ticket hypothesis <https://arxiv.org/abs/1803.03635>`_:

                - ``bool``. Whether to apply it or not.
                - ``Callable[[epoch], bool]``. For dynamic values. Will be called every epoch.

            resample_parameters: Used with ``use_lottery_ticket_hypothesis``. If True, the model parameters will
                be resampled, otherwise, the exact original parameters will be used.

            pruning_dim: If you are using a structured pruning method you need to specify the dimension.

            pruning_norm: If you are using ``ln_structured`` you need to specify the norm.

            verbose: Verbosity level. 0 to disable, 1 to log overall sparsity, 2 to log per-layer sparsity

            prune_on_train_epoch_end: whether to apply pruning at the end of the training epoch.
                If this is ``False``, then the check runs at the end of the validation epoch.

        Raises:
            MisconfigurationException:
                If ``parameter_names`` is neither ``"weight"`` nor ``"bias"``,
                if the provided ``pruning_fn`` is not supported,
                if ``pruning_dim`` is not provided when ``"unstructured"``,
                if ``pruning_norm`` is not provided when ``"ln_structured"``,
                if ``pruning_fn`` is neither ``str`` nor :class:`torch.nn.utils.prune.BasePruningMethod`, or
                if ``amount`` is none of ``int``, ``float`` and ``Callable``.

        Nz%The provided `parameter_names` name: z
 isn't in zThe provided `pruning_fn` z2 isn't available in PyTorch's built-in functions:  _structuredzKWhen requesting `structured` pruning, the `pruning_dim` should be provided.r   zOWhen requesting `ln_structured` pruning, the `pruning_norm` should be provided.ndimz\PyTorch `BasePruningMethod` is currently only supported with `use_global_unstructured=True`.z(`pruning_fn` is expected to be a str in z* or a PyTorch `BasePruningMethod`. Found: zI. HINT: if passing a `BasePruningMethod`, pass the class, not an instanceunstructuredzdOnly the "unstructured" PRUNING_TYPE is supported with `use_global_unstructured=True`. Found method z	 of type z. zO`amount` should be provided and be either an int, a float or Callable function.)r         z"`verbose` must be any of (0, 1, 2))_use_global_unstructured_parameters_to_prune_use_lottery_ticket_hypothesis_resample_parameters_prune_on_train_epoch_endPARAMETER_NAMES_parameter_names_global_kwargs_original_layers_pruning_method_namer   
isinstancer%   lower_PYTORCH_PRUNING_FUNCTIONSr"   keysendswith_create_pruning_fn_is_pruning_methodPRUNING_TYPEr-   _apply_pruning_make_pruning_permanentr$   floatcallabler1   _verbose)selfr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   namepruning_kwargss                   r(   __init__zModelPruning.__init__B   sh   B )@%$7!.K+$7!)A& / G43G3G.0@D37!))D4////;D6DL`L`Kab  * j#&N#))+J!;;/0 =,,01K1P1P1R,S+TTUW  ""=1&3e  0#+7m  +7N3'(3u%000N~NJ$$Z0*/r  ,:4@Z@_@_@a;b:c<ZL I[[  #z'>'>.'P+!!+Ij6M6M5NbR 
 %+'=$6C<0HV4D+a  )#+,PQQr'   c                     |S )zAThis function can be overridden to control which module to prune.r&   )rZ   r.   s     r(   filter_parameters_to_prunez'ModelPruning.filter_parameters_to_prune   s    ""r'   kwargsc                     | j                   r	t        |   nt        |   }t        |      sJ d       | j                   r|| _        |j
                  | _        | j                   r|S t        j                  |fi |S )a  This function takes `pruning_fn`, a function name.

        IF use_global_unstructured, pruning_fn will be resolved into its associated ``PyTorch BasePruningMethod`` ELSE,
        pruning_fn will be resolved into its function counterpart from `torch.nn.utils.prune`.

        z'Selected pruning method is not callable)	rC   _PYTORCH_PRUNING_METHODrO   rX   rJ   r   rL   r*   _wrap_pruning_fn)rZ   r-   r`   pruning_meths       r(   rR   zModelPruning._create_pruning_fn   s     ,, $J/+J7 	
 %P'PP%(("(D %1$9$9!((,,\DVDDr'   c                     t        | fi |S Nr   )r-   r`   s     r(   rc   zModelPruning._wrap_pruning_fn  s    z,V,,r'   modulec                     |j                         D ]g  \  }}t        |j                        D ]J  }|j                  |   }t        |t        j
                        s-|j                  |       |j                  |= L i y)zRemoves pruning buffers from any pruned modules.

        Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/nn/utils/prune.py#L1118-L1122

        N)named_modulesr"   _forward_pre_hooksrM   pytorch_pruneBasePruningMethodremove)rZ   rg   _khooks        r(   r3   z#ModelPruning.make_pruning_permanent	  sj      --/IAv&334003dM$C$CDKK'11!4	 5 0r'   newoldr[   c                    t        | |dz         rt        | |dz         nt        | |      }t        ||      }|"| t        |t              rt        |t              sy |j                  j                  |j                        |_        y )N_orig)hasattrgetattrrM   r   r   todevice)rq   rr   r[   dstsrcs        r(   _copy_paramzModelPruning._copy_param  sq     /6c4'>.Jgc4'>*PWX[]aPbc4 ;#+ZV-DJWZ\bLc88;;szz*r'   c                 `   | j                   J | j                   j                         D ]  }|d   }|d   }| j                  r<t        |d      r0t	        |j
                        rt        |      }|j                          |D ]*  \  }}| j                  |   \  }}| j                  |||       ,  y)a  Lottery ticket hypothesis algorithm (see page 2 of the paper):

            1. Randomly initialize a neural network :math:`f(x; \theta_0)` (where :math:`\theta_0 \sim \mathcal{D}_\theta`).
            2. Train the network for :math:`j` iterations, arriving at parameters :math:`\theta_j`.
            3. Prune :math:`p\%` of the parameters in :math:`\theta_j`, creating a mask :math:`m`.
            4. Reset the remaining parameters to their values in :math:`\theta_0`, creating the winning ticket :math:`f(x; m \odot \theta_0)`.

        This function implements the step 4.

        The ``resample_parameters`` argument can be used to reset the parameters with a new :math:`\theta_z \sim \mathcal{D}_\theta`

        Nr   r   reset_parameters)	rK   valuesrF   ru   rX   r}   r   rD   r{   )rZ   dcopyr   ir[   rq   rn   s           r(   apply_lottery_ticket_hypothesisz,ModelPruning.apply_lottery_ticket_hypothesis  s     $$000&&--/AV9DgJE((WT;M-NS[\`\q\qSr~%%' 42215Q  dD1 ! 0r'   c                 T    | j                   D ]  \  }}| j                  |||        y )N)r[   r1   )rD   r-   )rZ   r1   rg   r[   s       r(   _apply_local_pruningz!ModelPruning._apply_local_pruning7  s(     55LFDOOFfO= 6r'   c                    || j                   d<   t        t        j                  | j                        j
                        }|j                  d       | j                   j                         D ci c]  \  }}||v s|| c}}S c c}}w )Nr1   rZ   )rJ   setinspect	signaturer-   
parametersdiscarditems)rZ   r1   paramsro   vs        r(   _resolve_global_kwargsz#ModelPruning._resolve_global_kwargs;  st    (.H%W&&t7BBCv!%!4!4!:!:!<L!<AV1!<LLLs   0B=Bc                 z    t        j                  | j                  fd| j                  i| j	                  |       y )Npruning_method)rk   global_unstructuredrD   r-   r   )rZ   r1   s     r(   _apply_global_pruningz"ModelPruning._apply_global_pruningA  s9    ))%%	
6:oo	
IMIdIdekIl	
r'   c                     | d}t        | |      syt        | |      }|dk(  j                         j                         |j	                         fS )N_mask)r   rA   r   )ru   rv   sumitemnumel)rg   r[   attrmasks       r(   _get_pruned_statszModelPruning._get_pruned_statsF  sK    u~vt$vt$	 %%'55r'   c                    | j                   r.| j                  D cg c]  \  }}| j                  ||       }}}| j                  r| j	                  |       n| j                  |       | j                   rC| j                  D cg c]  \  }}| j                  ||       }}}| j                  ||       yyc c}}w c c}}w )z+Applies pruning to ``parameters_to_prune``.)r1   N)rY   rD   r   rC   r   r   _log_sparsity_stats)rZ   r1   mr>   
prev_stats
curr_statss         r(   r2   zModelPruning.apply_pruningN  s    ==CGC\C\]C\41a$00A6C\J]((&&v.%%f-==CGC\C\]C\41a$00A6C\J]$$ZF$K  ^ ^s   B:C prevcurrc                    t        d |D              }t        d |D              }t        d |D              }t        j                  d| j                   d| d| d||z  dd	| d| d||z  dd
       | j                  dk(  rqt        | j                        D ]X  \  }\  }}	||   \  }
}||   \  }}t        j                  d| j                   d|d|	 d| d|
 d|
|z  dd	| d||z  dd
       Z y y )Nc              3   &   K   | ]	  \  }}|  y wrf   r&   ).0rn   totals      r(   	<genexpr>z3ModelPruning._log_sparsity_stats.<locals>.<genexpr>`  s     6XQ5   c              3   &   K   | ]	  \  }}|  y wrf   r&   r   zerosrn   s      r(   r   z3ModelPruning._log_sparsity_stats.<locals>.<genexpr>a       :TuTr   c              3   &   K   | ]	  \  }}|  y wrf   r&   r   s      r(   r   z3ModelPruning._log_sparsity_stats.<locals>.<genexpr>b  r   r   z	Applied `z`. Pruned: /z (z.2%z) -> )rB   z` to `.z` with amount=z
. Pruned: )r   loginforL   rY   	enumeraterD   )rZ   r   r   r1   total_paramsprev_total_zeroscurr_total_zerosr   rg   r[   prev_mask_zerosprev_mask_sizecurr_mask_zeroscurr_mask_sizes                 r(   r   z ModelPruning._log_sparsity_stats\  sV    666:T:::T::112 3 !<.3Cl3RSV2W X !<.3Cl3RSV2WWXZ	

 ==A%.t/H/H%I!>FD26q'/26q'/ 9 9:&
!D6Q_`f_g h'(?^+KC*P Q'(?^+KC*PPQS &J r'   trainerz
pl.Trainer	pl_modulestagec           	         | j                  || j                  | j                        }| j                  |      | _        | j                  ri | _        t        | j                        D ]f  \  }\  }}t        |      }| j
                  j                  |t        t        |      g              | j
                  |   d   j                  ||f       h y y )N)r/   )r   r   r   )sanitize_parameters_to_prunerD   rI   r_   rE   rK   r   id
setdefaultr   r   append)	rZ   r   r   r   r.   r   rg   r[   id_s	            r(   setupzModelPruning.setupr  s    "??t00$BWBW @ 
 %)$C$CDW$X!.. %'D!%.t/H/H%I!>FDj%%00iXfEU]_6`a%%c*73::At9E &J	 /r'   current_epochc                    t        | j                        r| j                  |      n| j                  }t        | j                        r| j                  |      n| j                  }|r|sy | j                  |       t        | j                        r| j	                  |      rn| j                  r| j                          y y y rf   )rX   rU   r1   r2   rE   r   )rZ   r   pruner1   s       r(   _run_pruningzModelPruning._run_pruning  s    6>t?R?R6S##M2Y]YlYl/7/D]+$++F6" ;;< //>44002 5 ?r'   c                 j    | j                   r't        d       | j                  |j                         y y )Nz3`ModelPruning.on_train_epoch_end`. Applying pruning)rG   r   r   r   rZ   r   r   s      r(   on_train_epoch_endzModelPruning.on_train_epoch_end  s-    ))QRi556 *r'   c                     |j                   s4| j                  s't        d       | j                  |j                         y y y )Nz8`ModelPruning.on_validation_epoch_end`. Applying pruning)sanity_checkingrG   r   r   r   r   s      r(   on_validation_epoch_endz$ModelPruning.on_validation_epoch_end  s9    &&t/M/MVWi556 0N&r'   c                 V    | j                   rt        d       | j                  |       y y )NzJ`ModelPruning.on_train_end`. Pruning is made permanent for this checkpoint)rV   r   r3   r   s      r(   on_train_endzModelPruning.on_train_end  s'    ''hi''	2 (r'   c                 r   |j                         }|D ch c]&  }|j                  d      s|j                  dd      ( }}|D ]L  }|j                  |dz         }|j                  |dz         }|j	                  |j
                        |z  ||<   N dt        dt        fd}t        |t        |      S c c}w )Nr    rt   )dtypetensorr:   c                 "    | j                         S rf   )cpu)r   s    r(   move_to_cpuzGModelPruning._make_pruning_permanent_on_state_dict.<locals>.move_to_cpu  s    ::<r'   )
state_dictrQ   replacepoprw   r   r   r   )	rZ   r   r   ro   map_pruned_paramstensor_nameorigr   r   s	            r(   %_make_pruning_permanent_on_state_dictz2ModelPruning._make_pruning_permanent_on_state_dict  s    ))+
 >H_Z1::V]K^QYYw3Z_,K>>+"78D>>+"78D&*ggDJJg&?$&FJ{#	 -	  	 6 	  #:v{CC `s
   B4B4
checkpointc                 \    | j                   r t        d       | j                  |      |d<   y y )NzP`ModelPruning.on_save_checkpoint`. Pruning is made permanent for this checkpointr   )rV   r   r   )rZ   r   r   r   s       r(   on_save_checkpointzModelPruning.on_save_checkpoint  s/    ''no'+'Q'QR['\J|$ (r'   c                    |xs t         j                  }| j                         D cg c]  }t        |t              r| }}|sN|D cg c]?  }|D ]8  }t        ||d      )t        t        ||d      t        j                        r||f: A }}}|S t        |t        t        f      rt        |      dkD  r}t        d |D              rkt        d |D              rYg g }}|D ]9  \  }	}
|	|vr|j                  |	       t        |	|
      r)|j                  |
       ; |s|rt        d| d|       |S t        d      c c}w c c}}w )a  This function is responsible of sanitizing ``parameters_to_prune`` and ``parameter_names``. If
        ``parameters_to_prune is None``, it will be generated with all parameters of the model.

        Raises:
            MisconfigurationException:
                If ``parameters_to_prune`` doesn't exist in the model, or
                if ``parameters_to_prune`` is neither a list nor a tuple.

        Nr   c              3   8   K   | ]  }t        |      d k(    yw)rB   N)len)r   ps     r(   r   z<ModelPruning.sanitize_parameters_to_prune.<locals>.<genexpr>  s     =)<ACFaK)<s   c              3   z   K   | ]3  \  }}t        |t        j                        xr t        |t               5 y wrf   )rM   r   r    r%   )r   abs      r(   r   z<ModelPruning.sanitize_parameters_to_prune.<locals>.<genexpr>  s1     dPc1Jq")),CAs1CCPcs   9;zUSome provided `parameters_to_prune` don't exist in the model. Found missing modules: z and missing parameters: zThe provided `parameters_to_prune` should either be list of tuple with 2 elements: (nn.Module, parameter_name_to_prune) or None)r*   rH   modulesrM   _MODULE_CONTAINERSrv   r   	Parameterr"   r#   r   allr   ru   r   )r   r.   r/   r   r   current_modulesr   missing_modulesmissing_parametersrg   r[   s              r(   r   z)ModelPruning.sanitize_parameters_to_prune  s    %D(D(D
&/&7&7&9c&9AOaAb1&9c" $##A(A1a&2z'!QPTBUWYWcWc7d A( #   #> #"1 *T5M:'(1,=)<==dPcdd24b/O 30#**62vt,&--d3 !4 "4///>.??XYkXln  #" ,Q ; d#s   D?D?
AEmethodc                 b    t        j                  |       syt        | t        j                        S )NF)r   isclass
issubclassrk   rl   )r   s    r(   rS   zModelPruning._is_pruning_method  s$    v&&-"A"ABBr'   )r&   NTg      ?TTTFNNr   T)r&   )r:   N)r   )r&   r&   )/r   r   r   rH   r
   r   r%   _PARAM_LISTr	   r"   boolr$   rW   r]   r_   r   rk   rl   rR   staticmethodrc   r   r    r3   r{   r   r   dictr   r   r#   r   r2   r   r   r   r   r   r   r   r   r   r   r   r   r   rS   r&   r'   r(   r*   r*   ?   sy   (O
 ,./3(,HK<@'+LP$)%)&*)-g (C-(g  )g  "$s),	g 
 "&g  c5(C5%U
2C+C"DDEg  T8SE4K#889g  !%g  (-T8SE4K3H-H'Ig  "g  c]g  smg  g  #'g  
g R#k #S^ #ES EC EE(TaTsTsJsDt E, -X - - - -5RYY 54 5 + + +# +$ + +20>5 >T >MU MtCH~ M
E 
d 

 6")) 63 65c? 6 6LE#u*$5 L$ L desCx)15eCHo1FPUVY[`V`Pa	 * F\ Fo Fc FVZ F F 3# 3$ 3 7, 7? 7W[ 7 7
 7| 7 7\` 7 7
 3L 3_ 3QU 3 3
D DSWX[]`X`Sa D" ], ]? ]`dehjmem`n ]sw ] ] ln0#"0#9D0#\deh\i0#	0# 0#d C3 C4 C Cr'   r*   )=__doc__r   loggingcollections.abcr   r   r   	functoolsr   typingr   r   r	   r
   torch.nn.utils.pruner   utilsr   rk   #lightning_utilities.core.apply_funcr   torchr   typing_extensionsr   r   pytorch_lightningpl$pytorch_lightning.callbacks.callbackr   pytorch_lightning.core.moduler   &pytorch_lightning.utilities.exceptionsr   %pytorch_lightning.utilities.rank_zeror   r   	getLoggerr   r   r   r   r   r   rO   LnStructuredL1UnstructuredRandomStructuredRandomUnstructuredrb   r#   r    r%   _PARAM_TUPLEr   
Sequential
ModuleList
ModuleDictr   r   r*   r&   r'   r(   <module>r     s  
   $   1 1 , , C  1  9 9 L Qg! #00$44&88(<<	  #//$33&77(;;	  RYY^$|$%r}}bmmR]]S !	 !
sC8 sCr'   