
      iS                        d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl m
Z
 d dlmZ d dlmZmZmZmZmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZmZ d dl m!Z! d dl"m#Z#m$Z$m%Z% d dl&m'Z'  ejP                  e)      Z* ed      Z+ G d de!      Z,deej0                  gdf   dededdfdZ-dej\                  j^                  deddfdZ0	 d&dej\                  jb                  dej0                  dee2   dej\                  jb                  fdZ3 e jh                  d       defd!       Z5d'dej\                  j^                  d"e6d#e7e8   d$e8ddf
d%Z9y)(    N)OrderedDict)AbstractContextManager	ExitStack)partial)
ModuleType)AnyCallableLiteralOptionalcast)apply_to_collection)RequirementCache)Tensor)init)_IncompatibleKeys)Selfoverride)	Precision)_ClassReplacementContextManager_convert_fp_tensor_DtypeContextManager)_DEVICEbitsandbytesc            	       :   e Zd ZdZ	 	 dded   deej                     deee	      ddfdZ
ed	ej                  j                  dej                  j                  fd
       Zedefd       Zedefd       Zedefd       Zededefd       Zededefd       Zy)BitsandbytesPrecisiona  Plugin for quantizing weights with `bitsandbytes <https://github.com/bitsandbytes-foundation/bitsandbytes>`__.

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    .. note::
        The optimizer is not automatically replaced with ``bitsandbytes.optim.Adam8bit`` or equivalent 8-bit optimizers.

    Args:
        mode: The quantization mode to use.
        dtype: The compute dtype to use.
        ignore_modules: The submodules whose Linear layers should not be replaced, for example. ``{"lm_head"}``.
            This might be desirable for numerical stability. The string will be checked in as a prefix, so a value like
            "transformer.blocks" will ignore all linear layers in all of the transformer blocks.
    Nmode)nf4nf4-dqfp4fp4-dqint8int8-trainingdtypeignore_modulesreturnc                    t                |~|j                  d      rt        j                  }n\t        j                  j                         r.t        j                  j                         rt        j                  nt        j                  }|j                  d      r#|t        j                  urt        |d| d      t               }|d   |d   |d   |d   |d   |d	   d
}||   | _
        || _        |xs
 t               | _        y )Nr!   z7 only works with `dtype=torch.float16`, but you chose ``
_NF4Linear_NF4DQLinear
_FP4Linear_FP4DQLinear_Linear8bitLt_Int8LinearInference)r   r   r   r    r"   r!   )_import_bitsandbytes
startswithtorchfloat16cudais_availableis_bf16_supportedbfloat16
ValueErrorglobals_linear_clsr#   setr$   )selfr   r#   r$   globals_mode_to_clss         /Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning/fabric/plugins/precision/bitsandbytes.py__init__zBitsandbytesPrecision.__init__F   s     	=v& ',jj&=&=&?EJJD`D`DbENNhmhuhu  ??6"uEMM'Ax'^_d^eefghh9L)~.L)~.%o634
 't,
,5    modulec                    t        d |j                         D              st        d      t               t        fd|j                         D              s!t	        || j
                  | j                         |j                         D ];  }t        |j                  j                        s$| j                  |_        d|_        = |S )Nc              3   d   K   | ](  }t        |t        j                  j                         * y wN)
isinstancer0   nnLinear).0ms     r=   	<genexpr>z7BitsandbytesPrecision.convert_module.<locals>.<genexpr>j   s"     L;Ka:a1;Ks   .0z|You are using the bitsandbytes precision plugin, but your model has no Linear layers. This plugin won't work for your model.c              3      K   | ]:  }t        |j                  j                  j                  j                  f       < y wrC   )rD   rE   Linear8bitLt
Linear4bit)rG   rH   bnbs     r=   rI   z7BitsandbytesPrecision.convert_module.<locals>.<genexpr>r   s4     eTdq:a#&&"5"5svv7H7H!IJTds   A AF)anymodules	TypeErrorr.   _convert_layersr8   r$   rD   rE   rL   r#   compute_dtypecompute_type_is_set)r:   r@   rH   rM   s      @r=   convert_modulez$BitsandbytesPrecision.convert_moduleg   s     L6>>;KLL.  #$eTZTbTbTdeeFD$4$4d6I6IJ !A!SVV../"&**(-% " r?   c                 ,    t        | j                        S rC   r   r#   r:   s    r=   tensor_init_contextz)BitsandbytesPrecision.tensor_init_context}       #DJJ//r?   c                     | j                   rt        d| j                    d      | j                         }t        d| j                  i      }t               }|j                  |       |j                  |       |S )NzInstantiating your model under the `init_module` context manager is not supported when used with `BitsandbytesPrecision(..., ignore_modules=z)` as this may initialize the layers on-device, defeating the purpose of quantization. You can remove `ignore_modules` or remove the `init_module` context manager.ztorch.nn.Linear)r$   RuntimeErrorrX   r   r8   r   enter_context)r:   	dtype_ctxcontext_managerstacks       r=   module_init_contextz)BitsandbytesPrecision.module_init_context   s    ??C?R?R>S TQQ  ,,.	9;LdN^N^:_`I&O,r?   c                 ,    t        | j                        S rC   rV   rW   s    r=   forward_contextz%BitsandbytesPrecision.forward_context   rY   r?   datac                 D    t        |t        t        | j                        S N)functionr#   dst_type)r   r   r   r#   r:   rc   s     r=   convert_inputz#BitsandbytesPrecision.convert_input   s    "42DF]a]g]ghhr?   c                 T    t        |t        t        t        j                               S re   )r   r   r   r0   get_default_dtyperh   s     r=   convert_outputz$BitsandbytesPrecision.convert_output   s    "42DF]b]t]t]vwwr?   NN)__name__
__module____qualname____doc__r
   r   r0   r#   r9   strr>   r   rE   ModulerT   r   rX   r`   rb   r   ri   rl    r?   r=   r   r   .   s%   4 (,-1	6OP6 $6 !S*	6
 
6B UXX__   * 0%; 0 0 %;  " 0!7 0 0 i# i# i i x3 x3 x xr?   r   quantize_fn
state_dict_r%   c                 b    t        d |D        d       }|y |j                  |      } | |       y )Nc              3   D   K   | ]  }|j                  d       s|  yw)weightN)endswith)rG   names     r=   rI   z)_quantize_on_load_hook.<locals>.<genexpr>   s     N
dmmH6Mt
s     )nextpop)ru   rv   rw   
weight_keyrz   s        r=   _quantize_on_load_hookr      s4    N
NPTUJ^^J'Fr?   r@   incompatible_keysc                     t        |j                        D ]/  }|j                  d      s|j                  j                  |       1 y )Nrz   )reversedmissing_keysr{   remove)r@   r   keys      r=   _ignore_missing_weights_hookr      s:     )667<<!**11#6 8r?   paramrc   quant_statec                    t               }| j                  j                  dk(  rt        | |j                  j
                        rj|j                  j                  ||j                  || j                  | j                  | j                  | j                  | j                  | j                  	      S t        j                  j                  ||j                        S || _        t        | |j                  j
                        r|| _        | S )Nmeta)	rc   requires_gradr   	blocksizecompress_statistics
quant_typequant_storager@   bnb_quantized)r   )r.   devicetyperD   rE   
Params4bitr   r   r   r   r   r@   r   r0   	Parameterrc   r   )r   rc   r   rM   s       r=   _replace_paramr      s     
 C ||F"eSVV../66$$"00'//$)$=$= ++#11||#11 % 
 
 xx!!$d6H6H!IIEJ%**+'Lr?      )maxsizec                  T  	 t         st        t        t                     dt        j                  v } | sdt        j                  d<   t        j                  dd       t        j                  dd       dd l	| st        j                  d=  G 	fdd		j                  j                        } G 	fd
d	j                  j                        } G d d|      } G d d|      } G d d|      } G d d|      } G d d|      }|||||||d}t               j                  |       	S )NBITSANDBYTES_NOWELCOME1ignorez1.*bitsandbytes was compiled without GPU support.*)messagezHMatMul8bitLt: inputs will be cast from .* to float16 during quantizationr   c                   f    e Zd ZdZddddedee   deded	df
 fd
Zddee	j                     dee	j                     d	dffdZedW j                  j                  de	j                  dee	j                     d	W j                  j                  ffd       Zdddeded	efdZdfdZ xZS )+_import_bitsandbytes.<locals>._Linear8bitLtzWraps `bnb.nn.Linear8bitLt` and enables instantiation directly on the device and re-quantizaton when loading
        the state dict.Ng      @r   	thresholdargsr   r   kwargsr%   c                   t        |   |||d| t        j                  j                  | j
                        | _        | j                  | _        t        j                  d|      j                  j                  dk(  r| j                          | j                  t        t        | j                               | j                  t                y )Nr   r   r   r2   )superr>   r   rE   
Int8Paramsrz   biasr0   tensorr   r   	quantize_"_register_load_state_dict_pre_hookr   r   "register_load_state_dict_post_hookr   )r:   r   r   r   r   	__class__rM   s        r=   r>   z4_import_bitsandbytes.<locals>._Linear8bitLt.__init__   s    Gd6YQ&Qsvv00$++>DK6:iiDI ||Af-4499VC 33G<RTXTbTb4cd334PQr?   rz   c                 "   || j                   j                  }|j                  j                  t        j                  k(  ryt        | j                   j                  j                        sJ | j                  | j                   ||      | _         y)Inplace quantize.N)	rz   rc   r#   r0   r!   rD   rE   r   quantizer:   rz   r   rM   s      r=   r   z5_import_bitsandbytes.<locals>._Linear8bitLt.quantize_   sf    ~)){{  EJJ.dkk366+<+<===--VVDDKr?   
int8paramsc                     |xs t        j                  d      }|j                  dk7  rt        d|j                         |j	                         j                  |t         j                        }| j                  r	|| _        | S t        j                  d      r j                  j                  |      \  }}}n!j                  j                  |      \  }}}}}|| _        t        | d|       t        | d|       | S )Nr2   Unexpected device type: r   r#   int8_vectorwise_quantCBSCB)r0   r   r   r[   
contiguoustor1   has_fp16_weightsrc   hasattr
functionalr   double_quantsetattr)r   rz   r   Br   r   rw   rM   s          r=   r   z4_import_bitsandbytes.<locals>._Linear8bitLt.quantize   s     3u||F3F{{f$"%=fkk]#KLL!!#&&fEMM&JA**"#
  3>>+BC!$!E!Ea!HJBQ'*~~'B'B1'E$B31"$

D"-
E3/r?   Trecurser   c                8   | j                   j                  j                  dk(  rt        | j                   j                  t
        j                  k(  rt        t        j                  |      }t        j                  | j                   j                  |      }|j                  dk(  r| j                  ||       nt        | j                   |      | _         | j                  :t        | j                  t        j                  | j                  |            | _        | S )Nr   r   r2   )rz   r   r   NotImplementedErrorr#   r0   uint8
empty_likerc   r   r   r   r:   r   r   rz   s       r=   to_emptyz4_import_bitsandbytes.<locals>._Linear8bitLt.to_empty  s    {{!!&&&0)){{  EKK/))\\&)F%%dkk&6&6vFF{{f$vv.,T[[&Ayy$*499e6F6FtyyY_6`a	Kr?   c                 ^   | j                   wt        j                  j                  j	                  | j
                        \  }}|dkD  rdt        j                  |      z  nd}t        j                  | j                   | |       t        | j
                  j                  j                        }|r-| j
                  j                  t        j                  k(  rt        | j
                  j                  }t        j                  j                  j                  |t        j                  d             |rz| j
                  j                   j"                  dk(  rt        | j
                  j                   j"                  dk(  r| j%                  |       y t'        | j
                  |      | _        y y )Nr   r      ar   r2   )r   r0   rE   r   _calculate_fan_in_and_fan_outrz   mathsqrtuniform_rD   r   r#   r   r   rc   kaiming_uniform_r   r   r   r   r:   fan_inrw   boundlinear_init_finishedrz   rM   s         r=   reset_parametersz<_import_bitsandbytes.<locals>._Linear8bitLt.reset_parameters  s%   yy$!HHMMGGT	17!DIIf--dii%7#-dkk366;L;L#M #(9(9U[[(H))[[%%FHHMM**6TYYq\*B#;;%%**f4--;;%%**f4NN6*"0f"EDK $r?   rm   r%   N)rn   ro   rp   rq   r   r   r   floatr>   r0   r   r   r   staticmethodrE   r   r   boolr   r   r   __classcell__r   rM   s   @r=   r,   r      s   	 DH\_ 		R# 		Rx/@ 		RTY 		Rkn 		Rsw 		R	EHU\\$: 	E8TYT`T`Ka 	Emq 	E 
	))	38<<	IQRWR^R^I_	VV	 
	, @D 	g 	 	 	"	F 	Fr?   r,   c            	       `    e Zd ZdZdddedee   deddf fdZdd	eej                     deej                     ddffd
ZedW j                  j                  d	ej                  deej                     dW j                  j                  ffd       ZdddededefdZdfdZ xZS ))_import_bitsandbytes.<locals>._Linear4bitzWraps `bnb.nn.Linear4bit` to enable: instantiation directly on the device, re-quantizaton when loading the
        state dict, meta-device initialization, and materialization.Nr   r   r   r   r%   c                   t        |   |d|i| t        j                  j                  | j
                        | _        | j                  | _        t        j                  d|      j                  j                  dk(  r| j                          | j                  t        t        | j                               | j                  t                y )Nr   r   r   r2   )r   r>   r   rE   r   rz   r   r0   r   r   r   r   r   r   r   r   r   )r:   r   r   r   r   rM   s       r=   r>   z2_import_bitsandbytes.<locals>._Linear4bit.__init__9  s    Gd<6<V<svv00$++>DK6:iiDI ||Af-4499VC 33G<RTXTbTb4cd334PQr?   rz   c                 D   || j                   j                  }|j                  j                  t        j                  k(  ryt        | j                   j                  j                        sJ | j                  | j                   ||      | _         d| j                   _	        y)r   NT)
rz   rc   r#   r0   r   rD   rE   r   r   r   r   s      r=   r   z3_import_bitsandbytes.<locals>._Linear4bit.quantize_D  sr    ~)){{  EKK/dkk366+<+<===--VVDDK(,DKK%r?   
params4bitc                    |xs t        j                  d      }|j                  dk7  rt        d|j                         |j	                         j                  |t         j                        }j                  j                  || j                  | j                  | j                  | j                        \  }}t        | ||      S )Nr2   r   r   )r   r   r   r   )r0   r   r   r[   r   r   halfr   quantize_4bitr   r   r   r   r   )r   rz   r   ww_4bitr   rM   s         r=   r   z2_import_bitsandbytes.<locals>._Linear4bit.quantizeO  s     3u||F3F{{f$"%=fkk]#KLL!!#&&fEJJ&GA"%..">">$..$.$B$B%00(66 #? #FK "*fkBBr?   Tr   r   c                d   | j                   j                  t        j                  k(  rEt        j                  | j                   j
                  j                  |t        j                        }n+t        j                  | j                   j                  |      }t        j                  |      }|j                  dk(  r| j                  ||       nt        | j                   |      | _         | j                  :t        | j                  t        j                  | j                  |            | _        | S )Nr   r   r2   )rz   r#   r0   r   emptyr   shaper   r   rc   r   r   r   r   r   r   s       r=   r   z2_import_bitsandbytes.<locals>._Linear4bit.to_emptya  s    {{  EKK/T[[%<%<%B%B6Y^YcYcd))$++*:*:6J\\&)F{{f$vv.,T[[&Ayy$*499e6F6FtyyY_6`a	Kr?   c                    | j                   wt        j                  j                  j	                  | j
                        \  }}|dkD  rdt        j                  |      z  nd}t        j                  | j                   | |       t        | j
                  j                  j                        }|r| j
                  j                  t        j                  k(  rYt        j                  | j
                  j                  j                  | j
                  j                   t        j"                        }n| j
                  j$                  }t        j                  j                  j'                  |t        j                  d             |rQ| j
                  j                   j(                  dk(  r| j+                  |       y t-        | j
                  |      | _        y y )Nr   r   r   r   r   r2   )r   r0   rE   r   r   rz   r   r   r   rD   r   r#   r   r   r   r   r   r   rc   r   r   r   r   r   s         r=   r   z:_import_bitsandbytes.<locals>._Linear4bit.reset_parametersp  s4   yy$!HHMMGGT	17!DIIf--dii%7#-dkk366;L;L#M #(9(9U[[(HT[[%<%<%B%B4;;K]K]ejeoeop))HHMM**6TYYq\*B#;;%%**f4NN6*"0f"EDK	 $r?   rm   r   )rn   ro   rp   rq   r   r   r   r>   r0   r   r   r   r   rE   r   r   r   r   r   r   r   r   s   @r=   _Linear4bitr   5  s    	H DH 		R# 		Rx/@ 		RSV 		R[_ 		R		-HU\\$: 		-8TYT`T`Ka 		-mq 		- 
	C))	C38<<	CIQRWR^R^I_	CVV	C 
	C" @D 	g 	 	 		F 	Fr?   r   c                   ,     e Zd Zdededdf fdZ xZS )2_import_bitsandbytes.<locals>._Int8LinearInferencer   r   r%   Nc                 (    t        |   |ddi| y )Nr   Fr   r>   r:   r   r   r   s      r=   r>   z;_import_bitsandbytes.<locals>._Int8LinearInference.__init__  s    GdEUEfEr?   rn   ro   rp   r   r>   r   r   s   @r=   r-   r     s'    	F# 	F 	F 	F 	Fr?   r-   c                   ,     e Zd Zdededdf fdZ xZS )(_import_bitsandbytes.<locals>._FP4Linearr   r   r%   Nc                 *    t        |   |ddd| y )Nr   Fr   r   r   r   s      r=   r>   z1_import_bitsandbytes.<locals>._FP4Linear.__init__      Gdu%ZSYZr?   r   r   s   @r=   r*   r     '    	[# 	[ 	[ 	[ 	[r?   r*   c                   ,     e Zd Zdededdf fdZ xZS )*_import_bitsandbytes.<locals>._FP4DQLinearr   r   r%   Nc                 *    t        |   |ddd| y )Nr   Tr   r   r   s      r=   r>   z3_import_bitsandbytes.<locals>._FP4DQLinear.__init__      Gdu$YRXYr?   r   r   s   @r=   r+   r     '    	Z# 	Z 	Z 	Z 	Zr?   r+   c                   ,     e Zd Zdededdf fdZ xZS )(_import_bitsandbytes.<locals>._NF4Linearr   r   r%   Nc                 *    t        |   |ddd| y )Nr   Fr   r   r   s      r=   r>   z1_import_bitsandbytes.<locals>._NF4Linear.__init__  r   r?   r   r   s   @r=   r(   r     r   r?   r(   c                   ,     e Zd Zdededdf fdZ xZS )*_import_bitsandbytes.<locals>._NF4DQLinearr   r   r%   Nc                 *    t        |   |ddd| y )Nr   Tr   r   r   s      r=   r>   z3_import_bitsandbytes.<locals>._NF4DQLinear.__init__  r   r?   r   r   s   @r=   r)   r    r   r?   r)   )r,   r   r-   r*   r+   r(   r)   )_BITSANDBYTES_AVAILABLEModuleNotFoundErrorrr   osenvironwarningsfilterwarningsr   rE   rK   rL   r7   update)
nowelcome_setr,   r   r-   r*   r+   r(   r)   classesrM   s
            @r=   r.   r.      s"   "!#&=">??,

:M/2

+,H.bce JJ/0UF++ UFnMFcff'' MF`F} F[[ [Z{ Z[[ [Z{ Z '" 4 $ $G IWJr?   
linear_clsr$   prefixc                 >   | j                         D ]  \  }}|r| d| n|t        |t        j                  j                        rCt        fd|D              s.t        j                  dd       |j                  d u}t               d   }t        j                  t        ||      rdnd      } ||j                  |j                  ||      }	|r=t        |	j                  |j                  j                  j!                               |	_        d	t        ||      r|	j"                  j$                  nd i}
t        |	j"                  |j"                  j                  j!                         fi |
|	_        | j'                  ||	       zt)        |||
        y )N.c              3   @   K   | ]  }j                  |        y wrC   )r/   )rG   sfullnames     r=   rI   z"_convert_layers.<locals>.<genexpr>  s      9iZhUV(:M:Ma:PZhs   zReplacing layer z with bitsandbytes equivalentr   r   cpu)r   r   r   )r  )named_childrenrD   r0   rE   rF   rN   logdebugr   r7   r   
issubclassin_featuresout_featuresr   rc   clonerz   r   __setattr__rQ   )r@   r  r$   r  r|   childhas_biasr   r   replacementstater  s              @r=   rQ   rQ     sX   ,,.e)/fXQtf%TeUXX__-c9iZh9i6iII(4QRSzz-H ")M2K\\Jz;,O&UZ[F$!!""	K #1+2B2BEJJOODYDYD[#\ "jQ[]hFiK$6$6$B$BostE!/0B0BELLDUDUD[D[D]!gaf!gKt[1E:~hO- /r?   rC   ) ):	functoolsloggingr   r  r  collectionsr   
contextlibr   r   r   typesr   typingr   r	   r
   r   r   r0   lightning_utilitiesr    lightning_utilities.core.importsr   r   torch.nnr   torch.nn.modules.moduler   typing_extensionsr   r   ,lightning.fabric.plugins.precision.precisionr   (lightning.fabric.plugins.precision.utilsr   r   r    lightning.fabric.utilities.typesr   	getLoggerrn   r  r  r   r   rE   rs   r   r   tupler   	lru_cacher.   r   r9   rr   rQ   rt   r?   r=   <module>r3     s      	  # 8   9 9  3 =   5 , B 
 5g!*>: oxI oxd%,,1E(F T_ eh mq 7 7M^ 7cg 7 SW88%*\\@H
XX6 QXj X  XvPEHHOO P PsSVx Pad Pnr Pr?   