Ë
      çiÝ  ã                   óì   — d dl mZ d dlmZmZmZmZ d dlZd dlm	Z	 d dlm
Z
 d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZmZ d dlmZ d dlmZ erd dlmZ d dl m!Z! ed   Z" G d„ de«      Z#y)é    )ÚAbstractContextManager)ÚTYPE_CHECKINGÚAnyÚLiteralÚOptionalN)Úapply_to_collection)ÚTensor)ÚModule)Ú	Optimizer)Úget_argsÚoverride)Ú_optimizer_handles_unscaling)Ú	Precision)Ú_convert_fp_tensorÚ_DtypeContextManager)Úrank_zero_warn)ÚOptimizable©ÚMixedPrecision©ÚShardedGradScaler)ú32-trueú16-trueú	bf16-trueú16-mixedú
bf16-mixedc                   ó†  ‡ — e Zd ZdZddeded   ddfd„Zededefd	„«       Z	e
dd
„«       Zedefd„«       Zedefd„«       Zedefd„«       Zededefd„«       Zededefd„«       Zededee   dededdf
ˆ fd„«       Zedededefˆ fd„«       Zededdfd„«       Zedeeef   fd„«       Zedeeef   ddfd„«       Zˆ xZS )ÚFSDPPrecisionaù  Precision plugin for training with Fully Sharded Data Parallel (FSDP).

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    Args:
        precision: Full precision (32-true), half precision (16-true, bf16-true) or
            mixed precision (16-mixed, bf16-mixed).
        scaler: An optional :class:`torch.distributed.fsdp.sharded_grad_scaler.ShardedGradScaler` to use.

    Raises:
        ValueError:
            If unsupported ``precision`` is provided.

    NÚ	precisionÚscalerr   Úreturnc                 óœ  — t        t        «      }||vrt        d|›d|› d�«      ‚ddlm} |�!| j
                  dk7  rt        d|›d|› d�«      ‚|€|dk(  r |«       nd | _        || _        t        j                  t        j                  t        j                  t        j                  t        j                  dœ}|| j
                     | _        y )	Nz`precision=z9)` is not supported in FSDP. `precision` must be one of: Ú.r   r   r   z` does not use a scaler, found )r   r   r   r   r   )r   Ú_PRECISION_INPUTÚ
ValueErrorÚ*torch.distributed.fsdp.sharded_grad_scalerr   r   r    ÚtorchÚfloat32Úbfloat16Úfloat16Ú_desired_input_dtype)Úselfr   r    Úsupported_precisionr   Úprecision_to_types         ú|/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning_fabric/plugins/precision/fsdp.pyÚ__init__zFSDPPrecision.__init__5   sÓ   € Ü&Ô'7Ó8ÐØÐ/Ñ/ÜØ˜i˜]ð +0Ø0CÐ/DÀAðGóð õ
 	QàÐ $§.¡.°JÒ">Ü˜{¨9¨-Ð7VÐW]ÐV^Ð^_Ð`ÓaÐaà-3¨^À	ÈZÒ@WÑ'Ô)Ð]aˆŒØ"ˆŒô  Ÿ-™-ÜŸ™ÜŸ™Ü—}‘}Ü—}‘}ñ
Ðð %6°d·n±nÑ$EˆÕ!ó    Úmodulec                 óZ   — d| j                   v r|j                  | j                  ¬«      S |S )NÚtrue©Údtype)r   Útor+   )r,   r2   s     r/   Úconvert_modulezFSDPPrecision.convert_moduleN   s*   € à�T—^‘^Ñ#Ø—9‘9 4×#<Ñ#<�9Ó=Ð=Øˆr1   c                 óx  — ddl m} | j                  dv rt        d| j                  › d�«       | j                  dv rt        j
                  x}x}}n`| j                  dv rt        j                  x}x}}n=| j                  dk(  rt        j                  x}x}}nt        d	| j                  ›d
�«      ‚ ||||¬«      S )Nr   r   )r   r   zFSDP with `z}` enables computation in lower precision. FSDP will always retain a full-precision copy of the model parameters for sharding.)r   r   )r   r   r   z-Was unable to infer precision type, received r#   )Úparam_dtypeÚreduce_dtypeÚbuffer_dtype)	Ú2torch.distributed.fsdp.fully_sharded_data_parallelr   r   r   r'   r*   r)   r(   r%   )r,   ÚTorchMixedPrecisionr:   r;   r<   s        r/   Úmixed_precision_configz$FSDPPrecision.mixed_precision_configT   sÄ   € ålà�>‰>Ð5Ñ5ÜØ˜dŸn™nÐ-ð .fð fôð
 �>‰>Ð4Ñ4Ü8=¿¹ÐEˆKÐE˜,©Ø�^‰^Ð:Ñ:Ü8=¿¹ÐFˆKÐF˜,©Ø�^‰^˜yÒ(Ü8=¿¹ÐEˆKÐE˜,©äÐLÈTÏ^É^ÐL^Ð^_Ð`ÓaÐaá"Ø#Ø%Ø%ô
ð 	
r1   c                 ó,   — t        | j                  «      S ©N)r   r+   ©r,   s    r/   Útensor_init_contextz!FSDPPrecision.tensor_init_contextm   s   € ä# D×$=Ñ$=Ó>Ð>r1   c                 ód   — t        | j                  j                  xs t        j                  «      S rA   )r   r?   r:   r'   r(   rB   s    r/   Úmodule_init_contextz!FSDPPrecision.module_init_contextq   s"   € ä# D×$?Ñ$?×$KÑ$KÒ$\ÌuÏ}É}Ó]Ð]r1   c                 óÐ   — d| j                   v rIt        j                  d| j                   dk(  rt        j                  ¬«      S t        j                  ¬«      S | j                  «       S )NÚmixedÚcudar   r5   )r   r'   Úautocastr)   r*   rC   rB   s    r/   Úforward_contextzFSDPPrecision.forward_contextu   sM   € à�d—n‘nÑ$Ü—>‘> &À4Ç>Á>ÐUaÒCa´·±ÔvÐvÔgl×gtÑgtÔvÐvØ×'Ñ'Ó)Ð)r1   Údatac                 óD   — t        |t        t        | j                  ¬«      S ©N)Úfunctionr6   Údst_type)r   r   r	   r+   ©r,   rK   s     r/   Úconvert_inputzFSDPPrecision.convert_input{   s   € ä" 4Ô2DÌFÐ]a×]vÑ]vÔwÐwr1   c                 óT   — t        |t        t        t        j                  «       ¬«      S rM   )r   r   r	   r'   Úget_default_dtyperP   s     r/   Úconvert_outputzFSDPPrecision.convert_output   s   € ä" 4Ô2DÌFÔ]b×]tÑ]tÓ]vÔwÐwr1   ÚtensorÚmodelÚargsÚkwargsc                 ó|   •— | j                   �| j                   j                  |«      }t        ‰| �  ||g|¢­i |¤Ž y rA   )r    ÚscaleÚsuperÚbackward)r,   rU   rV   rW   rX   Ú	__class__s        €r/   r\   zFSDPPrecision.backwardƒ   s:   ø€ à�;‰;Ð"Ø—[‘[×&Ñ& vÓ.ˆFÜ‰Ñ˜ Ð8¨Ò8°Ó8r1   Ú	optimizerc                 ó®   •— | j                   €t        ‰| �  |fi |¤ŽS  | j                   j                  |fi |¤Ž}| j                   j	                  «        |S rA   )r    r[   Úoptimizer_stepÚstepÚupdate)r,   r^   rX   Ústep_outputr]   s       €r/   r`   zFSDPPrecision.optimizer_step‰   sU   ø€ ð �;‰;Ðä‘7Ñ)¨)Ñ>°vÑ>Ð>à&�d—k‘k×&Ñ& yÑ;°FÑ;ˆØ�‰×ÑÔØÐr1   c                 óp   — | j                   }|�(t        |«      rt        d«      ‚|j                  |«       y y )NzKGradient clipping is not implemented for optimizers handling the unscaling.)r    r   ÚNotImplementedErrorÚunscale_)r,   r^   r    s      r/   Úunscale_gradientszFSDPPrecision.unscale_gradients—   s6   € à—‘ˆØÐÜ+¨IÔ6Ü)Ð*wÓxÐxØ�O‰O˜IÕ&ð r1   c                 óR   — | j                   �| j                   j                  «       S i S rA   )r    Ú
state_dictrB   s    r/   ri   zFSDPPrecision.state_dictŸ   s$   € à�;‰;Ð"Ø—;‘;×)Ñ)Ó+Ð+Øˆ	r1   ri   c                 óT   — | j                   �| j                   j                  |«       y y rA   )r    Úload_state_dict)r,   ri   s     r/   rk   zFSDPPrecision.load_state_dict¥   s#   € à�;‰;Ð"Ø�K‰K×'Ñ'¨
Õ3ð #r1   rA   )r!   r>   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r$   r   r0   r   r
   r8   Úpropertyr?   r   rC   rE   rJ   r   rQ   rT   r	   r\   r   r`   r   rg   ÚdictÚstrri   rk   Ú__classcell__)r]   s   @r/   r   r   %   sä  ø„ ññFÐ"2ð F¸HÐEXÑ<Yð FÐeió Fð2 ð Vð °ò ó ðð
 ò
ó ð
ð0 ð?Ð%;ò ?ó ð?ð ð^Ð%;ò ^ó ð^ð ð*Ð!7ò *ó ð*ð
 ðx #ð x¨#ò xó ðxð ðx 3ð x¨3ò xó ðxð ð9˜vð 9¨h°vÑ.>ð 9Àsð 9ÐVYð 9Ð^bô 9ó ð9ð
 ðàðð ðð 
ô	ó ðð ð'¨9ð '¸ò 'ó ð'ð ð˜D  c ™Nò ó ðð
 ð4¨$¨s°C¨x©.ð 4¸Tò 4ó ô4r1   r   )$Ú
contextlibr   Útypingr   r   r   r   r'   Úlightning_utilitiesr   r	   Útorch.nnr
   Útorch.optimr   Útyping_extensionsr   r   Ú&lightning_fabric.plugins.precision.ampr   Ú,lightning_fabric.plugins.precision.precisionr   Ú(lightning_fabric.plugins.precision.utilsr   r   Úlightning_fabric.utilitiesr   Ú lightning_fabric.utilities.typesr   r=   r   r>   r&   r   r$   r   © r1   r/   Ú<module>r€      sT   ðõ .ß 8Ó 8ã Ý 3Ý Ý Ý !ß 0å OÝ Bß ]Ý 5Ý 8áÝhÝLàÐVÑWÐ ôC4�Iõ C4r1   