+
    QV-j&  ã                   ó‚   € ^ RI t ^ RIt^ RIt^ RIHt ^RIHtHtHtH	t	 ]	P                  ! ]4      tR R ltR tRR ltR# )	é    N©Ú
DataLoader)ÚWEIGHTS_NAMEÚPushToHubMixinÚis_torch_xla_availableÚloggingc                ó$   € V ^8„  d   QhR\         /# )é   Ú
dataloaderr   )Úformats   "Ún/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/integrations/tpu.pyÚ__annotate__r      s   € ÷ ñ ¤Jñ ó    c                 óî   € \        4       '       de   ^ RIHu Hp \	        WP
                  4      '       g   Q R4       h^ RIHu Hp VP                  VP                  4       R4      pW0P                  R&   V # V # )r   NzPThe dataloader must be a `torch_xla.distributed.parallel_loader.MpDeviceLoader`.Úinput_sharding)ÚfsdpN)r   Ú%torch_xla.distributed.parallel_loaderÚdistributedÚparallel_loaderÚ
isinstanceÚMpDeviceLoaderÚtorch_xla.distributed.spmdÚspmdÚShardingSpecÚget_global_meshÚ_parallel_loader_kwargs)r   ÚplÚxsÚsharding_specs   &   r   Útpu_spmd_dataloaderr       sn   € Ü×Òß:Ð:ä˜*×&7Ñ&7×8Ò8ð 	
Ø^ó	
Ð8÷ 	0Ð/àŸ™¨×(:Ñ(:Ó(<¸nÓMˆØ?L×*Ñ*Ð+;Ñ<ØÐàÐr   c                ó˜  aaaaaa€ ^ RI Hu Ho ^ RIHu Ho ^RIHp  ^ RIH	o ^ RIH
o ^ RIHpHp S'       d   ^ RIHo RpRp\#        T RR4      pTP$                  P'                  R	T4      p	TP$                  R
,          ^ 8”  d*   \(        P*                  ! YAP$                  R
,          R7      pMWT	eT   \-        4       p
T	 F+  pT! Y4      pTf   \/        R4      hT
P1                  T4       K-  	  \(        P*                  ! TT
R7      pTP2                  pTP$                  R,          '       dL   T P4                  P6                  '       d'   \8        P;                  R4       RT P4                  n        TTTT3R lpS'       d   T3R lpS! T TTTR7      p MS! T 3RTRT/TB p R/ 3T3R llpTSn        T #   \          d    \!        R4      hi ; i)a  
Wraps a model with XLA Fully Sharded Data Parallelism (FSDP).

Handles both FSDP v1 (`XlaFullyShardedDataParallel`) and v2 (`SpmdFullyShardedDataParallel`),
including auto-wrap policies, gradient checkpointing, and patching `xm.optimizer_step`.

Args:
    model (`torch.nn.Module`): The model to wrap.
    args (`TrainingArguments`): The training arguments containing FSDP configuration.
    is_fsdp_xla_v2_enabled (`bool`): Whether FSDP v2 (SPMD) is enabled.

Returns:
    `torch.nn.Module`: The FSDP-wrapped model.
N)Úget_module_class_from_name)ÚXlaFullyShardedDataParallel)Úcheckpoint_module)Úsize_based_auto_wrap_policyÚtransformer_auto_wrap_policy)ÚSpmdFullyShardedDataParallelzJMissing XLA FSDP related module; please make sure to use torch-xla >= 2.0.Ú_no_split_modulesÚtransformer_layer_cls_to_wrapÚmin_num_params)r*   z@Could not find the transformer layer class to wrap in the model.)Útransformer_layer_clsÚxla_fsdp_grad_ckptzX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fc                 ó@   <€ S'       g   SMSpV! S! V 4      .VO5/ VB # ©N© )ÚmÚargsÚkwargsÚ
target_clsÚFSDPÚFSDPv2r$   Úis_fsdp_xla_v2_enableds   &*, €€€€r   Úauto_wrapper_callableÚ2wrap_model_xla_fsdp.<locals>.auto_wrapper_callablet   s'   ø€ ß%;™ÀˆJÙÑ/°Ó2ÐD°TÒD¸VÑDÐDr   c                 ó  <€ ^RI Hp Rp\        V \        P                  4      '       d   T pM=\        V \
        4      '       d   V ^ ,          pM\        W4      '       d   V P                  pVf   \        R4      hSP                  W1R4       R# )r
   )ÚCausalLMOutputWithPastNzASomething went wrong, the output of the model shouldn't be `None`)r   NN)	Úmodeling_outputsr:   r   ÚtorchÚTensorÚtupleÚlogitsÚ
ValueErrorÚmark_sharding)ÚoutputÚmeshr:   Úreal_outputr   s   &&  €r   Úshard_outputÚ)wrap_model_xla_fsdp.<locals>.shard_output{   sn   ø€ ÝAàˆKÜ˜&¤%§,¡,×/Ò/Ø$‘Ü˜F¤E×*Ò*Ø$ Q�i‘Ü˜F×;Ò;Ø$Ÿm™m�àÒ"Ü Ð!dÓeÐeØ×Ñ˜[Ð0DÖEr   )rE   Úauto_wrap_policyr7   rG   r7   c                 ó\   <€ V P                   ! R/ VB pV'       d   SP                  4        V# )Nr/   )ÚstepÚ	mark_step)Ú	optimizerÚbarrierÚoptimizer_argsÚlossÚxms   &&& €r   Úpatched_optimizer_stepÚ3wrap_model_xla_fsdp.<locals>.patched_optimizer_stepš   s%   ø€ Ø�~Š~Ñ/ Ñ/ˆßØ�L‰LŒNØˆr   )Útorch_xla.core.xla_modelÚcoreÚ	xla_modelr   r   r   Útrainer_pt_utilsr"   Útorch_xla.distributed.fsdpr#   r$   Útorch_xla.distributed.fsdp.wrapr%   r&   Ú7torch_xla.experimental.spmd_fully_sharded_data_parallelr'   ÚImportErrorÚgetattrÚfsdp_configÚgetÚ	functoolsÚpartialÚsetÚ	ExceptionÚaddÚxla_fsdp_configÚconfigÚ	use_cacheÚloggerÚwarning_onceÚoptimizer_step)Úmodelr1   r6   r"   r%   r&   rG   r7   Ú%default_transformer_cls_names_to_wrapÚ"fsdp_transformer_layer_cls_to_wrapÚtransformer_cls_to_wrapÚlayer_classÚtransformer_clsÚfsdp_kwargsrE   rP   r4   r5   r$   rO   r   s   &&f             @@@@@r   Úwrap_model_xla_fsdpro   .   sá  ý€ ÷ *Ð)ß+Ð+å=ðhÝRÝ@÷	
÷
 "õð ÐØ ÐÜ,3°EÐ;NÐPTÓ,UÐ)Ø)-×)9Ñ)9×)=Ñ)=Ø'Ð)Nó*Ð&ð ×ÑÐ(Õ)¨AÔ-Ü$×,Ò,Ø'×8HÑ8HÐIYÕ8Zô
Ñð 
,Ò	7Ü"%£%ÐÛ=ˆKÙ8¸ÓLˆOØÒ&ÜÐ bÓcÐcà'×+Ñ+¨OÖ<ñ >ô %×,Ò,Ø(à"9ô
Ðð ×&Ñ&€KØ×ÑÐ,×-Ô-Ø�<‰<×!×!Ð!Ü×ÑØjôð &+ˆE�L‰LÔ"÷	Eð 	E÷
 õ	Fñ ØØ%Ø-Ø"7ô	
‰ñ Øñ
à-ð
ð #8ð
ð ñ	
ˆð 38È÷ ð /€BÔà€Løôi ô hÜÐfÓgÐgðhús    F2 ¼F2 Æ2G	c           	     ó   € ^ RI Hu Hp Ve   TMVP                  p\        P                  RV 24       VP                  4        VP                  RR7      '       dN   \        P                  ! VRR7       \        P                  ! V\        P                  P                  VR4      4       \        3pVP                  R4       V'       Edi   R	V P!                  4       R
V P#                  4       /p\        P                  P                  VRVP$                   RVP&                   R\(         24      p	VP                  W‰RR7       VP                  R4       VP*                  '       dÎ   ^ RIHp
 V
! \        P                  P                  VR4      R\(         2RR7      w  r¼V P0                  P0                  p VP3                  V 4      p\5        W×4      '       d   VP7                  W[R7       EMl\        P                  R4       VP                  V\        P                  P                  V\(        4      4       EM!\5        W4      '       gÖ   \5        VP3                  V 4      V4      '       dL   VP3                  V 4      P7                  VVP*                  VP9                  V P!                  4       4      R7       M¤\        P                  R4       VP9                  V P!                  4       4      pVP                  V\        P                  P                  V\(        4      4       M;V P7                  VVP*                  VP9                  V P!                  4       4      R7       Ve(   VP*                  '       d   VP7                  V4       R# R# R# )ah  
Saves a model checkpoint on TPU/XLA devices.

Handles FSDP v1 sharded checkpoints (with consolidation on master), as well as
standard XLA model saving via `save_pretrained` or `xm.save`.

Args:
    model (`torch.nn.Module`): The model to save.
    args (`TrainingArguments`): The training arguments.
    accelerator (`Accelerator`): The accelerator instance.
    processing_class: The processing class (tokenizer/processor) to save alongside the model.
    is_fsdp_xla_v1_enabled (`bool`): Whether FSDP XLA v1 is enabled.
    output_dir (`str`, *optional*): The directory to save to. Defaults to `args.output_dir`.
NzSaving model checkpoint to F)ÚlocalT)Úexist_okztraining_args.binÚsaving_checkpointrh   Úshard_metadataÚrankz-of-Ú-)Úmaster_onlyÚsave_full_checkpoints)Ú%consolidate_sharded_model_checkpointsÚ zrank*-of-*-)Úckpt_prefixÚckpt_suffixÚ
save_model)Ú
state_dictzETrainer.model is not a `PreTrainedModel`, only saving its state dict.)Úis_main_processr~   )rR   rS   rT   Ú
output_dirre   ÚinforJ   Úis_master_ordinalÚosÚmakedirsr<   ÚsaveÚpathÚjoinr   Ú
rendezvousr~   Úget_shard_metadataÚprocess_indexÚ
world_sizer   Úshould_saverV   ry   ÚmoduleÚunwrap_modelr   Úsave_pretrainedÚ_maybe_convert_to_cpu)rh   r1   ÚacceleratorÚprocessing_classÚis_fsdp_xla_v1_enabledr€   rO   Úsupported_classesÚckptÚ	ckpt_pathry   Úfull_state_dictÚ_Úunwrapped_modelr~   s   &&&&&&         r   Úsave_tpu_checkpointrš   ¥   s¹  € ÷ *Ð)à)Ò5‘¸4¿?¹?€Jä
‡K�KÐ-¨j¨\Ð:Ô;Ø‡L�L„Nà	×Ñ %Ð×(Ò(Ü
�Š�J¨Õ.Ü�
Š
�4œŸ™Ÿ™ jÐ2EÓFÔGô (Ð)ÐØ‡M�MÐ%Ô&ßÐà�U×%Ñ%Ó'Ø˜e×6Ñ6Ó8ð
ˆô —G‘G—L‘L ¨t°D×4FÑ4FÐ3GÀtÈDÏOÉOÐK\Ð\]Ô^jÐ]kÐ-lÓmˆ	à
�‰�¨UˆÔ3à
�‰Ð-Ô.à××ÐÝXá!FÜŸG™GŸL™L¨°RÓ8Ø)¬,¨Ð8Ø ô"ÑˆOð
 —L‘L×'Ñ'ˆEØ)×6Ñ6°uÓ=ˆOÜ˜/×=Ò=Ø×/Ñ/°
Ð/ÖWä—‘ÐcÔdØ—‘˜¬¯©¯©°jÄ,Ó)OÔPùÜ˜×1Ò1Ü�k×.Ñ.¨uÓ5Ð7H×IÒIØ×$Ñ$ UÓ+×;Ñ;ØØ $× 0Ñ 0Ø×3Ñ3°E×4DÑ4DÓ4FÓGð <õ ô �K‰KÐ_Ô`Ø×1Ñ1°%×2BÑ2BÓ2DÓEˆJØ�G‰G�J¤§¡§¡¨Z¼Ó FÕGà×ÑØØ ×,Ñ,Ø×/Ñ/°×0@Ñ0@Ó0BÓCð 	ô 	
ð
 Ò#¨×(8×(8Ð(8Ø×(Ñ(¨Ö4ñ )9Ñ#r   r.   )r]   rƒ   r<   Útorch.utils.datar   Úutilsr   r   r   r   Ú
get_loggerÚ__name__re   r    ro   rš   r/   r   r   Ú<module>rŸ      s=   ðó Û 	ã Ý 'ç QÓ Qð 
×	Ò	˜HÓ	%€õò&tönJ5r   