(ti, ddlZddlZddlZddlmZddlmZddlmZddl Z ddl Z ddl m Z ddl mZmZmZmZmZmZmZmZmZmZmZmZddlmZmZd d lmZmZd d l m!Z!ejDe#Z$id e!d e!ddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4Z%Gd5d6Z&y)7N)partial)Path)Literal*_maybe_remove_and_reapply_group_offloading) MIN_PEFT_VERSIONUSE_PEFT_BACKENDcheck_peft_version.convert_sai_sd_control_lora_state_dict_to_peftconvert_unet_state_dict_to_peftdelete_adapter_layersget_adapter_nameis_peft_availableis_peft_versionloggingset_adapter_layers!set_weights_and_activate_adapters)_create_lora_config_maybe_warn_for_unhandled_keys)_fetch_state_dict#_func_optionally_disable_offloading)_maybe_expand_lora_scalesUNet2DConditionModelUNetMotionModelSD3Transformer2DModelc|SN model_clsweightss e/Volumes/fast/ai/experiments/MLX_z-image/.venv/lib/python3.12/site-packages/diffusers/loaders/peft.pyr%2FluxTransformer2DModelc|Srr r!s r$r%r%3r'CogVideoXTransformer3DModelc|Srr r!s r$r%r%4gr'ConsisIDTransformer3DModelc|Srr r!s r$r%r%5Wr'HeliosTransformer3DModelc|Srr r!s r$r%r%67r'MochiTransformer3DModelc|Srr r!s r$r%r%7'r'HunyuanVideoTransformer3DModelc|Srr r!s r$r%r%8r'LTXVideoTransformer3DModelc|Srr r!s r$r%r%9r0r'SanaTransformer2DModelc|Srr r!s r$r%r%:r*r'AuraFlowTransformer2DModelc|Srr r!s r$r%r%;r0r'Lumina2Transformer2DModelc|Srr r!s r$r%r%<Gr'WanTransformer3DModelc|Srr r!s r$r%r%=r&r'CogView4Transformer2DModelc|Srr r!s r$r%r%>r0r'HiDreamImageTransformer2DModelc|Srr r!s r$r%r%?r9r''HunyuanVideoFramepackTransformer3DModelc|Srr r!s r$r%r%@s'r'c|Srr r!s r$r%r%ArBr'c|Srr r!s r$r%r%Br3r'c|Srr r!s r$r%r%Cswr'c|Srr r!s r$r%r%Dr-r'c|Srr r!s r$r%r%Er6r'c|Srr r!s r$r%r%Fr3r'c|Srr r!s r$r%r%Gr-r'c|Srr r!s r$r%r%HsWr')WanVACETransformer3DModelChromaTransformer2DModelChronoEditTransformer3DModelQwenImageTransformer2DModelFlux2Transformer2DModelZImageTransformer2DModelLTX2VideoTransformer3DModelLTX2TextConnectorsc veZdZUdZdZdZedzed<edZ d"de fdZ d#de d e d e d e dzfd Z d$d ee e zdeezeezeezedzdzfdZd%de ddfdZde ee zddfdZd&dZd&dZdee fdZd'dZd$dZdZdZdZdZdZd ee e zfdZ d(deded ddfd!Z y))PeftAdapterMixina A class containing all functions for loading and using adapters weights that are supported in PEFT library. For more details about adapters and injecting them in a base model, check out the PEFT [documentation](https://huggingface.co/docs/peft/index). Install the latest version of PEFT, and use this mixin to: - Attach new adapters in the model. - Attach multiple adapters and iteratively activate/deactivate them. - Activate/deactivate all adapters from the model. - Get a list of the active adapters. FN_prepare_lora_hotswap_kwargsct|S)N) _pipeline)r)clsr_s r$_optionally_disable_offloadingz/PeftAdapterMixin._optionally_disable_offloading^s3YGGr'hotswapc 2ddlm}m}ddlm}ddlm}|jdd} |jdd } |jd d} |jd d} |jd d} |jd d}|jdd}|jdd}|jdd}|jdd2|jdd}|jdd}|jdd }|jdd}d }|rtddr tdddd}t|||| | | | | ||||| \}}| | td|r |r td||jDcic]/\}}|j|ds|j|d|1}}}|J|jDcic]/\}}|j|ds|j|d|1}}}t|dkDr2t|d ivr|std!2d"2t|d ivr|rtd#2d$t!t#|j%}d%|vr t'|}d&|v}|r t)|}i}|jD]/\}}d'|vs |j*d(kDs|j,d(|d)|<1|zt|d(k\rl|j%Dcgc]}|j|ds|} }|jDcic]\}}|| vs |j|d|!}}}2 t/|2t1|||||j32*}!|rA|!j4|!_|!j8|!_d+|!_|!j>|!_ d|!_|jC|\}"}#}$i}%td,d-r||%d<|s |jD$td.d/r dd0l#m$}&m%}'m&}(n d1})tO|)|r2fd2}* |r.*|}&|jP2|! '||2|!3d},n@||!|f2|d6|%|||2fi|%},|jD(|fd7|!i|jDd|_"|jXsd8|_,te|,2|"r|jgna|#r|jinN|$rL|jjjmD]/}0t_|0tnjpjrs(||01|9|s6|jtjv}1tTjyd:|1d;|d<|1d=yyycc}}wcc}}wcc}wcc}}w#tR$r!}+tTjWd42d5|+d}+~+wwxYw#tR$r}+t[|d rf|j]D]8}-t_|-|s|-j`}.|.D]}/2|/vs|-jc2:|jPj2tTjWd92d5|+d}+~+wwxYw)>ah Loads a LoRA adapter into the underlying model. Parameters: pretrained_model_name_or_path_or_dict (`str` or `os.PathLike` or `dict`): Can be either: - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on the Hub. - A path to a *directory* (for example `./my_model_directory`) containing the model weights saved with [`ModelMixin.save_pretrained`]. - A [torch state dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). prefix (`str`, *optional*): Prefix to filter the state dict. cache_dir (`str | os.PathLike`, *optional*): Path to a directory where a downloaded pretrained model configuration is cached if the standard cache is not used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier allowed by Git. subfolder (`str`, *optional*, defaults to `""`): The subfolder location of a model file within a larger model repository on the Hub or locally. network_alphas (`dict[str, float]`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning). low_cpu_mem_usage (`bool`, *optional*): Speed up model loading by only loading the pretrained LoRA weights and not initializing the random weights. hotswap : (`bool`, *optional*) Defaults to `False`. Whether to substitute an existing (LoRA) adapter with the newly loaded adapter in-place. This means that, instead of loading an additional adapter, this will take the existing adapter weights and replace them with the weights of the new adapter. This can be faster and more memory efficient. However, the main advantage of hotswapping is that when the model is compiled with torch.compile, loading the new adapter does not require recompilation of the model. When using hotswapping, the passed `adapter_name` should be the name of an already loaded adapter. If the new adapter and the old adapter have different ranks and/or LoRA alphas (i.e. scaling), you need to call an additional method before loading the adapter: ```py pipeline = ... # load diffusers pipeline max_rank = ... # the highest rank among all LoRAs that you want to load # call *before* compiling and loading the LoRA adapter pipeline.enable_lora_hotswap(target_rank=max_rank) pipeline.load_lora_weights(file_name) # optionally compile the model now ``` Note that hotswapping adapters of the text encoder is not yet supported. There are some further limitations to this technique, which are documented here: https://huggingface.co/docs/peft/main/en/package_reference/hotswap metadata: LoRA adapter metadata. When supplied, the metadata inferred through the state dict isn't used to initialize `LoraConfig`. r)inject_adapter_in_modelset_peft_model_state_dictBaseTunerLayerrr cache_dirNforce_downloadFproxieslocal_files_onlytokenrevision subfolder weight_nameuse_safetensors adapter_namenetwork_alphasr_low_cpu_mem_usagemetadataz<=z0.13.0zq`low_cpu_mem_usage=True` is not compatible with this `peft` version. Please update it with `pip install -U peft`.attn_procs_weightspytorch) file_type framework) %pretrained_model_name_or_path_or_dictrorprkrhrirjrlrmrn user_agent allow_picklertz6`network_alphas` cannot be None when `prefix` is None.z9Both `network_alphas` and `metadata` cannot be specified.. peft_config Adapter name z@ already in use in the model - please select a new adapter name.z Trying to hotswap LoRA adapter 'z' but there is no existing adapter by that name. Please choose an existing adapter name or set `hotswap=False` to prevent hotswapping.lora_Alora_controlnetlora_Br^)model_state_dictrqallz>=z0.13.1>z0.14.0) check_hotswap_configs_compatiblehotswap_adapter_from_state_dict"prepare_model_for_compiled_hotswapzeHotswapping requires PEFT > v0.14. Please upgrade PEFT to a higher version or install it from source.ci}|jD]j\}}|jds|jdr|dtd ddz}n'|jdr|dtd ddz}|||<l|S)Nz lora_A.weightz lora_B.weightz.weightr|z lora_B.biasz.bias)itemsendswithlen)sdnew_sdkvrqs r$map_state_dict_for_hotswapzFPeftAdapterMixin.load_lora_adapter..map_state_dict_for_hotswap;sF " 1::o6!**_:U !"3S^O 4<.7P PAZZ 6 !/S\M 2qe5L LA$%q !+ "Mr')model state_dictrqconfigz Hotswapping z- was unsuccessful with the following error: )rqrrTzLoading zNo LoRA keys associated to z found with the prefix=zG. This is safe to ignore if LoRA state dict didn't originally have any z related params. You can also try specifying `prefix=None` to resolve the warning. Otherwise, open an issue if you think it's unexpected: https://github.com/huggingface/diffusers/issues/new)=peftrdrepeft.tuners.tuners_utilsrghooks.group_offloadingrpopr ValueErrorrr startswith removeprefixrgetattrnextiterkeysr r ndimshaperrrr lora_alpha rank_pattern alpha_patternbiasexclude_modulesmodules_to_saverar]peft.utils.hotswaprrr ImportErrorr} Exceptionloggererror_hf_peft_config_loadedhasattrmodules isinstanceactive_adaptersdelete_adapterrenable_model_cpu_offloadenable_sequential_cpu_offload componentsvaluestorchnnModule __class____name__warning)3selfryprefixrbkwargsrdrergrrhrirjrkrlrmrnrorprrr_rsrtr{rzrrr first_keyis_sai_sd_control_lorarankkeyval alpha_keys lora_configis_model_cpu_offloadis_sequential_cpu_offloadis_group_offload peft_kwargsrrrmsgreincompatible_keysmoduleractive_adapter componentmodel_class_namerqs3 @r$load_lora_adapterz"PeftAdapterMixin.load_lora_adaptercsR L;WJJ{D1 $4e<**Y-!::&8$? 7D)::j$/JJ{D1 jj5  **%6=zz.$7 $4d;JJ{D1 "JJ':EB::j$/ x!@D $8iP 02W#+-)!%  H  %&.UV V hXY Y  FPFVFVFXwFXda\]\h\hlrksstiu\v!..F8169FXJw#HPHXwHX1\]\h\hlrksstiu\vANNfXQ<8!;HXw z?Q wt]B?? #L>1qrWT="%EE' 6|nEll T*//"345Iy(])>A!,,RXQYYZ|B\a)> ]@N@T@T@V"@V1Z[_iZiANNfXQ<0!3@V" #/5 .!%!2) K&)4 &,7,D,D )#(  .9.I.I +.2 +QUPsPsQ M ";=MKtX.3D /04<<H"31'&c** "2 !;J!GJ4T5E5El5SU`a 7"&'1)5#. )-%+#T8DQ[_j)B$ T`(pdo(p%88D ; )48<8Y8Y=A92226D/ ++I!)UXX__=B9M"?  j#~~66  NN-.>-??WPVyYX#$%FF '1 [xwJ^"^% |L>Aopqor%st6 4/"&,,.%fn=.4.D.DO2A#/>#A$*$9$9,$G3B#1$$((6 x ~5cdecfgh szU$U$ U*)U*:U0U0- U5:U5% V( U;AV(; V%V  V%%V(( Y1+X?X?3A X??Yrqupcast_before_savingsafe_serializationrocT ddlm}ddlm m}m}| t |}|t|divrtd|d|j|j ||j|rtjnd | } tjj!|rtd |d |r fd } ntj"} tj$|d||r|}n|}t'||j)} | | | t*j-d| y)al Save the LoRA parameters corresponding to the underlying model. Arguments: save_directory (`str` or `os.PathLike`): Directory to save LoRA parameters to. Will be created if it doesn't exist. adapter_name: (`str`, defaults to "default"): The name of the adapter to serialize. Useful when the underlying model has multiple adapters loaded. upcast_before_saving (`bool`, defaults to `False`): Whether to cast the underlying model to `torch.float32` before serialization. safe_serialization (`bool`, *optional*, defaults to `True`): Whether to save the model using `safetensors` or the traditional PyTorch way with `pickle`. weight_name: (`str`, *optional*, defaults to `None`): Name of the file to serialize the state dict with. r)get_peft_model_state_dictr)LORA_ADAPTER_METADATA_KEYLORA_WEIGHT_NAMELORA_WEIGHT_NAME_SAFENr}r~z not found in the model.)dtype)rqzProvided path (z#) should be a directory, not a filecddi}RjD]$\}}t|tst||<&t j dd|<t jj|||S)NformatptrT)indent sort_keys)rt) rrsetlistjsondumps safetensorsr save_file)r#filenamertrvaluerlora_adapter_metadatas r$ save_functionz9PeftAdapterMixin.save_lora_adapter..save_functions$d+(4&;&A&A&C U%eS19=e1#6'D;?**EZcdpt:uH67"((227Hx2XXr'T)exist_okzModel weights saved in ) peft.utilsr lora_baserrrrrrr}to_dicttorfloat32ospathisfilesavemakedirsras_posixrinfo)rsave_directoryrqrrrorrrlora_layers_to_saver save_pathrrs @@r$save_lora_adapterz"PeftAdapterMixin.save_lora_adapters, 9aa  +D1L wt]B? ?}\N:RST T $ 0 0 > F F H7 GG+?%--TG JYe  77>>. )~.>>abc c  Y"JJM NT2  !3 . 5>>@ )95 -i[9:r' adapter_namesr#cts tdt|tr|gn|}t|ts|gt |z}t |t |k7r$tdt |dt |d|Dcgc]}||nd }}t |jj}|||}t|||ycc}w)a: Set the currently active adapters for use in the diffusion network (e.g. unet, transformer, etc.). Args: adapter_names (`list[str]` or `str`): The names of the adapters to use. weights (`Union[List[float], float]`, *optional*): The adapter(s) weights to use with the UNet. If `None`, the weights are set to `1.0` for all the adapters. Example: ```py from diffusers import AutoPipelineForText2Image import torch pipeline = AutoPipelineForText2Image.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 ).to("cuda") pipeline.load_lora_weights( "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic" ) pipeline.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors", adapter_name="pixel") pipeline.unet.set_adapters(["cinematic", "pixel"], weights=[0.5, 0.5]) ``` z.PEFT backend is required for `set_adapters()`.zLength of adapter names z- is not equal to the length of their weights r|N?) r rrstrrr_SET_ADAPTER_SCALE_FN_MAPPINGrrr)rrr#wscale_expansion_fns r$ set_adapterszPeftAdapterMixin.set_adapterss> MN N+5mS+I} '4(i#m"44G } W -*3}+=*>>klopwlxkyyz{  9@@1 13.@;4>>;R;RS$T73)$ wG As CreturncLttts tdddlm}m}|jsd|_n||jvrtd|dt||stdt|d d |_ |||||j|y ) a Adds a new adapter to the current model for training. If no adapter name is passed, a default name is assigned to the adapter to follow the convention of the PEFT library. If you are not familiar with adapters and PEFT methods, we invite you to read more about them in the PEFT [documentation](https://huggingface.co/docs/peft). Args: adapter_config (`[~peft.PeftConfig]`): The configuration of the adapter to add; supported adapters are non-prefix tuning and adaption prompt methods. adapter_name (`str`, *optional*, defaults to `"default"`): The name of the adapter to add. If no name is passed, a default name is assigned to the adapter.  min_versionTPEFT is not available. Please install PEFT to use this function: `pip install peft`.r) PeftConfigrdTzAdapter with name z- already exists. Please use a different name.z8adapter_config should be an instance of PeftConfig. Got z instead.N)r r rrrr rdrr}rrtypebase_model_name_or_path set_adapter)radapter_configrqr rds r$ add_adapterzPeftAdapterMixin.add_adapters '78 "tu u<***.D ' T-- -1,?lmn n.*5J4P^K_J``ij  26.lC &r'c dtt|js tdt |t r|g}t |t |jz }t|dkDrAtddj|dt|jjddl m }d}|jD]^\}}t ||st|d r|j!|n,t|d st|d k7r td ||_d }`|s td y)a Sets a specific adapter by forcing the model to only use that adapter and disables the other adapters. If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT [documentation](https://huggingface.co/docs/peft). Args: adapter_name (str | list[str])): The list of adapters to set or the adapter name in the case of a single adapter. r0No adapter loaded. Please load an adapter first.rz)Following adapter(s) could not be found: z, zV. Make sure you are passing the correct adapter name(s). current loaded adapters are: rfFr rzYou are trying to set multiple adapters and you have a PEFT version that does not support multi-adapter inference. Please upgrade to the latest version of PEFT. `pip install -U peft` or `pip install -U git+https://github.com/huggingface/peft.git`TzhDid not succeeded in setting the adapter. Please make sure you are using a model that supports adapters.N)r r rrrrrr}rjoinrrrrg named_modulesrr r)rrqmissingrg_adapters_has_been_set_rs r$r zPeftAdapterMixin.set_adapter5s5 '78**OP P lC ((>Ll#c$*:*:&;; w>=C\ L FLL (< ( .r'c\ts td|j|jy)Nz-PEFT backend is required for `unfuse_lora()`.)r rr"_unfuse_lora_applyrs r$ unfuse_lorazPeftAdapterMixin.unfuse_loras#LM M 4**+r'cJddlm}t||r|jyy)Nrrf)rrgrunmerge)rrrgs r$r0z#PeftAdapterMixin._unfuse_lora_applys; fn - NN  .r'cts tdddlm}ddlm}||t |dr|`t |drd|_||y)Nz-PEFT backend is required for `unload_lora()`.rr)recurse_remove_peft_layersr}r) r rrrutilsr6rr}r)rrr6s r$ unload_lorazPeftAdapterMixin.unload_lorasJLM MW6"4( 4 ' 41 2*.D '248r'c@ts tdt|dy)a1 Disables the active LoRA layers of the underlying model. Example: ```py from diffusers import AutoPipelineForText2Image import torch pipeline = AutoPipelineForText2Image.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 ).to("cuda") pipeline.load_lora_weights( "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic" ) pipeline.unet.disable_lora() ``` )PEFT backend is required for this method.FrNr rrr1s r$ disable_lorazPeftAdapterMixin.disable_loras& HI I4/r'c@ts tdt|dy)a/ Enables the active LoRA layers of the underlying model. Example: ```py from diffusers import AutoPipelineForText2Image import torch pipeline = AutoPipelineForText2Image.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 ).to("cuda") pipeline.load_lora_weights( "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic" ) pipeline.unet.enable_lora() ``` r:TrNr;r1s r$ enable_lorazPeftAdapterMixin.enable_loras& HI I4.r'cts tdt|tr|g}|D]7}t ||t |ds|j j|d9t|y)a Delete an adapter's LoRA layers from the underlying model. Args: adapter_names (`list[str, str]`): The names (single string or list of strings) of the adapter to delete. Example: ```py from diffusers import AutoPipelineForText2Image import torch pipeline = AutoPipelineForText2Image.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 ).to("cuda") pipeline.load_lora_weights( "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_names="cinematic" ) pipeline.unet.delete_adapters("cinematic") ``` r:r}N) r rrrrrr}rr)rrrqs r$delete_adaptersz PeftAdapterMixin.delete_adapterssd. HI I mS )*OM)L !$ 5t]+  $$\48 * 348r' target_rankcheck_compiled)rwarnignorect|dir?|dk(r td|dk(rtjdn|dk7rt d|d||d |_y ) aEnables the possibility to hotswap LoRA adapters. Calling this method is only required when hotswapping adapters and if the model is compiled or if the ranks of the loaded adapters differ. Args: target_rank (`int`, *optional*, defaults to `128`): The highest rank among all the adapters that will be loaded. check_compiled (`str`, *optional*, defaults to `"error"`): How to handle the case when the model is already compiled, which should generally be avoided. The options are: - "error" (default): raise an error - "warn": issue a warning - "ignore": do nothing r}rzr@intrrGr r'r$r\r\Ls #04 $+4HH\anTXnf &%*#'"& B;B;# B; ! B; 4Z B;NPT6HCy36HU +d4j84:EL6Hp$'$'D$'L/d3i/D/b3,4.-c-*P),,  90./.$9T#Y_$9N\ckk6=>W6Xk kr'r\)'r)rr functoolsrpathlibrtypingrrrrrr7r r r r r rrrrrrrutils.peft_utilsrrrrrunet_loader_utilsr get_loggerrrrr\r r'r$rZse  O    SM8   H %!5!0!?!@ ! "#E ! !"D ! B!A!%&H!!"D!@!!"D! !C!?!!"D! %&H!!"./Q#!$"D B$F#EA B#E<3!:G kG kr'