(tiIIddlZddlmcmZddlmZddlmZmZddlm Z e rddl m Z m Z ejeZdej fdZdej fd Zdd efd ZGd d ejj ZGddej ZGddej ZGddej0ZGddej4Zy)N)nn) deprecatelogging)is_transformers_available) CLIPTextModelCLIPTextModelWithProjection text_encoderc&g}t|ttfrXt|jj j D]*\}}d|d}|j}|j||f,|Std|jj)Ntext_model.encoder.layers.z .self_attnz.do not know how to get attention modules for: ) isinstancerr enumerate text_modelencoderlayers self_attnappend ValueError __class____name__)r attn_modulesilayernamemods d/Volumes/fast/ai/experiments/MLX_z-image/.venv/lib/python3.12/site-packages/diffusers/models/lora.pytext_encoder_attn_modulesr'sL,0K LM!,"9"9"A"A"H"HIHAu/s*=D//C   s ,J I,J`J`JiJiIjkllc&g}t|ttfrXt|jj j D]*\}}|j}d|d}|j||f,|Std|jj)Nr z.mlpz(do not know how to get mlp modules for: ) r rr rrrrmlprrrr)r mlp_modulesrrmlp_modrs rtext_encoder_mlp_modulesr#5sK,0K LM!,"9"9"A"A"H"HIHAuiiG/s$7D   g /J CLDZDZDcDcCdeffr lora_scalect|D]d\}}t|jts!||j_||j _||j _||j_ft|D]B\}}t|jts!||j_||j_DyN) rr q_projPatchedLoraProjectionr$k_projv_projout_projr#fc1fc2)r r$_ attn_module mlp_modules radjust_lora_scale_text_encoderr1Cs3LA; k((*? @,6K   ),6K   ),6K   ).8K + B2,? : jnn&; <(2JNN %(2JNN %@rcHeZdZd fd Zddddfd Zd dZdZd ZxZS) r(Ncbd}tdd|t | ddlm}||_|j j j}| |j j j}||j j|j j|||||_ ||_ y)NzsUse of `PatchedLoraProjection` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.r(1.0.0r)LoRALinearLayer) network_alphadevicedtyperank) rsuper__init__ models.lorar5regular_linear_layerweightr7r8 in_features out_featureslora_linear_layerr$) selfr=r$r6r9r8deprecation_messager5r7rs rr;zPatchedLoraProjection.__init__RsT)74GH 1$8!**1188 =--44::E!0  % % 1 1  % % 2 2' " %rF destinationprefix keep_varscz|j|jj||||dSt|||||dS)NrE)rAr= state_dictr:)rBrFrGrHargsrs rrJz PatchedLoraProjection.state_dictmsL  ! ! )74,,77;v w!4[[deerc 4|jy|jjjj|jjjj }}|jjjj }|jjjjj }|jjjjj }|jj0||jjz|jjz }||tj|dddf|dddfdzz}|r@tj|jjrt!d|d|j#|||jj_d|_|j%|_|j%|_||_yNraThis LoRA weight seems to be broken. Encountered NaN values when trying to fuse LoRA weights for .LoRA weights will not be fused.r7r8)rAr=r>datar8r7floatupdownr6r9torchbmmisnananyitemrtocpuw_upw_downr$ rBr$ safe_fusingr8r7w_origr\r] fused_weights r _fuse_loraz PatchedLoraProjection._fuse_loraus  ! ! ) 1188==CCTE^E^EeEeEjEjEqEqv**1166<<>%%((//44::<'',,3388>>@  ! ! / / ;$00>>>AWAWA\A\\DeiiT1W vdTUg.WXY.Z!Z[ 5;;|488:??AOOSfU22  1=vUZ0[!!((-"&HHJ jjl $rc Nt|dd t|ddy|jjj}|j|j }}|j j|j}|jj|j}|j|jtj|dddf|dddfdzz }|j|||jj_d|_d|_ yNr\r]r7rrP) getattrr=r>rQr8r7r\rZrRr]r$rUrVrBrar8r7r\r]unfused_weights r _unfuse_loraz"PatchedLoraProjection._unfuse_lorasfd+7GD(TX0A0AW\0A0]!!((-  rc|jd|_|j|j|S|j||j|j|zzS)N?)r$rAr=)rBinputs rforwardzPatchedLoraProjection.forwards[ ?? "!DO  ! ! ),,U3 3((/4??TE[E[\aEb3bccr)NNrkF) r __module__ __qualname__r;rJrbrirm __classcell__rs@rr(r(Qs(%6-1uf%@ drr(ceZdZdZ d dededededzdejezdzdejdzf fd Z d ejd ejfd Z xZ S)r5a A linear layer that is used with LoRA. Parameters: in_features (`int`): Number of input features. out_features (`int`): Number of output features. rank (`int`, `optional`, defaults to 4): The rank of the LoRA layer. network_alpha (`float`, `optional`, defaults to `None`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning device (`torch.device`, `optional`, defaults to `None`): The device to use for the layer's weights. dtype (`torch.dtype`, `optional`, defaults to `None`): The dtype to use for the layer's weights. Nr?r@r9r6r7r8ct|d}tdd|tj||d|||_tj||d|||_||_||_||_ ||_ tjj|j jd|z tjj|j jy)NzmUse of `LoRALinearLayer` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.r5r4F)biasr7r8rnstd)r:r;rrLinearrTrSr6r9r@r?initnormal_r>zeros_) rBr?r@r9r6r7r8rCrs rr;zLoRALinearLayer.__init__s N#W.ABIIk4eFRWX ))D,U6QVW+ (&  ((a$h7 tww~~&r hidden_statesreturnc.|j}|jjj}|j|j|}|j |}|j ||j |j z z}|j|Sr&r8rTr>rZrSr6r9rBr~ orig_dtyper8down_hidden_statesup_hidden_statess rrmzLoRALinearLayer.forward"((    &&!YY}'7'7'>?77#56    )  2 2TYY > > "":..r)roNNN)rrqrr__doc__intrRrUr7strr8r;Tensorrmrsrts@rr5r5s0&*,0$(''' ' t| '  s"T) '{{T!'4 /U\\ /ell /rr5ceZdZdZ ddedededeeeefzdeeeefzdeeeefzezd edzffd Zd e jd e jfd Z xZ S)LoRAConv2dLayera" A convolutional layer that is used with LoRA. Parameters: in_features (`int`): Number of input features. out_features (`int`): Number of output features. rank (`int`, `optional`, defaults to 4): The rank of the LoRA layer. kernel_size (`int` or `tuple` of two `int`, `optional`, defaults to 1): The kernel size of the convolution. stride (`int` or `tuple` of two `int`, `optional`, defaults to 1): The stride of the convolution. padding (`int` or `tuple` of two `int` or `str`, `optional`, defaults to 0): The padding of the convolution. network_alpha (`float`, `optional`, defaults to `None`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning Nr?r@r9 kernel_sizestridepaddingr6ct |d}tdd|tj|||||d|_tj||ddd|_||_||_tjj|j jd|z tjj|j jy) NzmUse of `LoRAConv2dLayer` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.rr4F)rrrrwrnrn)rrrwrnrx) r:r;rrConv2drTrSr6r9r{r|r>r}) rBr?r@r9rrrr6rCrs rr;zLoRAConv2dLayer.__init__s N#W.ABIIk4[QWahotu ))D,F6X]^+   ((a$h7 tww~~&rr~rc.|j}|jjj}|j|j|}|j |}|j ||j |j z z}|j|Sr&rrs rrmzLoRAConv2dLayer.forwardrr)rorrrN) rrqrrrrtuplerrRr;rUrrmrsrts@rrrs4-3(./0&*''' ' 5c?* ' eCHo% 'uS#X&,'t|'8 /U\\ /ell /rrceZdZdZdddedzffdZdedzfdZddedefd Z d Z dd e jd ed e jfdZ xZS)LoRACompatibleConvz; A convolutional layer that can be used with LoRA. N lora_layerrcPd}tdd|t||i|||_y)NzpUse of `LoRACompatibleConv` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.rr4rr:r;rrBrrKkwargsrCrs rr;zLoRACompatibleConv.__init__.s3Q&1DE $)&)$rc0d}tdd|||_yNznUse of `set_lora_layer()` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.set_lora_layerr4rrrBrrCs rrz!LoRACompatibleConv.set_lora_layer5s O"G-@A$rr$r_c<|jy|jjj|jjj}}|jjj }|jj jjj }|jjjjj }|jj0||jjz|jjz }tj|jd|jd}|j|j}|||zz} |r@tj| j!j#rt%d|d| j'|||j_d|_|j)|_|j)|_||_y)Nrn start_dimrNrOrP)rr>rQr8r7rRrSrTr6r9rUmmflattenreshapeshaperWrXrYrrZr[r\r] _lora_scale) rBr$r_r8r7r`r\r]fusionras rrbzLoRACompatibleConv._fuse_lora;s ?? "  ((.. 0@0@0G0Gv!!'')!!((--335%%,,11779 ?? ( ( 4$//777$//:N:NND$,,,3V^^a^5PQ/f!45 5;;|488:??AOOSfU22  (??&?F HHJ jjl %rct|dd t|ddy|jj}|jj|jj}}|j j |j|_|jj |j|_tj|j jd|jjd}|j|j}|j|j|zz }|j |||j_d|_d|_y)Nr\r]rernrrP)rfr>rQr8r7r\rZrRr]rUrrrrr)rBrar8r7rrhs rrizLoRACompatibleConv._unfuse_lora]s fd+7GD(TXrwrdilationgroupsr)rBr~rroriginal_outputss rrmzLoRACompatibleConv.forwardos    'EE-1V1V]a]n]noMGllG88 4;; 4;;X\XcXc  ?? "# ##ut}/M'MN Nrrprk)rrqrrrrr;rrRboolrbrirUrrmrsrts@rrr)spDH%/D*@%%4)?% &U &t &D$OU\\O%O%,,OrrceZdZdZdddedzffdZdedzfdZddedefd Z d Z dd e jd ed e jffd Z xZS)LoRACompatibleLinearz4 A Linear layer that can be used with LoRA. NrrcPd}tdd|t||i|||_y)NzrUse of `LoRACompatibleLinear` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.rr4rrs rr;zLoRACompatibleLinear.__init__s3S('3FG $)&)$rc0d}tdd|||_yrrrs rrz#LoRACompatibleLinear.set_lora_layers O"G-@A$rr$r_c |jy|jjj|jjj}}|jjj }|jj jjj }|jjjjj }|jj0||jjz|jjz }||tj|dddf|dddfdzz}|r@tj|jjrtd|d|j!|||j_d|_|j#|_|j#|_||_yrM)rr>rQr8r7rRrSrTr6r9rUrVrWrXrYrrZr[r\r]rr^s rrbzLoRACompatibleLinear._fuse_loras ?? "  ((.. 0@0@0G0Gv!!'')!!((--335%%,,11779 ?? ( ( 4$//777$//:N:NNDeiiT1W vdTUg.WXY.Z!Z[ 5;;|488:??AOOSfU22  (??&?F HHJ jjl %rc &t|dd t|ddy|jj}|j|j}}|j j |j}|jj |j}|j|jtj|dddf|dddfdzz }|j |||j_d|_d|_yrd) rfr>rQr8r7r\rZrRr]rrUrVrgs rriz!LoRACompatibleLinear._unfuse_lorasfd+7GD(TXrs0 &:G   H % BII  299  3U 3YdEHHOOYdx9/bii9/x=/bii=/@TOTOnG299Gr