
    ti@                         d dl mZ d dlmZ d dlZd dlZd dlZd dlm	Z	  G d dej                        Zd Zd Zd Zd	dZy)
    N)	safe_openc                   &     e Zd Zd fd	Zd Z xZS )LoRALinearWrapperc                 Z    t         |           || _        || _        || _        || _        y )N)super__init__
base_layerlora_alora_bscale)selfr	   r
   r   r   	__class__s        6/Volumes/fast/ai/experiments/MLX_z-image/lora_utils.pyr   zLoRALinearWrapper.__init__
   s*    $
    c                    | j                  |      }|j                  }|j                  | j                  j                        }| j                  }| j                  }|j
                  d   |j
                  d   k7  r|j                  }|j
                  d   |j
                  d   k7  r|j                  }||z  |z  | j                  z  }||j                  |      z   S )Nr   )r	   dtypeastyper
   r   shapeTr   )r   xbase_outr   ablora_outs          r   __call__zLoRALinearWrapper.__call__   s    ??1%
 HHT[[&&' KKKK 771:$A 771:$AEAI+(//%000r         ?)__name__
__module____qualname__r   r   __classcell__)r   s   @r   r   r   	   s    1r   r   c                 ^   |j                  d      }| }|D ]  }	 |j                         r[t        |      }t        |t              r||   }nTt        |t
              r||v r||   n||   }n5t        ||      rt        ||      nd }nt        ||      rt        ||      nd }| y  |S #  Y  y xY w)N.)splitisdigitint
isinstancelistdicthasattrgetattr)modelmodule_namepartsobjpartidxs         r   get_module_by_namer3   *   s    c"E
C	||~$ic4(c(CT*&)Sj#c(c$iC07T0B'#t,C,3C,>gc4(D{4{   J	s   BB''B,c                    |j                  d      }| }|d d D ]k  }|j                         rMt        |      }t        |t              r||   }4t        |t
              r||v r||   n||   }St        ||      }`t        ||      }m |d   }|j                         r8t        |      }t        |t              r|||<   y t        |t
              r|||<   y y t        |||       y )Nr$   r   )r%   r&   r'   r(   r)   r*   r,   setattr)r-   r.   
new_moduler/   parentr1   r2   lasts           r   set_module_by_namer9   @   s    c"EFcr
<<>d)C&$'FD)(+v6$< .VT*F  9D||~$ifd#$F3K%$F3K & 	j)r   c                    | j                  dd      }|j                  dd      }t        j                  dd|      }|j                  dd      }|j                  dd	      }|j                  d
d      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }d|v r+d|v r	 |S |j                  dd      }|j                  dd      }|S )N
lora_unet_ zdiffusion_model.zlayers_(\d+)_z
layers.\1.attention_to_qzattention.to_qattention_to_kzattention.to_kattention_to_vzattention.to_vattention_to_out_0zattention.to_outattention_to_outfeed_forward_w1zfeed_forward.w1feed_forward_w2zfeed_forward.w2feed_forward_w3zfeed_forward.w3adaLN_modulationzadaLN_modulation.1adaLN_modulation_0adaLN_modulation_1)replaceresub)keynew_keys     r   convert_unet_key_to_mlxrM   Z   s   kk,+Goo0"5G ff%}g>G oo.0@AGoo.0@AGoo.0@AG oo24FGGoo02DEG oo/1BCGoo/1BCGoo/1BCG
 W$7* N oo&:<NOGoo&:<NOGNr   c                    t         j                  j                  |      st        d|        | S t        d| d| d       i }	 t	        |dd      5 }|j                         D ]b  }t        j                  |j                  |      j                         j                               j                  t        j                        ||<   d 	 d d d        i }|j                         D ]  }d	|v r'|j                  d
d      }	|	|vri ||	<   ||   ||	   d	<   .d|vr3d }
d }	d|v r|j                  d      d   }	d}
nPd|v r|j                  d      d   }	d}
n5d|v r|j                  d      d   }	d}
nd|v r|j                  d      d   }	d}
|	s|
s|	|vri ||	<   ||   ||	   |
<    d}t        d       |j!                         D ]  \  }}d|vsd|vrt#        |      }t%        | |      }|r|d   }|d   }t'        |j(                        }d	|v r|d	   j+                         }||z  }nd}||z  }t-        |t.              r|j0                  }	n|}	t/        |	|||      }t3        | ||       |dz  } |dk(  rt        d       | S t        d| d       | S # 1 sw Y   xY w# t        $ r}t        d|        | cY d }~S d }~ww xY w)NzLoRA file not found: z   [LoRA] Loading weights from z (User Scale: )ptcpu)	frameworkdevicezFailed to load LoRA: alphaz.alphar<   lora	lora_downz
.lora_downr   Alora_upz.lora_upBlora_Az.lora_Alora_Bz.lora_Bz1   [LoRA] Applying adapters with Alpha scaling...r      z:    Failed to apply any layers. Naming mismatch suspected.z   [LoRA] Applied z% layers. (Logic: Auto-Alpha & Rename))ospathexistsprintr   keysmxarray
get_tensorfloatnumpyr   bfloat16	ExceptionrH   r%   itemsrM   r3   minr   itemr(   r   r	   r9   )r-   	lora_pathr   tensorsfkelora_groupsrK   basetype_applied_countlora_keygroup	final_keytargetr
   r   rankrT   scale_factorfinal_scalewrappeds                          r   
apply_lorar}      s   77>>)$%i[12	+I;nUG1
MNGyD?1VVXXXall1o&;&;&=&C&C&EFMMbkkZ
  @ K||~c>;;x,D;&BD(9)0Kg&h #99\*1-DE#99Z(+DE_99Y'*DE_99Y'*DEE;&BD(9'.s|Ke$; > M	=?&,,.%es%/ ,H5	 $E95 3ZF3ZF v||$D%g++-$t|"  ,.K &"34(( (ffkJGui9QM
 U /X JK L 	"=/1VWXL{ @?  %aS)*s7   J A6J
J JJ 	J?%J:4J?:J?r   )mlx.corecorerb   mlx.nnnnr]   rI   mathsafetensorsr   Moduler   r3   r9   rM   r}    r   r   <module>r      s>      	 	  !1		 1B,*4$Nfr   