
    ti%                     J   d dl mZ d dlmZ d dlZ G d dej                        Z G d dej                        Z G d dej                        Z	 G d d	ej                        Z
 G d
 dej                        Z G d dej                        Z G d dej                        Zy)    Nc                   0     e Zd Zddedef fdZd Z xZS )RMSNormdimsepsc                 d    t         |           t        j                  |      | _        || _        y N)super__init__mxonesweightr   )selfr   r   	__class__s      7/Volumes/fast/ai/experiments/MLX_z-image/mlx_z_image.pyr
   zRMSNorm.__init__   s$    ggdm    c                 l    t         j                  j                  || j                  | j                        S r   )r   fastrms_normr   r   r   xs     r   __call__zRMSNorm.__call__   s#    ww4;;99r   )ư>)__name__
__module____qualname__intfloatr
   r   __classcell__r   s   @r   r   r      s    S u 
:r   r   c                   &     e Zd Zd fd	Zd Z xZS )TimestepEmbedderc                     t         |           ||}t        j                  ||      | _        t        j                  ||      | _        || _        y r   )r	   r
   nnLinearlinear1linear2frequency_embedding_size)r   out_sizemid_sizer'   r   s       r   r
   zTimestepEmbedder.__init__   sE    Xyy!98Dyy84(@%r   c           	         |j                  t        j                        }| j                  dz  }t        j                  t        j                  d       t        j                  d|t        j                        z  |z        }|d d d f   |d d d f   z  }t        j                  t        j                  |      t        j                  |      gd      }| j                  dz  r5t        j                  |t        j                  |d d d df         gd      }| j                  t        j                  | j                  |                  S )N   i'  r   dtypeaxis   )astyper   float32r'   expmathlogarangeconcatenatecossin
zeros_liker&   r#   silur%   )r   thalffreqsargs	embeddings         r   r   zTimestepEmbedder.__call__   s    HHRZZ ,,1'"))At2::*NNQUUV!T'
U47^+NNBFF4L"&&,#?bI	((1,	2==1bqb5AQ3R'SZ[\I||BGGDLL$;<==r   )N   r   r   r   r
   r   r   r   s   @r   r!   r!      s    A>r   r!   c                   .     e Zd Zdedef fdZd Z xZS )FeedForwarddim
hidden_dimc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y )NFbias)r	   r
   r#   r$   w1w2w3)r   rF   rG   r   s      r   r
   zFeedForward.__init__$   sJ    ))C%8))J%8))C%8r   c                     | j                  t        j                  | j                  |            | j	                  |      z        S r   )rL   r#   r<   rK   rM   r   s     r   r   zFeedForward.__call__*   s/    wwrwwtwwqz*TWWQZ788r   )r   r   r   r   r
   r   r   r   s   @r   rE   rE   #   s    9C 9S 99r   rE   c            	       F     e Zd Zd	dedededef fdZd Zd Zd
dZ xZ	S )	AttentionrF   nheads
rope_thetar   c                    t         |           || _        ||z  | _        | j                  dz  | _        t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _	        d | _
        t        j                  ||d      | _        t        | j                  |      | _        t        | j                  |      | _        g d| _        g d| _        i | _        y )Ng      FrI   )r   )    0   rU   )r   rT   P   )r	   r
   rQ   head_dimscaler#   r$   to_qto_kto_vto_qkvto_outr   norm_qnorm_kr   splitsfreqs_cache)r   rF   rQ   rR   r   r   s        r   r
   zAttention.__init__/   s    v]]d*
IIc3U3	IIc3U3	IIc3U3	iiSu5dmm5dmm5 	!r   c                    | j                   y | j                  j                  | j                  j                  | j                  j                  }}}t        j                  |||gd      }| j                  j                  j                  d   | j                  j                  j                  d   dz  }}t        j                  ||d      | _         || j                   _        | `| `| `y )Nr   r/   r1      FrI   )
r\   rY   r   rZ   r[   r   r8   shaper#   r$   )r   w_qw_kw_vfused_weightin_dimout_dims          r   fuse_qkvzAttention.fuse_qkvB   s    ;;"F		(($))*:*:DII<L<L#S~~sCoA>))**003TYY5E5E5K5KA5NQR5Riie<)Ity$)r   c           	         |j                   \  }}}|| j                  v r| j                  |   }ng }| j                  D ]k  }|dz  }t        j                  t        j
                  d       t        j                  d|t        j                        z  |z        }	|j                  |	       m || j                  |<   |}|d   j                  t        j                        }
|
d   |d   d d d d d f   z  }|d   j                  t        j                        }|d   |d   d d d d d f   z  }|d	   j                  t        j                        }|d   |d   d d d d d f   z  }t        j                  |||gd
      S )Nr+         p@r   r,   ).r   ).NN).r1   r1   ).r+   r.   r/   )rd   ra   r   r   r4   r6   r7   r3   appendr2   r8   )r   	positionsBL_freqs_tuple
freqs_listdr>   fpos_hargs_hpos_wargs_wpos_targs_ts                   r   _get_fused_args_cachedz Attention._get_fused_args_cachedK   sj   //1a   **1-KJYYAvFFBFF5M>BIIaRZZ,PPSWWX!!!$  #-DQ$K&!((4'+a.tT19L*MM&!((4'+a.tT19L*MM&!((4'+a.tT19L*MM~~vvv6R@@r   c                    |j                   \  }}}| j                  | j                  |      j                  ||d| j                  | j                        }	t        j                  |	dd      \  }
}}|
j                  d      |j                  d      |j                  d      }}}
n| j                  |      j                  ||| j                  | j                        }
| j                  |      j                  ||| j                  | j                        }| j                  |      j                  ||| j                  | j                        }| j                  |
      }
| j                  |      }||
ddd df   |
ddd df   }}t        j                  ||z  ||z  z
  ||z  ||z  z   gd      j                  ||| j                  | j                        }
|ddd df   |ddd df   }}t        j                  ||z  ||z  z
  ||z  ||z  z   gd      j                  ||| j                  | j                        }|
j                  dddd      |j                  dddd      |j                  dddd      }}}
t
        j                  j!                  |
||| j"                  |      }| j%                  |j                  dddd      j                  |||            S )	Nrc   r+   r/   .r   r1   r.   )rX   mask)rd   r\   reshaperQ   rW   r   splitsqueezerY   rZ   r[   r^   r_   stack	transposer   scaled_dot_product_attentionrX   r]   )r   r   r   ro   r9   r:   rp   rq   Dqkvqkvq1q2k1k2outputs                     r   r   zAttention.__call__a   s   ''1a;;"++a.((Aq$++t}}MChhsAA.GAq!iilAIIaL!))A,!qA		!$$Q4;;FA		!$$Q4;;FA		!$$Q4;;FAKKNKKN sADqDy\1S!$Q$Y<B"s(R#X-rCx"s(/BC"MUUVWYZ\`\g\gimivivwAsADqDy\1S!$Q$Y<B"s(R#X-rCx"s(/BC"MUUVWYZ\`\g\gimivivwA++aAq)1;;q!Q+BAKKPQSTVWYZD[a155aATZZVZ5[{{6++Aq!Q7??1aHIIr   )rm   h㈵>)NNNN)
r   r   r   r   r   r
   rk   r}   r   r   r   s   @r   rP   rP   .   s5    C  % e &,A,Jr   rP   c                   (     e Zd Zd fd	ZddZ xZS )ZImageTransformerBlockc                    t         |           |d   }|d   }|| _        t        |||j	                  dd      d      | _        t        |t        |dz  dz              | _        t        |      | _
        t        |      | _        t        |      | _        t        |      | _        |r!t        j                  d	d
|z  d      | _        y y )NrF   rQ   rR   rm   r   )rR   r   rc      rB      TrI   )r	   r
   
modulationrP   get	attentionrE   r   feed_forwardr   attention_norm1	ffn_norm1attention_norm2	ffn_norm2r#   r$   adaLN_modulation)r   configlayer_idr   rF   rQ   r   s         r   r
   zZImageTransformerBlock.__init__{   s    Um!$"36::lTY;Z`de'Sq1-=>&s| &s| ryya#gD/Qt,:r   c           
         | j                   r| j                  |      }t        j                  |dd      \  }}	}
}|dd d d f   |	dd d d f   }	}|
dd d d f   |dd d d f   }}
| j	                  |      d|z   z  }| j                  |||||      }|t        j                  |	      | j                  |      z  z   }| j                  |      d|
z   z  }|t        j                  |      | j                  | j                  |            z  z   }|S || j                  | j                  | j	                  |      ||||            z   }|| j                  | j                  | j                  |                  z   }|S )Nr   r.   r/   .r1   r9   r:   )r   r   r   r   r   r   tanhr   r   r   r   )r   r   r   ro   adaln_inputr9   r:   chunks	scale_msagate_msa	scale_mlpgate_mlpnorm_xattn_outnorm_ffns                  r   r   zZImageTransformerBlock.__call__   st   ??**;7F79xxPR7S4IxH"+CqL"98CqL;QxI"+CqL"98CqL;QxI))!,I>F~~fdI3C~PHBGGH%(<(<X(FFFA~~a(A	M:HBGGH%t7H7H7R(SSSA  D((8L8LQ8OQUW`begj)kllADNN4#4#4T^^A5F#GHHAr   )T)NNNrC   r   s   @r   r   r   z   s    Rr   r   c                   $     e Zd Z fdZd Z xZS )
FinalLayerc                    t         |           t        j                  |dd      | _        t        j
                  ||d      | _        t        j                  t        j                         t        j
                  d|d            | _	        y )Nr   F)r   affineTrI   rB   )
r	   r
   r#   	LayerNorm
norm_finalr$   linear
SequentialSiLUr   )r   rF   out_channelsr   s      r   r
   zFinalLayer.__init__   s]    ,,sUCii\= "bggi3RV9W Xr   c           	           | j                   j                  d    | j                   j                  d   |            }| j                  | j                  |      d|d d d d d f   z   z        S )Nr1   r   )r   layersr   r   )r   r   crX   s       r   r   zFinalLayer.__call__   sf    /%%,,Q/0O0E0E0L0LQ0OPQ0RS{{4??1-U1dA:5F1FGHHr   rC   r   s   @r   r   r      s    YIr   r   c                   2     e Zd Z fdZd Zd ZddZ xZS )ZImageTransformerMLXc                    t         |           || _        |d   }|j                  dd      | _        t        dd      | _        t        j                  |d   dz  |d	
      | _	        t        j                  t        |d         t        j                  |d   |d	
            | _        t        ||d   dz        | _        t        j                   d|f      | _        t        j                   d|f      | _        t'        |d         D cg c]  }t)        ||d	       c}| _        t'        |d         D cg c]  }t)        ||d       c}| _        t'        |d         D cg c]  }t)        ||d	       c}| _        y c c}w c c}w c c}w )NrF   t_scaleg     @@rB   i   )r)   in_channelsr   TrI   cap_feat_dimr1   n_refiner_layersFn_layers)r	   r
   r   r   r   r!   
t_embedderr#   r$   
x_embedderr   r   cap_embedderr   final_layerr   zerosx_pad_tokencap_pad_tokenranger   noise_refinercontext_refinerr   )r   r   rF   ir   s       r   r
   zZImageTransformerMLX.__init__   sj   Umzz)V4*3>))F=$9A$=sNMM'&2H*I*,))F>4JCVZ*[]%c6-+@1+DE88QH-XXq#h/OTU[\nUoOpqOp!4VQEOpqRWX^_qXrRstRsQ 6vq% HRstHMfU_N`HabHa1-fa>Hab rtbs   E<3FFc                     | j                   d   j                  }|j                  |      }t        j                  |      t        j
                  |      fS )Nr   )r   r   r}   r   r9   r:   )r   ro   
dummy_attnr@   s       r   prepare_ropez!ZImageTransformerMLX.prepare_rope   s@    [[^--
00;vvd|RVVD\))r   c                    | j                   D ]  }|j                  j                           | j                  D ]  }|j                  j                           | j                  D ]  }|j                  j                           y r   )r   r   rk   r   r   )r   layers     r   
fuse_modelzZImageTransformerMLX.fuse_model   sV    ''E)A)A)C'))E5??+C+C+E)[[E%//":":"<[r   c
                    | j                  || j                  z        }
| j                  |      }|$t        j                  |d   | j
                  |      } | j                  j                  d    | j                  j                  d   |            }|	$t        j                  |	d   | j                  |      }| j                  D ]<  } ||d ||
|d d d |j                  d   f   |d d d |j                  d   f         }> | j                  D ]<  } ||d |d |d d |j                  d   d f   |d d |j                  d   d f         }> t        j                  ||gd      }t        j                  ||gd      }d }| j                  D ]  } |||||
||      } | j                  |d d d |j                  d   d d f   |
      S )N).Nr1   r   r   r/   )r   r   r   r   wherer   r   r   r   r   rd   r   r8   r   )r   r   r=   	cap_featsx_poscap_posr9   r:   x_maskcap_masktemblunifiedunified_posunified_masks                  r   r   zZImageTransformerMLX.__call__   s   q4<</0OOA288F9,=t?O?OQR#Sq/D%%,,Q/0K0A0A0H0H0KI0VW	RXXhy6I4K]K]_h-i##A!T5$C;AGGAJ;,?SKQWWUVZKEXYA $ %%A)T7Dc!QWWQZ[.>QWZ[\^_^e^efg^h^i[iWjkI & ..!Ya8nneW%5A>A{DcsSG  ;AGGAJ;(9 :DAAr   )NN)r   r   r   r
   r   r   r   r   r   s   @r   r   r      s    c$*=
Br   r   )mlx.corecorer   mlx.nnr#   r5   Moduler   r!   rE   rP   r   r   r    r   r   <module>r      s      :bii :>ryy >&9")) 9IJ		 IJXRYY B	I 	I4B299 4Br   