
    ti"                     (   d dl mZ d dlmZ d dlZd dlZ G d dej                        Z	 G d dej                        Z
 G d dej                        Z G d d	ej                        Z G d
 dej                        Z G d dej                        Zy)    Nc                   0     e Zd Zddedef fdZd Z xZS )RMSNormdimsepsc                 d    t         |           t        j                  |      | _        || _        y N)super__init__mxonesweightr   )selfr   r   	__class__s      </Volumes/fast/ai/experiments/MLX_z-image/mlx_text_encoder.pyr
   zRMSNorm.__init__   s$    ggdm    c                 l    t         j                  j                  || j                  | j                        S r   )r   fastrms_normr   r   r   xs     r   __call__zRMSNorm.__call__   s#    ww4;;99r   )ư>)__name__
__module____qualname__intfloatr
   r   __classcell__r   s   @r   r   r      s    S u 
:r   r   c                   &     e Zd Z fdZddZ xZS )	Attentionc                    t         |           |d   | _        |d   | _        |j	                  d| j                  | j                  z        | _        |d   | _        |j	                  dd      | _        t        j                  | j                  | j                  | j
                  z  d      | _
        t        j                  | j                  | j                  | j
                  z  d      | _        t        j                  | j                  | j                  | j
                  z  d      | _        t        j                  | j                  | j
                  z  | j                  d      | _        t        | j
                  |j	                  d	d
            | _        t        | j
                  |j	                  d	d
            | _        y )Nhidden_sizenum_attention_headshead_dimnum_key_value_heads
rope_thetag    .AFbiasrms_norm_epsr   r   )r	   r
   r#   	num_headsgetr%   r&   r'   nnLinearq_projk_projv_projo_projr   q_normk_normr   configr   s     r   r
   zAttention.__init__   sK   !-0 56

:t/?/?4>>/QR#)*?#@  **\9= ii 0 0$..4==2PW\]ii 0 0$2J2JT]]2Zafgii 0 0$2J2JT]]2Zafgii >@P@PW\] dmmND1QRdmmND1QRr   c                 H   |j                   \  }}| j                  |      }| j                  |      }| j                  |      }|j	                  || j
                  | j                        }|j	                  || j                  | j                        }|j	                  || j                  | j                        }| j                  |      }| j                  |      }fd} ||| j                        } ||| j                        }|j                  dddd      }|j                  dddd      }|j                  dddd      }| j
                  | j                  z  }	|	dkD  r0t        j                  ||	d      }t        j                  ||	d      }t        j                  | j                        }
||j                  dddd      z  |
z  }|||z   }t        j                   |d      }||z  j                  dddd      j	                  |d      }| j#                  |      S )Nc                 T   | j                   d   }|dz  }t        j                  t        j                  |       t        j                  d|t        j
                        z  |z        }t        j                  t        j
                        }|d d d f   |d d d f   z  }t        j                  |      j                  d|      }t        j                  |      j                  d|      }| dd |f   }	| d|d f   }
t        j                  |	|z  |
|z  z
  |	|z  |
|z  z   gd      S )N   r   )dtype   .axis)
shaper   explogarangefloat32cosreshapesinconcatenate)r   thetadimhalffreqsposargsrE   rG   x1x2Ls              r   
apply_ropez&Attention.__call__.<locals>.apply_rope6   s   ''"+C!8DFFBFF5M>BIIaRZZ,PPSWWXE))ARZZ0Cq$w<%a.0D&&,&&q!T2C&&,&&q!T2C3:B3:B>>28b3h#6S288K"LSUVVr   r   r;   r=      r>   r:   )r@   r0   r1   r2   rF   r,   r%   r&   r4   r5   r'   	transposer   repeatmathsqrtsoftmaxr3   )r   r   maskBDqkvrR   n_repscalescoresprobsoutputrQ   s                 @r   r   zAttention.__call__$   s   ''1a KKNKKNKKN IIaDNNDMM:IIaD44dmmDIIaD44dmmD KKNKKN	W q$//*q$//* KK1a#KK1a#KK1a#$":"::19		!U+A		!U+A 		$--(akk!Q1--6d]F

6+!)&&q!Q2::1aD{{6""r   r   r   r   r   r
   r   r   r   s   @r   r!   r!      s    S$7#r   r!   c                   $     e Zd Z fdZd Z xZS )MLPc                     t         |           t        j                  |d   |d   d      | _        t        j                  |d   |d   d      | _        t        j                  |d   |d   d      | _        y )Nr#   intermediate_sizeFr(   )r	   r
   r.   r/   	gate_projup_proj	down_projr6   s     r   r
   zMLP.__init___   sn    6-#8&AT:U\abyy!6?R8SZ_`6*=#>}@U\abr   c                     | j                  t        j                  | j                  |            | j	                  |      z        S r   )rk   r.   siluri   rj   r   s     r   r   zMLP.__call__e   s0    ~~bggdnnQ&784<<?JKKr   rd   r   s   @r   rf   rf   ^   s    cLr   rf   c                   &     e Zd Z fdZddZ xZS )TransformerBlockc                     t         |           t        |      | _        t	        |      | _        t        |d   |d         | _        t        |d   |d         | _        y )Nr#   r*   r+   )	r	   r
   r!   	self_attnrf   mlpr   input_layernormpost_attention_layernormr6   s     r   r
   zTransformerBlock.__init__j   sX    "6*v;&vm'<&BXY(/}0E6R`Ka(b%r   c                     || j                  | j                  |      |      z   }|| j                  | j                  |            z   }|S r   )rq   rs   rr   rt   )r   r   rY   houts        r   r   zTransformerBlock.__call__q   sE    t33A6==$((488;<<
r   r   rd   r   s   @r   ro   ro   i   s    cr   ro   c                   $     e Zd Z fdZd Z xZS )
Qwen3Modelc                     t         |           t        j                  |d   |d         | _        t        |d         D cg c]  }t        |       c}| _        t        |d   |d         | _	        y c c}w )N
vocab_sizer#   num_hidden_layersr*   r+   )
r	   r
   r.   	Embeddingembed_tokensrangero   layersr   norm)r   r7   _r   s      r   r
   zQwen3Model.__init__x   sr    LL)=vm?TU9>vFY?Z9[\9[A'/9[\F=1vn7MN	 ]s   A4c                 D   | j                  |      }|j                  \  }}t        j                  t        j                  ||fd      d      }g }| j
                  D ]  } |||      }|j                  |        | j                  |      }|j                  |       |d   S )Ng    er=   )r]   )r~   r@   r   triufullr   appendr   )	r   	input_idsr   rZ   rQ   rY   hidden_stateslayer	final_outs	            r   r   zQwen3Model.__call__~   s    i(1 wwrww1vt,2[[EaA  # ! IIaL	Y'R  r   rd   r   s   @r   ry   ry   w   s    O!r   ry   c                   $     e Zd Z fdZd Z xZS )TextEncoderMLXc                 B    t         |           t        |      | _        y r   )r	   r
   ry   modelr6   s     r   r
   zTextEncoderMLX.__init__   s    '
r   c                 $    | j                  |      S r   )r   )r   r   s     r   r   zTextEncoderMLX.__call__   s    zz)$$r   rd   r   s   @r   r   r      s    (%r   r   )mlx.corecorer   mlx.nnr.   numpynprV   Moduler   r!   rf   ro   ry   r    r   r   <module>r      sv       :bii :J#		 J#ZL")) Lryy ! !2%RYY %r   