Ë
    îÿæirS  ã                   óª  — d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	 d dl
mZ d dlmZ d dlmZmZ ddlmZmZmZ ddlmZmZ e G d	„ d
e«      «       Ze G d„ de«      «       Z G d„ dej6                  «      Z G d„ dej6                  «      Z G d„ dej6                  «      Z eej>                  d¬«      d„ «       Z  G d„ dej6                  «      Z! G d„ dej6                  «      Z" G d„ dej6                  «      Z# eej>                  d¬«      d„ «       Z$ G d„ dej6                  «      Z% G d„ d ej6                  «      Z& G d!„ d"ej6                  «      Z'y)#é    N)Ú	dataclass)Úpartial)ÚAnyÚDictÚListÚOptional)Útree_flattenÚtree_unflattené   )ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCachec                   ó  — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   ee   ed<   ee   ed<   eed<   eed<   eed<   eed<   eed<   eed<   dZ	e
e   ed<   y)Ú
TextConfigÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚhead_dimÚrms_norm_epsÚ
vocab_sizeÚnum_key_value_headsÚnum_kv_shared_layersÚvocab_size_per_layer_inputÚsliding_windowÚmax_position_embeddingsÚrope_local_base_freqÚ
rope_thetaÚfinal_logit_softcappingÚlayer_typesÚactivation_sparsity_patternÚhidden_size_per_layer_inputÚaltup_num_inputsÚaltup_coef_clipÚaltup_correct_scaleÚaltup_active_idxÚlaurel_rankNÚrope_scaling)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr   Úboolr+   r   r   © ó    új/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/gemma3n.pyr   r      s¥   … àƒOØÓØÓØÓØÓØƒMØÓØƒOØÓØÓØ #Ó#ØÓØ Ó ØÓØÓØ"Ó"Ø�c‘ÓØ!% e¡Ó,Ø!$Ó$ØÓØÓØÓØÓØÓØ#'€L�(˜4‘.Ô'r5   r   c                   ó"   — e Zd ZU eed<   eed<   y)Ú	ModelArgsr   Útext_configN)r,   r-   r.   r/   r0   Údictr4   r5   r6   r8   r8   -   s   … àƒOØÔr5   r8   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )Ú
RMSNoScaleÚepsc                 ó0   •— t         ‰| �  «        || _        y ©N)ÚsuperÚ__init__r=   )Úselfr=   Ú	__class__s     €r6   rA   zRMSNoScale.__init__4   s   ø€ Ü‰ÑÔØˆ�r5   c                 óX   — t         j                  j                  |d | j                  «      S r?   )ÚmxÚfastÚrms_normr=   )rB   Úxs     r6   Ú__call__zRMSNoScale.__call__8   s   € Ü�w‰w×Ñ  4¨¯©Ó2Ð2r5   )gñhãˆµøä>)r,   r-   r.   r2   rA   rI   Ú__classcell__©rC   s   @r6   r<   r<   3   s   ø„ ñ˜Eõ ö3r5   r<   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚGemma3nLaurelBlockz Learned Augmented Residual LayerÚconfigc                 óÌ  •— t         ‰| �  «        || _        t        j                  | j                  j
                  | j                  j                  d¬«      | _        t        j                  | j                  j                  | j                  j
                  d¬«      | _        t        j                  | j                  j
                  | j                  j                  ¬«      | _        y )NF©Úbias©Údimsr=   )r@   rA   rN   ÚnnÚLinearr   r*   Úlinear_leftÚlinear_rightÚRMSNormr   Úpost_laurel_norm©rB   rN   rC   s     €r6   rA   zGemma3nLaurelBlock.__init__?   sœ   ø€ Ü‰ÑÔØˆŒäŸ9™9Ø�K‰K×#Ñ# T§[¡[×%<Ñ%<À5ô
ˆÔô ŸI™IØ�K‰K×#Ñ# T§[¡[×%<Ñ%<À5ô
ˆÔô !#§
¡
Ø—‘×(Ñ(Ø—‘×(Ñ(ô!
ˆÕr5   rH   Úreturnc                 ór   — | j                  |«      }| j                  |«      }| j                  |«      }||z   S r?   )rV   rW   rY   )rB   rH   Úlaurel_xÚnormed_laurel_xs       r6   rI   zGemma3nLaurelBlock.__call__N   s=   € Ø×#Ñ# AÓ&ˆØ×$Ñ$ XÓ.ˆØ×/Ñ/°Ó9ˆØ�?Ñ"Ð"r5   )
r,   r-   r.   Ú__doc__r   rA   rE   ÚarrayrI   rJ   rK   s   @r6   rM   rM   <   s-   ø„ Ù*ð
˜zõ 
ð#˜"Ÿ(™(ð # r§x¡x÷ #r5   rM   c            	       ó’   ‡ — e Zd Zdededefˆ fd„Z	 	 d
dej                  de	ej                     de	e
   dej                  fd	„Zˆ xZS )ÚGemma3nAttentionrN   Ú	layer_idxÚis_kv_shared_layerc                 óŒ  •— t         ‰| �  «        |j                  |   dk(  | _        |j                  }|j
                  x| _        }|j                  x| _        }||z  | _	        |j                  x| _
        }|| _        d| _        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  ||z  |d¬«      | _        t        j&                  |j                  |j(                  ¬«      | _        t        j&                  |j                  |j(                  ¬«      | _        t/        |j(                  ¬«      | _        || _        t        j4                  |d| j                  r|j6                  n|j8                  ¬«      | _        y )NÚsliding_attentionç      ð?FrP   rR   ©r=   )ÚtraditionalÚbase)r@   rA   r#   Ú
is_slidingr   r   Ún_headsr   Ú
n_kv_headsÚrepeatsr   rc   ÚscalerT   rU   Úq_projÚk_projÚv_projÚo_projrX   r   Úq_normÚk_normr<   Úv_normrd   ÚRoPEr    r!   Úrope)	rB   rN   rc   rd   Údimrl   rm   r   rC   s	           €r6   rA   zGemma3nAttention.__init__V   sb  ø€ Ü‰ÑÔØ ×,Ñ,¨YÑ7Ð;NÑNˆŒà× Ñ ˆØ!'×!;Ñ!;Ð;ˆŒ�wØ'-×'AÑ'AÐAˆŒ˜*Ø *Ñ,ˆŒØ#)§?¡?Ð2ˆŒ˜Ø"ˆŒàˆŒ
ä—i‘i  W¨xÑ%7¸eÔDˆŒÜ—i‘i  Z°(Ñ%:ÀÔGˆŒÜ—i‘i  Z°(Ñ%:ÀÔGˆŒÜ—i‘i ¨(Ñ 2°C¸eÔDˆŒä—j‘j f§o¡o¸6×;NÑ;NÔOˆŒÜ—j‘j f§o¡o¸6×;NÑ;NÔOˆŒÜ  V×%8Ñ%8Ô9ˆŒà"4ˆÔä—G‘GØØà/3¯ª�×+Ò+ÀF×DUÑDUô	
ˆ�	r5   rH   ÚmaskÚcacher[   c                 ó¢  — |j                   \  }}}| j                  |«      }|j                  ||d| j                  «      }| j	                  |«      }d}| j
                  r|�|j                  \  }	}
|j                  }nÜ|�|j                  }| j                  |«      j                  ||d| j                  «      }	| j                  |	«      }	|	j                  dddd«      }	| j                  |	|¬«      }	| j                  |«      j                  ||d| j                  «      }
| j                  |
«      }
|
j                  dddd«      }
|�|j                  |	|
«      \  }	}
|j                  dddd«      }| j                  ||¬«      }t        ||	|
|| j                   |¬«      }|j                  dddd«      j                  ||d«      }| j#                  |«      S )Néÿÿÿÿr   é   r   é   )Úoffset)r{   ro   rz   )Úshaperp   Úreshaper   rt   rd   Ústater€   rq   ru   Ú	transposerx   rr   rv   Úupdate_and_fetchr   ro   rs   )rB   rH   rz   r{   ÚBÚLÚ_Úqueriesr€   ÚkeysÚvaluesÚoutputs               r6   rI   zGemma3nAttention.__call__v   s±  € ð —'‘'‰ˆˆ1ˆaà—+‘+˜a“.ˆØ—/‘/ ! Q¨¨D¯M©MÓ:ˆØ—+‘+˜gÓ&ˆàˆØ×"Ò" uÐ'8à Ÿ;™;‰LˆD�&Ø—\‘\‰Fð Ð ØŸ™�Ø—;‘;˜q“>×)Ñ)¨!¨Q°°D·M±MÓBˆDØ—;‘;˜tÓ$ˆDØ—>‘> ! Q¨¨1Ó-ˆDØ—9‘9˜T¨&�9Ó1ˆDà—[‘[ “^×+Ñ+¨A¨q°"°d·m±mÓDˆFØ—[‘[ Ó(ˆFØ×%Ñ% a¨¨A¨qÓ1ˆFàÐ Ø$×5Ñ5°d¸FÓC‘��fà×#Ñ# A q¨!¨QÓ/ˆØ—)‘)˜G¨F�)Ó3ˆä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆà�{‰{˜6Ó"Ð"r5   ©NN©r,   r-   r.   r   r1   r3   rA   rE   r`   r   r   rI   rJ   rK   s   @r6   rb   rb   U   sf   ø„ ð
˜zð 
°cð 
Ètõ 
ðF $(Ø#ñ	*#à�8‰8ð*#ð �r—x‘xÑ ð*#ð ˜‰}ð	*#ð
 
�‰÷*#r5   rb   T)Ú	shapelessc                 óü   — t        j                  | dd¬«      }t        j                  | dd¬«      }|||j                  |j                  «      z  z   }t        j                  t        j                  d| |z
  «      «      S )Nr}   T©ÚaxisÚkeepdimsr   )rE   ÚmeanÚstdÚastypeÚdtyperT   Úgelu_approxÚmaximum)ÚinputsÚstd_multiplierÚinputs_meanÚ
inputs_stdÚcutoff_xs        r6   Ú	gelu_topkrŸ   £   se   € ä—'‘'˜& r°DÔ9€KÜ—‘˜ R°$Ô7€JØ˜Z¨.×*?Ñ*?À
×@PÑ@PÓ*QÑQÑQ€HÜ�>‰>œ"Ÿ*™* Q¨°Ñ(9Ó:Ó;Ð;r5   c                   óJ   ‡ — e Zd Zddedefˆ fd„Zdej                  fd„Zˆ xZ	S )ÚMLPrN   rc   c                 óÜ  •— t         ‰| �  «        || _        |j                  | _        t	        |j
                  t        «      r|j
                  |   n|j
                  | _        t        j                  | j                  | j
                  d¬«      | _	        t        j                  | j                  | j
                  d¬«      | _
        t        j                  | j
                  | j                  d¬«      | _        |j                  �|j                  |   | _        nd| _        | j                  dkD  rAt        j                  d«      t!        j"                  d| j                  z  dz
  «      z  | _        y y )NFrP   ç        r   g       @r~   r   )r@   rA   rN   r   Ú
isinstancer   ÚlistrT   rU   Ú	gate_projÚup_projÚ	down_projr$   Úactivation_sparsityÚmathÚsqrtrE   ÚerfinvÚ_std_multiplier)rB   rN   rc   rC   s      €r6   rA   zMLP.__init__¬   s(  ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔô ˜&×2Ñ2´DÔ9ð ×$Ñ$ YÒ/à×)Ñ)ð 	Ôô
 Ÿ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒØ×-Ñ-Ð9Ø'-×'IÑ'IÈ)Ñ'TˆDÕ$à'*ˆDÔ$Ø×#Ñ# aÒ'Ü#'§9¡9¨S£>´B·I±IØ�D×,Ñ,Ñ,¨qÑ0ó5ñ $ˆDÕ ð (r5   rH   c                 óè   — | j                  |«      }| j                  dkD  rt        || j                  «      }nt	        j
                  |«      }| j                  |«      }| j                  ||z  «      }|S )Nr£   )r¦   r©   rŸ   r­   rT   r˜   r§   r¨   )rB   rH   r¦   Úactivationsr§   r¨   s         r6   rI   zMLP.__call__Á   sd   € Ø—N‘N 1Ó%ˆ	Ø×#Ñ# cÒ)Ü# I¨t×/CÑ/CÓD‰KäŸ.™.¨Ó3ˆKØ—,‘,˜q“/ˆØ—N‘N ;°Ñ#8Ó9ˆ	ØÐr5   )r   )
r,   r-   r.   r   r1   rA   rE   r`   rI   rJ   rK   s   @r6   r¡   r¡   «   s&   ø„ ñ˜zð °cõ ð*˜"Ÿ(™(÷ r5   r¡   c                   óÐ   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zdej                  dej                  fd„Z	dej                  d	ej                  fd
„Z
ˆ xZS )ÚGemma3nAltUpzAlternating Updates (AltUp)rN   c                 óº  •— t         ‰| �  «        || _        t        j                  | j                  j
                  f«      | _        t        j                  | j                  j                  | j                  j                  d¬«      | _
        t        j                  | j                  j                  | j                  j                  dz  d¬«      | _        t        j                  | j                  j
                  | j                  j                  d¬«      | _        t        j                  | j                  j
                  | j                  j                  ¬«      | _        y )NFrP   r~   rR   )r@   rA   rN   rE   Úzerosr   Úcorrect_output_scalerT   rU   r&   Úcorrection_coefsÚprediction_coefsÚmodality_routerrX   r   Úrouter_normrZ   s     €r6   rA   zGemma3nAltUp.__init__Ï   sî   ø€ Ü‰ÑÔØˆŒä$&§H¡H¨d¯k©k×.EÑ.EÐ-GÓ$HˆÔ!Ü "§	¡	Ø�K‰K×(Ñ(¨$¯+©+×*FÑ*FÈUô!
ˆÔô !#§	¡	Ø�K‰K×(Ñ(¨$¯+©+×*FÑ*FÈÑ*IÐPUô!
ˆÔô  "Ÿy™yØ�K‰K×#Ñ# T§[¡[×%AÑ%AÈô 
ˆÔô Ÿ:™:Ø—‘×(Ñ(Ø—‘×(Ñ(ô
ˆÕr5   rH   r[   c                 óÞ   — | j                  |«      | j                  j                  dz  z  }| j                  |«      j	                  t
        j                  «      }t        j                  |«      S )Ng      ð¿)r¸   rN   r   r·   r–   rE   Úfloat32Útanh)rB   rH   Úrouter_inputsÚrouteds       r6   Úcompute_router_modalitiesz&Gemma3nAltUp.compute_router_modalitiesâ   sS   € Ø×(Ñ(¨Ó+¨t¯{©{×/FÑ/FÈÑ/LÑMˆØ×%Ñ% mÓ4×;Ñ;¼B¿J¹JÓGˆÜ�w‰w�v‹Ðr5   c                 óš  — | j                  || j                  j                     «      }| j                  j                  j                  t        j                  «      | j                  _        | j                  j                  �ct        j                  | j                  j                  | j                  j                   | j                  j                  «      | j                  _         | j                  |«      j                  g |j                  d d ¢| j                  j                  ‘| j                  j                  ‘­Ž j                  dddd«      }|j                  t        j                  «      }|j                  dddd«      }t        j                  ||«      }|j                  dddd«      }||z  }|j                  |j                  «      S )Nr}   r   r   r   r~   )r¾   rN   r)   r¶   Úweightr–   rE   rº   r'   Úclipr‚   r�   r&   r„   Úmatmulr—   )rB   rH   Ú
modalitiesÚ	all_coefsÚx_upÚ
x_permutedÚpredictionss          r6   ÚpredictzGemma3nAltUp.predictç   sz  € Ø×3Ñ3°A°d·k±k×6RÑ6RÑ4SÓTˆ
à'+×'<Ñ'<×'CÑ'C×'JÑ'JÌ2Ï:É:Ó'Vˆ×ÑÔ$à�;‰;×&Ñ&Ð2Ü+-¯7©7Ø×%Ñ%×,Ñ,Ø—‘×,Ñ,Ð,Ø—‘×+Ñ+ó,ˆD×!Ñ!Ô(ðˆD×!Ñ! *Ó-ß‰Wðà×!Ñ! # 2Ð&ðð —‘×,Ñ,ðð —‘×,Ñ,ò	÷ ‰Y�q˜!˜Q Ó"ð 	ð �x‰xœŸ
™
Ó#ˆØ—^‘^ A q¨!¨QÓ/ˆ
Ü—i‘i 
¨IÓ6ˆØ!×+Ñ+¨A¨q°!°QÓ7ˆØ�tÑˆØ×!Ñ! !§'¡'Ó*Ð*r5   rÇ   Ú	activatedc                 óx  — | j                  |«      }| j                  j                  j                  t        j
                  «      | j                  _        | j                  j                  �ct	        j                  | j                  j                  | j                  j                   | j                  j                  «      | j                  _        | j                  |«      dz   }|| j                  j                     }||z
  }|j                  dd«      }|d    |d   z  }||z  }|j                  |j                  «      S )Nrg   r~   r   ).N)r¾   rµ   rÀ   r–   rE   rº   rN   r'   rÁ   r)   Úmoveaxisr—   )rB   rÇ   rÉ   rÃ   rÄ   Úactive_xÚ
innovationÚ	correcteds           r6   ÚcorrectzGemma3nAltUp.correct  s  € Ø×3Ñ3°IÓ>ˆ
à'+×'<Ñ'<×'CÑ'C×'JÑ'JÌ2Ï:É:Ó'Vˆ×ÑÔ$à�;‰;×&Ñ&Ð2Ü+-¯7©7Ø×%Ñ%×,Ñ,Ø—‘×,Ñ,Ð,Ø—‘×+Ñ+ó,ˆD×!Ñ!Ô(ð ×)Ñ)¨*Ó5¸Ñ;ˆ	Ø˜tŸ{™{×;Ñ;Ñ<ˆØ Ñ)ˆ
à×&Ñ& q¨!Ó,ˆ	Ø˜tÑ$ y°Ñ';Ñ;ˆ	Ø�[Ñ ˆ	à×Ñ 	§¡Ó0Ð0r5   )r,   r-   r.   r_   r   rA   rE   r`   r¾   rÈ   rÏ   rJ   rK   s   @r6   r±   r±   Ì   sc   ø„ Ù%ð
˜zõ 
ð&¨2¯8©8ð ¸¿¹ó ð
+˜Ÿ™ð + b§h¡hó +ð:1 2§8¡8ð 1¸¿¹÷ 1r5   r±   c            
       óš   ‡ — e Zd Zdededefˆ fd„Z	 	 	 d
dej                  de	ej                     de	e
   de	ej                     fd	„Zˆ xZS )ÚGemma3nDecoderLayerrN   rc   rd   c                 óä  •— t         ‰| �  «        || _        |j                  | _        || _        t        |||«      | _        t        ||¬«      | _        t        j                  | j                  |j                  ¬«      | _        t        j                  | j                  |j                  ¬«      | _        t        j                  | j                  |j                  ¬«      | _        t        j                  | j                  |j                  ¬«      | _        | j                  j                   | _        |j"                  | _        t%        |«      | _        t)        |«      | _        t        j,                  | j                  | j"                  d¬«      | _        t        j,                  | j"                  | j                  d¬«      | _        t        j                  | j                  |j                  ¬«      | _        y )N)rc   rh   FrP   )r@   rA   rN   r   rc   rb   Ú	self_attnr¡   ÚmlprT   rX   r   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormrk   r%   r±   ÚaltuprM   ÚlaurelrU   Úper_layer_input_gateÚper_layer_projectionÚpost_per_layer_input_norm)rB   rN   rc   rd   rC   s       €r6   rA   zGemma3nDecoderLayer.__init__  sƒ  ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ"ˆŒÜ)¨&°)Ð=OÓPˆŒÜ�v¨Ô3ˆŒÜ!Ÿz™zØ×ÑØ×#Ñ#ô 
ˆÔô
 )+¯
©
Ø×ÑØ×#Ñ#ô)
ˆÔ%ô *,¯©Ø×ÑØ×#Ñ#ô*
ˆÔ&ô +-¯*©*Ø×ÑØ×#Ñ#ô+
ˆÔ'ð Ÿ.™.×3Ñ3ˆŒà+1×+MÑ+MˆÔ(ä! &Ó)ˆŒ
Ü(¨Ó0ˆŒÜ$&§I¡IØ×Ñ˜d×>Ñ>ÀUô%
ˆÔ!ô %'§I¡IØ×,Ñ,¨d×.>Ñ.>ÀUô%
ˆÔ!ô *,¯©Ø×ÑØ×#Ñ#ô*
ˆÕ&r5   rH   rz   r{   Úper_layer_inputc                 ó  — | j                   j                  |«      }|| j                  j                     }| j	                  |«      }| j                  |«      }| j                  |||«      }	| j                  |	«      }	||	z   }
|
|z   dz  }| j                  |«      }| j                  |«      }| j                  |«      }||z   }| j                   j                  ||«      }|| j                  j                     }| j                  j                  r|| j                   j                  z  }| j                  |«      }t        j                   |«      }t#        j$                  ||«      }| j'                  |«      }| j)                  |«      }|dd  |z   |dd  |S )NçÍ;fž æ?r   )rÙ   rÈ   rN   r)   rÕ   rÚ   rÓ   rÖ   r×   rÔ   rØ   rÏ   r(   r´   rÛ   rT   r˜   rE   ÚmultiplyrÜ   rÝ   )rB   rH   rz   r{   rÞ   rÇ   Úactive_predictionÚactive_prediction_normedÚlaurel_outputÚattnÚ
attn_gatedÚattn_laurelÚ	attn_normÚattn_ffwÚattn_ffw_normÚattn_ffw_laurel_gatedÚcorrected_predictionsÚfirst_predictions                     r6   rI   zGemma3nDecoderLayer.__call__E  s�  € ð —j‘j×(Ñ(¨Ó+ˆØ'¨¯©×(DÑ(DÑEÐà#'×#7Ñ#7Ð8IÓ#JÐ ØŸ™Ð$<Ó=ˆà�~‰~Ø$ØØó
ˆð ×,Ñ,¨TÓ2ˆà&¨Ñ-ˆ
Ø! MÑ1°iÑ@ˆà×2Ñ2°;Ó?ˆ	Ø—8‘8˜IÓ&ˆØ×7Ñ7¸ÓAˆØ +¨mÑ ;ÐØ $§
¡
× 2Ñ 2°;Ð@UÓ VÐà0°·±×1MÑ1MÑNÐØ�;‰;×*Ò*Ø/°$·*±*×2QÑ2QÑQÐà×4Ñ4Ð5EÓFÐÜŸ>™>Ð*:Ó;ÐäŸ;™;Ð'7¸ÓIÐà×4Ñ4Ð5EÓFÐØ×9Ñ9Ð:JÓKÐà$9¸!¸"Ð$=Ð@PÑ$PÐ˜a˜bÐ!à$Ð$r5   ©NNNrŽ   rK   s   @r6   rÑ   rÑ     sn   ø„ ð'
˜zð '
°cð '
Ètõ '
ðX $(Ø#Ø.2ñ,%à�8‰8ð,%ð �r—x‘xÑ ð,%ð ˜‰}ð	,%ð
 " "§(¡(Ñ+÷,%r5   rÑ   c                 ó@   — t        j                  || z  «      }|| z  }|S r?   )rE   r»   )ÚsoftcaprH   Úouts      r6   Úlogit_softcaprò   t  s"   € ä
�'‰'�!�g‘+Ó
€CØ
�‰-€CØ€Jr5   c                   óò   ‡ — e Zd Zdefˆ fd„Z	 	 	 ddej                  dej                  fd„Zdej                  dej                  fd„Zd	ej                  d
ej                  dej                  fd„Z	d„ Z
ˆ xZS )ÚLanguageModelrN   c           	      ó  •— t         ‰	| �  «        || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  |j                  z
  | _
        t        j                  |j
                  |j                  «      | _        t        |j                  «      D �cg c]  }t        |||| j                  k\  ¬«      ‘Œ c}| _        t        j                  |j                  |j                  |j                  z  «      | _        t        j$                  |j                  |j                  |j                  z  d¬«      | _        t        j(                  |j                  |j*                  ¬«      | _        t        d| j                  j.                  «      D �cg c].  }t        j$                  |j                  |j                  d¬«      ‘Œ0 c}| _        t        d| j                  j.                  «      D �cg c].  }t        j$                  |j                  |j                  d¬«      ‘Œ0 c}| _        t        j(                  |j                  |j*                  ¬«      | _        |j6                  j9                  d«      | _        |j6                  j9                  d«      | _        |j>                  | _        |j6                  d | j                   }tA        |«      dz
  |d d d	…   j9                  d«      z
  }tA        |«      dz
  |d d d	…   j9                  d«      z
  }g | _!        tE        | j                  j6                  «      D ]  \  }}|| j                  k  r| jB                  jG                  |«       Œ1|dk(  r| jB                  jG                  |«       ŒR|dk(  r| jB                  jG                  |«       ŒstI        d
|› �«      ‚ y c c}w c c}w c c}w )N)rN   rc   rd   FrP   rR   r   rh   rf   Úfull_attentionr}   úUnknown layer type: )%r@   rA   rN   r   r%   r   r   r   r"   r   Úfirst_kv_shared_layer_idxrT   Ú	EmbeddingÚembed_tokensÚrangerÑ   ÚlayersÚembed_tokens_per_layerrU   Úper_layer_model_projectionrX   r   Úper_layer_projection_normr&   Úaltup_projectionsÚaltup_unembed_projectionsÚnormr#   ÚindexÚfirst_sliding_idxÚfirst_full_idxr   ÚlenÚlayer_idx_to_cache_idxÚ	enumerateÚappendÚNotImplementedError)
rB   rN   rc   rˆ   Úconcrete_layersÚshared_full_idxÚshared_sliding_idxÚiÚ
layer_typerC   s
            €r6   rA   zLanguageModel.__init__|  s‘  ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ+1×+MÑ+MˆÔ(Ø ×+Ñ+ˆŒØ*0×*KÑ*KˆÔ'Ø!'×!9Ñ!9ˆÔØ'-×'EÑ'EˆÔ$à×$Ñ$ v×'BÑ'BÑBð 	Ô&ô ŸL™L¨×):Ñ):¸F×<NÑ<NÓOˆÔô # 6×#;Ñ#;Ô<ó
ñ =�	ô  ØØ#Ø#,°×0NÑ0NÑ#Nöð
 =ñ
ˆŒô ')§l¡lØ×-Ñ-Ø×$Ñ$ v×'IÑ'IÑIó'
ˆÔ#ô
 +-¯)©)Ø×ÑØ×$Ñ$ v×'IÑ'IÑIØô+
ˆÔ'ô *,¯©Ø×3Ñ3Ø×#Ñ#ô*
ˆÔ&ô ˜1˜dŸk™k×:Ñ:Ô;ó"
á;�ô �I‰I�f×(Ñ(¨&×*<Ñ*<À5ÖIØ;ñ"
ˆÔô ˜1˜dŸk™k×:Ñ:Ô;ó*
á;�ô �I‰I�f×(Ñ(¨&×*<Ñ*<À5ÖIØ;ñ*
ˆÔ&ô
 —J‘JØ×ÑØ×#Ñ#ô
ˆŒ	ð
 "(×!3Ñ!3×!9Ñ!9Ð:MÓ!NˆÔØ$×0Ñ0×6Ñ6Ð7GÓHˆÔØ$×3Ñ3ˆÔà ×,Ñ,Ð-M¨t×/MÑ/MÐNˆä�Ó  1Ñ$ ±t¸°tÑ'<×'BÑ'BÐCSÓ'TÑTð 	ô �Ó  1Ñ$ ±t¸°tÑ'<×'BÑ'BÐCVÓ'WÑWð 	ð ')ˆÔ#Ü& t§{¡{×'>Ñ'>Ö?‰MˆAˆzØ�4×1Ñ1Ò1Ø×+Ñ+×2Ñ2°1Õ5àÐ!1Ò1Ø×/Ñ/×6Ñ6°ÕGØÐ#6Ò6Ø×/Ñ/×6Ñ6Ð7IÕJä-Ð0DÀZÀLÐ.QÓRÐRñ @ùòk
ùò2"
ùò
*
s   Ã!"O>Ç3PÈ43Prš   Úinput_embeddingsc                 ó  — |€"| j                  |«      | j                  dz  z  }n|}| j                  |«      }| j                  ||«      }|€d gt	        | j
                  «      z  }t        ||| j                     «      }t        ||| j                     | j                  ¬«      }|}t        j                  |dz  dd¬«      dz  }	|g}
|
j                  | j                  D �cg c]
  } ||«      ‘Œ c}«       t        j                  |
d¬«      }t        j                  |d	d  dz  dd¬«      dz  }|d	d  |	t        j                  |t        j                   |j"                  «      j$                  «      z  z  |d	d  t'        | j
                  «      D ]R  \  }}|d d …d d …|d d …f   }| j(                  j*                  |   d
k(  }|r|}n|} ||||| j,                  |      |«      }ŒT t        j                  |d   dz  dd¬«      dz  }	t'        | j.                  «      D ]  \  }} |||d	z      «      ||d	z   <   Œ t        j                  |d	d  dz  dd¬«      dz  }|d	d  |	t        j                  |t        j                   |j"                  «      j$                  «      z  z  |d	d  t        j                  |d¬«      }| j1                  |«      }| j                   j3                  |«      }| j4                  �t7        | j4                  |«      }|S c c}w )Nç      à?)Úwindow_sizer~   r}   Tr‘   r   )r’   r   rö   )rú   r   Úget_per_layer_inputsÚproject_per_layer_inputsr  rü   r   r  r  r   rE   r”   Úextendr   Ústackr™   Úfinfor—   Úminr  rN   r#   r  r  r  Ú	as_linearr"   rò   )rB   rš   r{   r  ÚhÚper_layer_inputsÚglobal_maskÚsliding_window_maskÚh0Útarget_magnitudeÚh_listÚprojÚmagsr  ÚlayerrÞ   Ú	is_globalrz   rñ   s                      r6   rI   zLanguageModel.__call__Ê  sñ  € ð Ð#Ø×!Ñ! &Ó)¨T×-=Ñ-=¸sÑ-BÑC‰Aà ˆAà×4Ñ4°VÓ<ÐØ×8Ñ8¸Ð<LÓMÐàˆ=Ø�FœS §¡Ó-Ñ-ˆEä+ØØ�$×%Ñ%Ñ&ó
ˆô 4ØØ�$×(Ñ(Ñ)Ø×+Ñ+ô
Ðð
 ˆô Ÿ7™7 2 q¡5¨r¸DÔAÀSÑHÐà�ˆØ�‰¨D×,BÒ,BÓCÑ,B D‘t˜B•xÐ,BÑCÔDÜ�H‰H�V !Ô$ˆÜ�w‰w�q˜˜�u ‘z¨°TÔ:¸cÑAˆØ�!�"�Ð)¬B¯J©J°t¼R¿X¹XÀbÇhÁhÓ=O×=SÑ=SÓ,TÑTÑUˆˆ!ˆ"ˆÜ! $§+¡+Ö.‰HˆAˆuØ.ªq²!°Qº¨zÑ:ˆOàŸ™×/Ñ/°Ñ2Ð6FÑFˆIáØ"‘à*�áØØØ�d×1Ñ1°!Ñ4Ñ5Øó	‰Að /ô$ Ÿ7™7 1 Q¡4¨1¡9°2ÀÔEÈÑLÐÜ  ×!?Ñ!?Ö@‰GˆAˆtÙ˜A˜a !™e™H“~ˆAˆa�!‰eŠHð Aä�w‰w�q˜˜�u ‘z¨°TÔ:¸cÑAˆØ�!�"�Ð)¬B¯J©J°t¼R¿X¹XÀbÇhÁhÓ=O×=SÑ=SÓ,TÑTÑUˆˆ!ˆ"ˆä�G‰G�A˜AÔˆà�i‰i˜‹lˆØ×Ñ×)Ñ)¨#Ó.ˆØ×'Ñ'Ð3Ü × <Ñ <¸cÓBˆCØˆ
ùòE Ds   ÃLÚ	input_idsr[   c                 ó  — || j                   k  }t        j                  ||t        j                  |«      «      }| j	                  |«      | j
                  dz  z  } |j                  g |j                  ¢| j                  ‘| j
                  ‘­Ž S )Nr  )	r   rE   ÚwhereÚ
zeros_likerý   r%   r‚   r�   r   )rB   r&  Úper_layer_inputs_maskÚtokensÚresults        r6   r  z"LanguageModel.get_per_layer_inputs  s‘   € Ø )¨D×,KÑ,KÑ KÐÜ—‘Ð/°¼B¿M¹MÈ)Ó<TÓUˆØ×,Ñ,¨VÓ4Ø×,Ñ,¨cÑ1ñ
ˆð ˆv�~‰~ð 
Ø�_‰_ð
à×"Ñ"ð
ð ×,Ñ,ò
ð 	
r5   Úinputs_embedsr  c                 ó  — | j                  |«      | j                  dz  z  } |j                  g |j                  d d ¢| j                  j
                  ‘| j                  j                  ‘­Ž }| j                  |«      }||z   dz  S )Ng      à¿r}   rà   )rþ   r   r‚   r�   rN   r   r%   rÿ   )rB   r-  r  rÜ   s       r6   r  z&LanguageModel.project_per_layer_inputs  sž   € ð
  $×>Ñ>¸}ÓMØ×Ñ˜dÑ"ñ 
Ðð  <Ð3×;Ñ;ð  
Ø× Ñ   "Ð%ð 
à�K‰K×)Ñ)ð 
ð �K‰K×3Ñ3ò 
Ðð
  $×=Ñ=Ð>RÓSÐØ$Ð'7Ñ7¸IÑFÐFr5   c                 ó  — g }| j                   j                  d | j                   D ]d  }|dk(  r|j                  t	        «       «       Œ"|dk(  r1|j                  t        | j                   j                  d¬«      «       ŒXt        d|› �«      ‚ |S )Nrö   rf   r   )Úmax_sizeÚkeepr÷   )rN   r#   rø   r	  r   r   r   r
  )rB   Úcachesr  s      r6   Ú
make_cachezLanguageModel.make_cache*  s†   € ØˆØŸ+™+×1Ñ1Ð2R°D×4RÑ4RÓSˆJØÐ-Ò-Ø—‘œg›iÕ(ØÐ2Ò2Ø—‘Ü#¨T¯[©[×-GÑ-GÈaÔPõô *Ð,@ÀÀÐ*MÓNÐNð Tð ˆr5   rî   )r,   r-   r.   r   rA   rE   r`   rI   r  r  r3  rJ   rK   s   @r6   rô   rô   {  s“   ø„ ðLS˜zõ LSð`  ØØ%)ñ	Bà—‘ðBð Ÿ(™(ó	BðH

¨b¯h©hð 

¸2¿8¹8ó 

ðGà—x‘xðGð Ÿ(™(ðGð 
�‰ó	Gö r5   rô   c                   ón   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     fd„Zd„ Z	ˆ xZ
S )ÚGemma3nÚargsc                 ó|   •— t         ‰| �  «        t        t        j	                  |j
                  «      «      | _        y r?   )r@   rA   rô   r   Ú	from_dictr9   Úlanguage_model©rB   r6  rC   s     €r6   rA   zGemma3n.__init__9  s,   ø€ Ü‰ÑÔÜ+¬J×,@Ñ,@À×AQÑAQÓ,RÓSˆÕr5   rš   r  c                 ó*   — | j                  |||¬«      S ©N)r{   r  )r9  ©rB   rš   r{   r  s       r6   rI   zGemma3n.__call__=  s$   € ð ×"Ñ"Ø˜%Ð2Bð #ó 
ð 	
r5   c                 ó6   — | j                   j                  «       S r?   )r9  r3  ©rB   s    r6   r3  zGemma3n.make_cacheG  s   € Ø×"Ñ"×-Ñ-Ó/Ð/r5   r�   )r,   r-   r.   r8   rA   rE   r`   r   rI   r3  rJ   rK   s   @r6   r5  r5  8  sB   ø„ ðT˜Yõ Tð Ø/3ñ	
à—‘ð
ð # 2§8¡8Ñ,ó	
ö0r5   r5  c                   ó„   ‡ — e Zd Zdefˆ fd„Z	 	 d	dej                  deej                     fd„Zd„ Z	e
d„ «       Zd„ Zˆ xZS )
ÚModelr6  c                 ór   •— t         ‰| �  «        || _        t        |«      | _        |j
                  | _        y r?   )r@   rA   r6  r5  Úmodelr   r:  s     €r6   rA   zModel.__init__L  s+   ø€ Ü‰ÑÔØˆŒ	Ü˜T“]ˆŒ
ØŸ/™/ˆ�r5   rš   r  c                 ó*   — | j                  |||¬«      S r<  )rC  r=  s       r6   rI   zModel.__call__R  s   € ð �z‰z˜&¨Ð@PˆzÓQÐQr5   c                 ó¦   — t        t        |j                  «       «      «      }dD ]  }|d   j                  |d «       Œ t	        t        |«      «      S )N)Úvision_towerÚaudio_towerÚembed_audioÚembed_visionrC  )r
   r¥   ÚitemsÚpopr:   r	   )rB   ÚweightsÚks      r6   ÚsanitizezModel.sanitizeZ  sG   € Ü ¤ g§m¡m£oÓ!6Ó7ˆÛOˆAØ�GÑ× Ñ   DÕ)ð Pä”L Ó)Ó*Ð*r5   c                 óB   — | j                   j                  j                  S r?   )rC  r9  rü   r?  s    r6   rü   zModel.layers`  s   € à�z‰z×(Ñ(×/Ñ/Ð/r5   c                 ó6   — | j                   j                  «       S r?   )rC  r3  r?  s    r6   r3  zModel.make_cached  s   € Ø�z‰z×$Ñ$Ó&Ð&r5   r�   )r,   r-   r.   r8   rA   rE   r`   r   rI   rN  Úpropertyrü   r3  rJ   rK   s   @r6   rA  rA  K  s\   ø„ ð*˜Yõ *ð Ø/3ñ	Rà—‘ðRð # 2§8¡8Ñ,ó	Rò+ð ñ0ó ð0ö'r5   rA  )(rª   Údataclassesr   Ú	functoolsr   Útypingr   r   r   r   Úmlx.coreÚcorerE   Úmlx.nnrT   Ú	mlx.utilsr	   r
   rj   r   r   r   r{   r   r   r   r8   ÚModuler<   rM   rb   ÚcompilerŸ   r¡   r±   rÑ   rò   rô   r5  rA  r4   r5   r6   Ú<module>r[     s9  ðó Ý !Ý ß ,Ó ,å Ý ß 2ç TÑ Tß +ð ô(�ó (ó ð(ð8 ô�ó ó ðô
3�—‘ô 3ô#˜Ÿ™ô #ô2K#�r—y‘yô K#ñ\ 	ˆ�‰˜tÔ$ñ<ó %ð<ôˆ"�)‰)ô ôBL1�2—9‘9ô L1ô^V%˜"Ÿ)™)ô V%ñr 	ˆ�‰˜tÔ$ñó %ðôz�B—I‘Iô zôz0ˆb�i‰iô 0ô&'ˆB�I‰Iõ 'r5   