Ë
    îÿæiðF  ã                   óÚ  — d dl mZmZ d dlmZmZmZmZmZ d dl	m
Z d dlmZ d dlmZmZmZ d dlmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZ ddlm Z! ddlm"Z# ddlm$Z% e G d„ de«      «       Z& G d„ dejN                  «      Z( G d„ dejN                  «      Z) G d„ dejN                  «      Z* G d„ dejN                  «      Z+e G d„ de«      «       Z, G d„ dejN                  «      Z-y)é    )Ú	dataclassÚfield)ÚAnyÚDictÚListÚOptionalÚUnionN)Úshard_inplaceÚshard_linearÚsum_gradients)Útree_mapé   )ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_mask)ÚArraysCacheÚKVCache)Úgated_delta_update)ÚQwen3NextAttention)ÚQwen3NextMLP)ÚQwen3NextRMSNormGated)ÚQwen3NextSparseMoeBlockc            	       óð  — e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   dZ
eed	<   d
Zeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee   ed <   dZeed!<   d"Zeed#<   d"Zeed$<   d%Zeed&<   d"Zeed'<   d"Zeed(<   d)Z eed*<    e!d+„ ¬,«      Z"ee#ee$eeee%e   f   f      ed-<   d.Z&eed/<   d0Z'eed1<   dZ(ee#ee$eef   f      ed2<   d3„ Z)y)4ÚTextModelArgsÚ Ú
model_typei   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsç�íµ ÷Æ°>Úrms_norm_epsi€Q Ú
vocab_sizeé   Únum_key_value_headsi   Úmax_position_embeddingsé@   Úlinear_num_value_headsé   Úlinear_num_key_headséÀ   Úlinear_key_head_dimé€   Úlinear_value_head_dimé   Úlinear_conv_kernel_dimFÚtie_word_embeddingsÚattention_biasNÚhead_dimÚfull_attention_intervalr   Únum_expertsÚnum_experts_per_tokr   Údecoder_sparse_stepÚshared_expert_intermediate_sizeÚmoe_intermediate_sizeTÚnorm_topk_probc                  ó   — dg d¢dddœS )NÚdefault)é   r>   é
   i † ç      Ð?)ÚtypeÚmrope_sectionÚ
rope_thetaÚpartial_rotary_factor© rE   ó    új/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen3_5.pyÚ<lambda>zTextModelArgs.<lambda>7   s   € ØÚ)Ø Ø%)ò	!
rF   )Údefault_factoryÚrope_parametersr@   rD   ç     jø@rC   Úrope_scalingc                 ó   — | j                   €| j                  | j                  z  | _         | j                  r˜d| j                  vr6d| j                  v r(| j                  j	                  d«      | j                  d<   | j                  j                  dd«      | _        | j                  j                  dd«      | _        | j                  | _        y y )NrA   Ú	rope_typerD   r@   rC   rK   )	r4   r   r!   rJ   ÚpopÚgetrD   rC   rL   ©Úselfs    rG   Ú__post_init__zTextModelArgs.__post_init__D   s·   € Ø�=‰=Ð Ø ×,Ñ,°×0HÑ0HÑHˆDŒMà×Òà˜d×2Ñ2Ñ2Ø 4×#7Ñ#7Ñ7à/3×/CÑ/C×/GÑ/GÈÓ/T�×$Ñ$ VÑ,à)-×)=Ñ)=×)AÑ)AØ'¨ó*ˆDÔ&ð #×2Ñ2×6Ñ6°|ÀXÓNˆDŒOØ $× 4Ñ 4ˆDÕð  rF   )*Ú__name__Ú
__module__Ú__qualname__r   ÚstrÚ__annotations__r   Úintr   r    r!   r#   Úfloatr$   r&   r'   r)   r+   r-   r/   r1   r2   Úboolr3   r4   r   r5   r6   r7   r8   r9   r:   r;   r   rJ   r   r	   r   rD   rC   rL   rS   rE   rF   rG   r   r      s‚  … à€J�ÓØ€K�ÓØ"Ð�sÓ"ØÐ�sÓØ!Ð˜Ó!Ø€L�%ÓØ€J�ÓØ Ð˜Ó Ø#)Ð˜SÓ)Ø"$Ð˜CÓ$Ø "Ð˜#Ó"Ø"Ð˜Ó"Ø!$Ð˜3Ó$Ø"#Ð˜CÓ#Ø %Ð˜Ó%Ø €N�DÓ Ø"€Hˆh�s‰mÓ"Ø#$Ð˜SÓ$ð €K�ÓØ Ð˜Ó Ø Ð˜Ó Ø+,Ð# SÓ,Ø!"Ð˜3Ó"Ø€N�DÓñ PUñ
ôP€O�X˜d 3¨¨e°S¸$ÀÀSÁ	Ð.IÑ(JÐ#JÑKÑLó ð $(Ð˜5Ó'Ø €J�Ó Ø;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?ó5rF   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚGatedDeltaNetÚconfigc                 ó  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _
        | j                  | j                  z  | _        | j                  | j                  z  | _        | j                  | j                  z  dk7  r&t        d| j                  › d| j                  › d�«      ‚|j                  | _        |j                   | _        | j                  dz  | j                  z   | _        t'        j(                  | j$                  | j$                  d| j                  | j$                  d¬«      | _        t'        j,                  | j                  | j                  dz  | j                  z   d¬«      | _        t'        j,                  | j                  | j                  d¬«      | _        t'        j,                  | j                  | j                  d¬«      | _        t'        j,                  | j                  | j                  d¬«      | _        t7        j8                  | j                  «      | _        t6        j<                  j?                  dd	| j                  f¬
«      }t7        j@                  |«      | _!        tE        | j                  | j"                  ¬«      | _#        t'        j,                  | j                  | j                  d¬«      | _$        d | _%        y )Nr   znum_v_heads (z$) must be divisible by num_k_heads (Ú)é   F)Úin_channelsÚout_channelsÚbiasÚkernel_sizeÚgroupsÚpadding©rd   r*   )ÚlowÚhighÚshape©Úeps)&ÚsuperÚ__init__r   r)   Únum_v_headsr+   Únum_k_headsr-   Ú
head_k_dimr/   Ú
head_v_dimÚkey_dimÚ	value_dimÚ
ValueErrorr1   Úconv_kernel_sizer#   Úlayer_norm_epsilonÚconv_dimÚnnÚConv1dÚconv1dÚLinearÚin_proj_qkvÚ	in_proj_zÚ	in_proj_bÚ	in_proj_aÚmxÚonesÚdt_biasÚrandomÚuniformÚlogÚA_logÚRMSNormGatedÚnormÚout_projÚsharding_group)rR   r^   ÚAÚ	__class__s      €rG   ro   zGatedDeltaNet.__init__W   sK  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ!×8Ñ8ˆÔØ!×6Ñ6ˆÔØ ×4Ñ4ˆŒØ ×6Ñ6ˆŒØ—‘¨×)9Ñ)9Ñ9ˆŒØŸ™¨4×+;Ñ+;Ñ;ˆŒØ×Ñ˜d×.Ñ.Ñ.°!Ò3ÜØ × 0Ñ 0Ð1Ð1UÐVZ×VfÑVfÐUgÐghÐióð ð !'× =Ñ =ˆÔØ"(×"5Ñ"5ˆÔàŸ™ qÑ(¨4¯>©>Ñ9ˆŒÜ—i‘iØŸ™ØŸ™ØØ×-Ñ-Ø—=‘=Øô
ˆŒô Ÿ9™9Ø×Ñ˜dŸl™l¨QÑ.°·±Ñ?Àeô
ˆÔô Ÿ™ 4×#3Ñ#3°T·^±^È%ÔPˆŒÜŸ™ 4×#3Ñ#3°T×5EÑ5EÈEÔRˆŒÜŸ™ 4×#3Ñ#3°T×5EÑ5EÈEÔRˆŒä—w‘w˜t×/Ñ/Ó0ˆŒä�I‰I×Ñ !¨"°T×5EÑ5EÐ4GÐÓHˆÜ—V‘V˜A“YˆŒ
ä  §¡°d×6MÑ6MÔNˆŒ	äŸ	™	 $§.¡.°$×2BÑ2BÈÔOˆŒà"ˆÕrF   ÚinputsÚmaskÚcacheÚreturnc                 óD  — |j                   \  }}}| j                  � t        | j                  «      |«      }| j                  |«      }| j	                  |«      j                  ||| j                  | j                  «      }| j                  |«      }	| j                  |«      }
|�|d   �|d   }n;t        j                  || j                  dz
  | j                  f|j                  ¬«      }|�t        j                  |d   |d«      }t        j                   ||gd¬«      }|�|d d …| j                  dz
   d …f   |d<   t#        j$                  | j'                  |«      «      }t)        t        j*                  || j,                  d| j,                  z  gd«      | j.                  | j.                  | j                  g| j0                  | j0                  | j                  g«      D ���cg c]  \  }}}|j                  ||||«      ‘Œ c}}}\  }}}|r|d   nd }|j                   d   dz  }|dz  t        j2                  j5                  |d d	«      z  }|t        j2                  j5                  |d d	«      z  }t7        ||||
|	| j8                  | j:                  ||| j<                   ¬
«
      \  }}|�||d<   | j?                  ||«      }| jA                  |j                  ||d«      «      }| j                  �+t        jB                  jE                  || j                  ¬«      }|S c c}}}w )Nr   r   )Údtype).N©Úaxisra   éÿÿÿÿg      à¿r"   )Ú
use_kernel©Úgroup)#rk   rŒ   r   r~   r   Úreshaperp   rs   r€   r�   r‚   Úzerosrw   ry   r”   ÚwhereÚconcatenaterz   Úsilur|   ÚzipÚsplitrt   rq   rr   ÚfastÚrms_normr   rˆ   r„   ÚtrainingrŠ   r‹   ÚdistributedÚall_sum)rR   r�   r�   r‘   ÚBÚSÚ_ÚqkvÚzÚbÚaÚ
conv_stateÚ
conv_inputÚconv_outÚtÚhÚdÚqÚkÚvÚstateÚ	inv_scaleÚouts                          rG   Ú__call__zGatedDeltaNet.__call__„   sè  € ð —,‘,‰ˆˆ1ˆaà×ÑÐ*Ø7”] 4×#6Ñ#6Ó7¸Ó?ˆFà×Ñ˜vÓ&ˆØ�N‰N˜6Ó"×*Ñ*¨1¨a°×1AÑ1AÀ4Ç?Á?ÓSˆØ�N‰N˜6Ó"ˆØ�N‰N˜6Ó"ˆàÐ  q¡Ð!5Ø˜q™‰JäŸ™Ø�D×)Ñ)¨AÑ-¨t¯}©}Ð=Ø—l‘lôˆJð
 ÐÜ—(‘(˜4 	™?¨C°Ó3ˆCÜ—^‘^ Z°Ð$5¸AÔ>ˆ
ØÐØ!¢! t×'<Ñ'<¸qÑ'@Ð%AÑ%CÐ"CÑDˆE�!‰HÜ—7‘7˜4Ÿ;™; zÓ2Ó3ˆô Ü—‘˜ D§L¡L°!°d·l±lÑ2BÐ#CÀRÓHØ×!Ñ! 4×#3Ñ#3°T×5EÑ5EÐFØ—‘ $§/¡/°4·?±?ÐCôõ
ñ‘��1�að �I‰I�a˜˜A˜qÕ!ðó
‰ˆˆ1ˆañ "��a’ tˆØ—G‘G˜B‘K 4Ñ'ˆ	Ø˜‰\œRŸW™W×-Ñ-¨a°°tÓ<Ñ<ˆØœŸ™×(Ñ(¨¨D°$Ó7Ñ7ˆä'ØØØØØØ�J‰JØ�L‰LØØØŸ=™=Ð(ô
‰
ˆˆUð ÐØˆE�!‰Hà�i‰i˜˜QÓˆØ�m‰m˜CŸK™K¨¨1¨bÓ1Ó2ˆà×ÑÐ*Ü—.‘.×(Ñ(¨°D×4GÑ4GÐ(ÓHˆCàˆ
ùôI
s   Ç"L©NN©rT   rU   rV   r   ro   r‚   Úarrayr   r   rº   Ú__classcell__©rŽ   s   @rG   r]   r]   V   s]   ø„ ð+#˜}õ +#ð` $(Ø#ñ	Cà—‘ðCð �r—x‘xÑ ðCð ˜‰}ð	Cð
 
�‰÷CrF   r]   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚDecoderLayerÚargsÚ	layer_idxc                 óú  •— t         ‰| �  «        |dz   |j                  z  dk7  | _        | j                  rt	        |«      | _        nt        |«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        |j                  dkD  rt        |«      | _        y t#        |j                  |j$                  «      | _        y )Nr   r   rl   )rn   ro   r5   Ú	is_linearr]   Úlinear_attnÚ	AttentionÚ	self_attnrz   ÚRMSNormr   r#   Úinput_layernormÚpost_attention_layernormr6   ÚSparseMoeBlockÚmlpÚMLPr   )rR   rÂ   rÃ   rŽ   s      €rG   ro   zDecoderLayer.__init__Ë   s¿   ø€ Ü‰ÑÔØ# a™-¨4×+GÑ+GÑGÈ1ÑLˆŒØ�>Š>Ü,¨TÓ2ˆDÕä& t›_ˆDŒNä!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð ×Ñ˜aÒÜ% dÓ+ˆD�Hä˜4×+Ñ+¨T×-CÑ-CÓDˆD�HrF   Úxr�   r‘   r’   c                 óø   — | j                   r#| j                  | j                  |«      ||«      }n"| j                  | j                  |«      ||«      }||z   }|| j	                  | j                  |«      «      z   }|S ©N)rÅ   rÆ   rÊ   rÈ   rÍ   rË   )rR   rÏ   r�   r‘   Úrr²   r¹   s          rG   rº   zDecoderLayer.__call__Ý   ss   € ð �>Š>Ø× Ñ  ×!5Ñ!5°aÓ!8¸$ÀÓF‰Aà—‘˜t×3Ñ3°AÓ6¸¸eÓDˆAØ�‰EˆØ�$—(‘(˜4×8Ñ8¸Ó;Ó<Ñ<ˆØˆ
rF   r»   )rT   rU   rV   r   rY   ro   r‚   r½   r   r   rº   r¾   r¿   s   @rG   rÁ   rÁ   Ê   sa   ø„ ðE˜]ð E°sõ Eð* $(Ø#ñ	à�8‰8ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷rF   rÁ   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  dee   deej                     dej                  fd„Z	ˆ xZ
S )	ÚQwen3_5TextModelrÂ   c                 ó†  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}| _
        t        j                  |j
                  |j                  ¬«      | _        d| _        |j                  dz
  | _        y c c}w )N)rÂ   rÃ   rl   r   r   )rn   ro   rz   Ú	Embeddingr$   r   Úembed_tokensÚranger    rÁ   ÚlayersrÉ   r#   rŠ   Ússm_idxr5   Úfa_idx)rR   rÂ   ÚirŽ   s      €rG   ro   zQwen3_5TextModel.__init__í   s—   ø€ Ü‰ÑÔÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔä:?À×@VÑ@VÔ:Wó
Ù:W°QŒL˜d¨aÖ0Ð:Wñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	ØˆŒØ×2Ñ2°QÑ6ˆ�ùò
s   ÁB>r�   r‘   Úinput_embeddingsr’   c                 ó\  — |�|}n| j                  |«      }|€d gt        | j                  «      z  }t        ||| j                     «      }t        ||| j                     «      }t        | j                  |«      D ]   \  }}|j                  r|n|}	 |||	|¬«      }Œ" | j                  |«      S )N)r�   r‘   )
r×   ÚlenrÙ   r   rÛ   r   rÚ   r    rÅ   rŠ   )
rR   r�   r‘   rÝ   Úhidden_statesÚfa_maskÚssm_maskÚlayerÚcr�   s
             rG   rº   zQwen3_5TextModel.__call__÷   s¨   € ð Ð'Ø,‰Mà ×-Ñ-¨fÓ5ˆMàˆ=Ø�FœS §¡Ó-Ñ-ˆEä'¨°u¸T¿[¹[Ñ7IÓJˆÜ" =°%¸¿¹Ñ2EÓFˆä˜DŸK™K¨Ö/‰HˆE�1Ø$Ÿš‘8°GˆDÙ! -°dÀ!ÔD‰Mð 0ð �y‰y˜Ó'Ð'rF   r»   r¼   r¿   s   @rG   rÔ   rÔ   ì   sW   ø„ ð7˜]õ 7ð  $Ø/3ñ	(à—‘ð(ð ˜‰}ð(ð # 2§8¡8Ñ,ð	(ð
 
�‰÷(rF   rÔ   c            	       óÆ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  dee   deej                     dej                  fd„Z	e
d„ «       Zd	„ Zd
„ Ze
d„ «       Ze
d„ «       Zˆ xZS )Ú	TextModelrÂ   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y )NFrh   )rn   ro   rÂ   r   rÔ   Úmodelr2   rz   r}   r   r$   Úlm_head©rR   rÂ   rŽ   s     €rG   ro   zTextModel.__init__  sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ% dÓ+ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (rF   r�   r‘   rÝ   r’   c                 óÊ   — | j                  |||¬«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S )N)rÝ   )rè   rÂ   r2   r×   Ú	as_linearré   )rR   r�   r‘   rÝ   r¹   s        rG   rº   zTextModel.__call__  s\   € ð �j‰j˜ Ð9IˆjÓJˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
rF   c                 ó.   — | j                   j                  S rÑ   )rè   rÙ   rQ   s    rG   rÙ   zTextModel.layers%  s   € à�z‰z× Ñ Ð rF   c                 ó€   — | j                   D �cg c]$  }|j                  rt        d¬«      n	t        «       ‘Œ& c}S c c}w )Nra   )Úsize)rÙ   rÅ   r   r   )rR   Úls     rG   Ú
make_cachezTextModel.make_cache)  s0   € ØKOÏ;Ê;ÓWÉ;Àa q§{¢{” Õ#¼»	ÑAÈ;ÑWÐWùÒWs   �);c                 ó  ‡— t        d„ |D «       «      }t        d„ |j                  «       D «       «      }|xs |}|j                  «       D ��ci c]  \  }}d|vsŒ||“Œ }}}| j                  j                  r|j	                  dd «       d}|j                  «       D ]`  \  Š}d‰v r'|j
                  d   dk7  r|j                  d	d«      |‰<   |sŒ4t        ˆfd
„|D «       «      sŒI|j                  dk(  sŒY|dz   |‰<   Œb |S c c}}w )Nc              3   ó$   K  — | ]  }d |v –— Œ
 y­w)úmtp.NrE   )Ú.0rµ   s     rG   Ú	<genexpr>z%TextModel.sanitize.<locals>.<genexpr>-  s   è ø€ Ð;±7¨a˜f¨œk±7ùs   ‚c              3   óR   K  — | ]  \  }}d |v xr |j                   d   dk7  –— Œ! y­w)úconv1d.weightr—   r   N)rk   )rõ   rµ   r¶   s      rG   rö   z%TextModel.sanitize.<locals>.<genexpr>.  s2   è ø€ ð %
ÙBQ¹$¸!¸QˆO˜qÐ Ò5 Q§W¡W¨R¡[°AÑ%5Ó5Á/ùs   ‚%'rô   zlm_head.weight)z.input_layernorm.weightz .post_attention_layernorm.weightzmodel.norm.weightz.q_norm.weightz.k_norm.weightrø   r—   r   ra   c              3   ó@   •K  — | ]  }‰j                  |«      –— Œ y ­wrÑ   ©Úendswith)rõ   Úsfxrµ   s     €rG   rö   z%TextModel.sanitize.<locals>.<genexpr>A  s   øè ø€ Ð0VÉIÀS°·±¸C·ÉIùs   ƒg      ð?)ÚanyÚitemsrÂ   r2   rO   rk   ÚmoveaxisÚndim)rR   ÚweightsÚhas_mtp_weightsÚhas_unsanitized_conv1dÚshould_shift_norm_weightsrµ   r¶   Ú	norm_keyss        `  rG   ÚsanitizezTextModel.sanitize,  s   ø€ ÜÑ;±7Ó;Ó;ˆÜ!$ñ %
ØBIÇ-Á-Ä/ó%
ó "
Ðð %4Ò$MÐ7MÐ!Ø$+§M¡M¤OÔG¡O™D˜A˜q°vÀQ²�1�a‘4 OˆÑGà�9‰9×(Ò(Ø�K‰KÐ(¨$Ô/ð
ˆ	ð —M‘M–O‰DˆAˆqØ !Ñ#¨¯©°©°qÒ(8ØŸZ™Z¨¨1Ó-�˜‘
Ú(¬SÓ0VÉIÓ0VÕ-VØ—6‘6˜Q“;Ø!" S¡�G˜A’Jð $ð ˆùó% Hs   ÁDÁDc                 ó@   — | j                   j                  dk  ry d„ }|S )Nr   c                 óR   — | j                  d«      s| j                  d«      rdddœS y)Nzmlp.gateÚshared_expert_gater(   r%   )Ú
group_sizeÚbitsTrú   )Úpathr©   s     rG   Ú	predicatez,TextModel.quant_predicate.<locals>.predicateK  s(   € Ø�}‰}˜ZÔ(¨D¯M©MÐ:NÔ,OØ&(°!Ñ4Ð4ØrF   )rÂ   r6   ©rR   r  s     rG   Úquant_predicatezTextModel.quant_predicateF  s$   € à�9‰9× Ñ  AÒ%Øò	ð
 ÐrF   c                 ó   — dt         fd„}|S )Nr  c                 ó(   — | j                  d«      ryy)Nrˆ   FTrú   )r  s    rG   r  z+TextModel.cast_predicate.<locals>.predicateT  s   € Ø�}‰}˜WÔ%ØØrF   )rW   r  s     rG   Úcast_predicatezTextModel.cast_predicateR  s   € ð	œCó 	ð
 ÐrF   r»   )rT   rU   rV   r   ro   r‚   r½   r   r   rº   ÚpropertyrÙ   rñ   r  r  r  r¾   r¿   s   @rG   ræ   ræ     s    ø„ ðT˜]õ Tð  $Ø/3ñ	à—‘ðð ˜‰}ðð # 2§8¡8Ñ,ð	ð
 
�‰óð ñ!ó ð!òXòð4 ñ	ó ð	ð ñó ôrF   ræ   c                   ó>   ‡ — e Zd ZU eed<   eed<   eˆ fd„«       Zˆ xZS )Ú	ModelArgsr   Útext_configc                 óD   •— d|vr | |d   |¬«      S t         ‰| �  |«      S )Nr  r   )r   r  )rn   Ú	from_dict)ÚclsÚparamsrŽ   s     €rG   r  zModelArgs.from_dicta  s-   ø€ à Ñ&Ù &¨Ñ"6ÀFÔKÐKÜ‰wÑ  Ó(Ð(rF   )	rT   rU   rV   rW   rX   ÚdictÚclassmethodr  r¾   r¿   s   @rG   r  r  \  s    ø… àƒOØÓàó)ó ô)rF   r  c                   ó¬   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     fd„Zd„ Z	dd„Z
ed„ «       Zd	„ Zed
„ «       Zed„ «       Zˆ xZS )ÚModelrÂ   c                 ó¬   •— t         ‰| �  «        || _        |j                  | _        t	        t
        j                  |j                  «      «      | _        y rÑ   )	rn   ro   rÂ   r   ræ   r   r  r  Úlanguage_modelrê   s     €rG   ro   zModel.__init__i  s>   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ'¬×(?Ñ(?À×@PÑ@PÓ(QÓRˆÕrF   r�   rÝ   c                 ó*   — | j                  |||¬«      S )N)r‘   rÝ   )r   )rR   r�   r‘   rÝ   s       rG   rº   zModel.__call__o  s$   € ð ×"Ñ"Ø˜%Ð2Bð #ó 
ð 	
rF   c                 óV  — i }|j                  «       D ]z  \  }}|j                  d«      s|j                  d«      rŒ)|j                  d«      rŒ;|j                  d«      r|j                  dd«      }n|j                  d«      rnd|z   }|||<   Œ| | j                  j	                  |«      S )NÚvision_towerzmodel.visualzmodel.language_modelzlanguage_model.modelzlanguage_model.)rþ   Ú
startswithÚreplacer   r  )rR   r  Ú	sanitizedÚkeyÚvalues        rG   r  zModel.sanitizey  sŸ   € Øˆ	Ø!Ÿ-™-ž/‰JˆC�Ø�~‰~˜nÔ-°·±ÀÔ1OØØ�~‰~˜nÔ-ØØ�~‰~Ð4Ô5Ø—k‘kÐ"8Ð:PÓQ‘Ø—‘Ð 1Ô2Øà'¨#Ñ-�Ø"ˆI�cŠNð *ð ×"Ñ"×+Ñ+¨IÓ6Ð6rF   c                 ó  ‡— |xs t         j                  j                  «       }|j                  «       Š|j	                  «       }d„ }ˆfd„}| j
                  D �]²  }|j                  �rc|j                  j                  }||j                  _	        t        |j                  j                   ||«      |¬«       |j                  j                  xj                  ‰z  c_        t        |j                  j                  d|d|z  g|¬«       t        |j                  j                  d|¬«       t        |j                  j                  d|¬«       t        |j                  j                   d|¬«       t        j"                  t        j$                  |j                  j&                  ‰«      |   «      |j                  _        t        j"                  t        j$                  |j                  j(                  ‰«      |   «      |j                  _        t        |j                  j*                  d|¬«       |j                  xj,                  ‰z  c_        |j                  xj.                  ‰z  c_        |j                  xj                  ‰z  c_        |j                  xj0                  ‰z  c_        |j                  xj2                  ‰z  c_        �nwt5        |j6                  j8                  d|¬«      |j6                  _        t5        |j6                  j:                  d|¬«      |j6                  _         ||j6                  j<                  |j6                  j>                  «        ||j6                  j@                  |j6                  j>                  «       t5        |j6                  j<                  d|¬«      |j6                  _        t5        |j6                  j@                  d|¬«      |j6                  _         |j6                  xjB                  ‰z  c_!        tE        d|j6                  j>                  ‰z  «      |j6                  _        tG        |jH                  tJ        «      r•t5        |jH                  jL                  d|¬«      |jH                  _&        t5        |jH                  jN                  d|¬«      |jH                  _'        t5        |jH                  jP                  d|¬«      |jH                  _(        �Œš||jH                  _	        t        |jH                  jR                  jL                  d|¬«       t        |jH                  jR                  jN                  d|¬«       t        |jH                  jR                  jP                  d|¬«       t        |jH                  jT                  jL                  d|¬«       t        |jH                  jT                  jN                  d|¬«       t        |jH                  jT                  jP                  d|¬«       �Œµ y )	Nc                 ó   ‡ — ˆ fd„S )Nc                 ó   •— d‰d‰z  gfS )Nr   ra   rE   )ÚpÚwrt   s     €rG   rH   z4Model.shard.<locals>.conv_sharding.<locals>.<lambda>�  s   ø€   W¨a°'©kÐ$:Ñ ;rF   rE   )rt   s   `rG   Úconv_shardingz"Model.shard.<locals>.conv_sharding�  s	   ø€ Û;Ð;rF   c                 ór   •‡— ‰‰k  ry ˆˆfd„}| j                  t        || j                  «       «      «       y )Nc                 ó¼   •— | j                   } | j                  ‰|d   ‰z  g|dd  ¢­Ž } t        j                  | ‰‰z  d¬«      }  | j                  dg|dd  ¢­Ž } | S )Nr   r   r•   r—   )rk   r›   r‚   Úrepeat)r,  ÚsÚNr²   s     €€rG   Ú_repeatz=Model.shard.<locals>.repeat_kv_layer_inplace.<locals>._repeat˜  sg   ø€ Ø—G‘G�Ø�A—I‘I˜a  1¡¨¡Ð3¨Q¨q¨r¨UÒ3�Ü—I‘I˜a  a¡¨aÔ0�Ø�A—I‘I˜bÐ) 1 Q R 5Ò)�Ø�rF   )Úupdater   Ú
parameters)rã   r²   r4  r3  s    ` €rG   Úrepeat_kv_layer_inplacez,Model.shard.<locals>.repeat_kv_layer_inplace’  s0   ù€ à�AŠvØõð �L‰Lœ '¨5×+;Ñ+;Ó+=Ó>Õ?rF   r™   zall-to-shardedra   )Úsegmentsrš   zsharded-to-allr   )+r‚   r¥   Úinitrï   ÚrankrÙ   rÅ   rÆ   rt   rŒ   r
   r|   rf   r~   r   r€   r�   Ú
contiguousr¡   r„   rˆ   r‹   rq   rp   ru   ry   r   rÈ   Úo_projÚq_projÚk_projr&   Úv_projr!   ÚmaxÚ
isinstancerÍ   rÎ   Ú	gate_projÚ	down_projÚup_projÚshared_expertÚ
switch_mlp)rR   rš   r:  r.  r7  rã   Úkdr3  s          @rG   ÚshardzModel.shard‰  s™  ø€ ØÒ.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ�z‰z‹|ˆò	<ô	@ð —[•[ˆEà�‹Ø×&Ñ&×.Ñ.�Ø38�×!Ñ!Ô0Ü˜e×/Ñ/×6Ñ6¹ÀbÓ8IÐQVÕWØ×!Ñ!×(Ñ(×/Ò/°AÑ5Õ/ÜØ×%Ñ%×1Ñ1Ø$Ø  ! b¡&˜\Øõ	ô Ø×%Ñ%×/Ñ/Ð1AÈõô Ø×%Ñ%×/Ñ/Ð1AÈõô Ø×%Ñ%×/Ñ/Ð1AÈõô -/¯M©MÜ—H‘H˜U×.Ñ.×6Ñ6¸Ó:¸4Ñ@ó-�×!Ñ!Ô)ô +-¯-©-Ü—H‘H˜U×.Ñ.×4Ñ4°aÓ8¸Ñ>ó+�×!Ñ!Ô'ô ˜e×/Ñ/×8Ñ8Ð:JÐRWÕXØ×!Ñ!×-Ò-°!Ñ3Õ-Ø×!Ñ!×-Ò-°!Ñ3Õ-Ø×!Ñ!×)Ò)¨aÑ/Õ)Ø×!Ñ!×+Ò+°Ñ1Õ+Ø×!Ñ!×*Ò*¨qÑ0×*ô *6Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Ô&ô *6Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Ô&ñ (Ø—O‘O×*Ñ*¨E¯O©O×,OÑ,Oôñ (Ø—O‘O×*Ñ*¨E¯O©O×,OÑ,Oôô *6Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Ô&ô *6Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Ô&ð —‘×3Ò3¸Ñ9Õ3Ü69Ø�u—‘×:Ñ:¸aÑ?ó7�—‘Ô3ô
 ˜%Ÿ)™)¤SÔ)Ü&2Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô '3Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô %1Ø—I‘I×%Ñ%Ð'7¸uô%�—	‘	Ö!ð ,1�—	‘	Ô(ÜØ—I‘I×+Ñ+×5Ñ5Ð7GÈuõô Ø—I‘I×+Ñ+×5Ñ5Ð7GÈuõô Ø—I‘I×+Ñ+×3Ñ3Ð5EÈUõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×0Ñ0Ð2BÈ%÷ñu !rF   c                 óB   — | j                   j                  j                  S rÑ   )r   rè   rÙ   rQ   s    rG   rÙ   zModel.layersÿ  s   € à×"Ñ"×(Ñ(×/Ñ/Ð/rF   c                 ó6   — | j                   j                  «       S rÑ   )r   rñ   rQ   s    rG   rñ   zModel.make_cache  s   € Ø×"Ñ"×-Ñ-Ó/Ð/rF   c                 ó.   — | j                   j                  S rÑ   )r   r  rQ   s    rG   r  zModel.quant_predicate  s   € à×"Ñ"×2Ñ2Ð2rF   c                 ó.   — | j                   j                  S rÑ   )r   r  rQ   s    rG   r  zModel.cast_predicate
  s   € à×"Ñ"×1Ñ1Ð1rF   r»   rÑ   )rT   rU   rV   r  ro   r‚   r½   r   rº   r  rH  r  rÙ   rñ   r  r  r¾   r¿   s   @rG   r  r  h  sŠ   ø„ ðS˜Yõ Sð Ø/3ñ	
à—‘ð
ð # 2§8¡8Ñ,ó	
ò7ó tðl ñ0ó ð0ò0ð ñ3ó ð3ð ñ2ó ô2rF   r  ).Údataclassesr   r   Útypingr   r   r   r   r	   Úmlx.coreÚcorer‚   Úmlx.nnrz   Úmlx.nn.layers.distributedr
   r   r   Ú	mlx.utilsr   Úbaser   r   r   r‘   r   r   Úgated_deltar   Ú
qwen3_nextr   rÇ   r   rÎ   r   r‰   r   rÌ   r   ÚModuler]   rÁ   rÔ   ræ   r  r  rE   rF   rG   Ú<module>rX     sÎ   ð÷ )ß 3Õ 3å Ý ß PÑ PÝ ÷ñ ÷
 (Ý +Ý 7Ý +Ý =Ý Að ô:5�Mó :5ó ð:5ôzq�B—I‘Iô qôh�2—9‘9ô ôD (�r—y‘yô  (ôFJ�—	‘	ô JðZ ô)�ó )ó ð)ôd2ˆB�I‰Iõ d2rF   