+
    TV-jDH  ã                   óÜ  € ^ RI HtHt ^ RIHtHtHtHtHt ^ RI	H
t ^ RIHt ^ RIHtHtHt ^ RIHt ^RIHtHtHt ^RIHtHt ^RIHt ^R	IHt ^R
IH t! ^RIH"t# ^RIH$t% ] ! R R]4      4       t& ! R R]PN                  4      t( ! R R]PN                  4      t) ! R R]PN                  4      t* ! R R]PN                  4      t+] ! R R]4      4       t, ! R R]PN                  4      t-R# )é    )Ú	dataclassÚfield)ÚAnyÚDictÚListÚOptionalÚUnionN)Úshard_inplaceÚshard_linearÚsum_gradients)Útree_map)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_mask)ÚArraysCacheÚKVCache)Úgated_delta_update)ÚQwen3NextAttention)ÚQwen3NextMLP)ÚQwen3NextRMSNormGated)ÚQwen3NextSparseMoeBlockc                   ó²   a € ] tR t^t o RtRtRt^ t^ tRt	Rt
^tRt^@t^t^Àt^€t^tRtRtRt^t^ t^ t^t^ t^ tR	t]! R
 R7      tRtRtRt R t!V 3R lt"Rt#V t$R# )ÚTextModelArgsÚ i   i 8  ç�íµ ÷Æ°>i€Q i   FNTc                 ó   € R RR. RORRRR/# )ÚtypeÚdefaultÚmrope_sectionÚ
rope_thetai † Úpartial_rotary_factorç      Ð?)é   r#   é
   © r%   ó    Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/qwen3_5.pyÚ<lambda>ÚTextModelArgs.<lambda>7   s   € Ø�IØš\Ø˜&Ø# Tñ	!
r&   )Údefault_factoryr"   ç     jø@c                óÈ  € V P                   f#   V P                  V P                  ,          V n         V P                  '       d    RV P                  9  d:   RV P                  9   d)   V P                  P	                  R4      V P                  R&   V P                  P                  RR4      V n        V P                  P                  RR4      V n        V P                  V n        R # R # )Nr   Ú	rope_typer!   r"   r    r+   )	Úhead_dimÚhidden_sizeÚnum_attention_headsÚrope_parametersÚpopÚgetr!   r    Úrope_scaling©Úselfs   &r'   Ú__post_init__ÚTextModelArgs.__post_init__D   sº   € Ø�=‰=Ò Ø ×,Ñ,°×0HÑ0HÕHˆDŒMà××Ðà˜d×2Ñ2Ô2Ø 4×#7Ñ#7Ô7à/3×/CÑ/C×/GÑ/GÈÓ/T�×$Ñ$ VÑ,à)-×)=Ñ)=×)AÑ)AØ'¨ó*ˆDÔ&ð #×2Ñ2×6Ñ6°|ÀXÓNˆDŒOØ $× 4Ñ 4ˆDÖñ  r&   c          
      ó
  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[S[ S[S[S[ S[S[S[,          3,          3,          ,          ;R&   S[;R&   S[;R&   S[S[S[ S[S[S[ 3,          3,          ,          ;R&   # )é   Ú
model_typer/   Úintermediate_sizeÚnum_hidden_layersr0   Úrms_norm_epsÚ
vocab_sizeÚnum_key_value_headsÚmax_position_embeddingsÚlinear_num_value_headsÚlinear_num_key_headsÚlinear_key_head_dimÚlinear_value_head_dimÚlinear_conv_kernel_dimÚtie_word_embeddingsÚattention_biasr.   Úfull_attention_intervalÚnum_expertsÚnum_experts_per_tokÚdecoder_sparse_stepÚshared_expert_intermediate_sizeÚmoe_intermediate_sizeÚnorm_topk_probr1   r!   r    r4   )ÚstrÚintÚfloatÚboolr   r   r	   r   )ÚformatÚ__classdict__s   "€r'   Ú__annotate__ÚTextModelArgs.__annotate__   s´  ø‡ ‚ áÑñ ñ Ññ ñ Ñ"ñ	 ñ
 Ññ ñ Ñ!ñ ñ Ññ ñ Ññ ñ Ñ ñ ñ !Ñ)ñ ñ  Ñ$ñ ñ Ñ"ñ ñ Ñ"ñ ñ Ñ$ñ ñ  Ñ#ñ ñ  Ñ%ñ! ñ" Ñ ñ# ñ$ ‘s�mÑ"ñ% ñ& !Ñ$ñ' ñ, Ññ- ñ. Ñ ñ/ ñ0 Ñ ñ1 ñ2 &)Ñ,ñ3 ñ4 Ñ"ñ5 ñ6 Ññ7 ñ< ™d¡3©©e±S¹$ÁÁSÅ	Ð.IÕ(JÐ#JÕKÕLñ ñ= ñP !Ñ'ñQ ñR Ñ ñS ñT ™4¡¡U©5±#¨:Õ%6Ð 6Õ7Õ8Ñ?òU r&   )r.   r!   r4   r    )%Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r;   r/   r<   r=   r0   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   r.   rI   rJ   rK   rL   rM   rN   rO   r   r1   r!   r    r4   r7   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)rU   s   @r'   r   r      sÂ   ø‡ € à€JØ€KØ"ÐØÐØ!ÐØ€LØ€JØ ÐØ#)ÐØ"$ÐØ "ÐØ"ÐØ!$ÐØ"#ÐØ %ÐØ €NØ"€HØ#$Ðð €KØ ÐØ ÐØ+,Ð#Ø!"ÐØ€Nñ PUñ
ôP€Oð $(ÐØ €JØ;?€Lò5÷Y ƒ r&   r   c                   óT   a a€ ] tR t^Vt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGatedDeltaNetc                ó    <€ V ^8„  d   QhRS[ /# )r:   Úconfig©r   )rT   rU   s   "€r'   rV   ÚGatedDeltaNet.__annotate__W   s   ø€ ÷ +#ñ +#™}ñ +#r&   c           	     óf  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n
        V P                  V P                  ,          V n        V P                  V P                  ,          V n        V P                  V P                  ,          ^ 8w  d'   \        RV P                   RV P                   R24      hVP                  V n        VP                   V n        V P                  ^,          V P                  ,           V n        \&        P(                  ! V P$                  V P$                  RV P                  V P$                  ^ R7      V n        \&        P,                  ! V P                  V P                  ^,          V P                  ,           RR7      V n        \&        P,                  ! V P                  V P                  RR7      V n        \&        P,                  ! V P                  V P                  RR7      V n        \&        P,                  ! V P                  V P                  RR7      V n        \6        P8                  ! V P                  4      V n        \6        P<                  P?                  ^ ^V P                  3R7      p\6        P@                  ! V4      V n!        \E        V P                  V P"                  R7      V n#        \&        P,                  ! V P                  V P                  RR7      V n$        R	V n%        R	# )
r   znum_v_heads (z$) must be divisible by num_k_heads (Ú)F)Úin_channelsÚout_channelsÚbiasÚkernel_sizeÚgroupsÚpadding©ri   )ÚlowÚhighÚshape©ÚepsN)&ÚsuperÚ__init__r/   rB   Únum_v_headsrC   Únum_k_headsrD   Ú
head_k_dimrE   Ú
head_v_dimÚkey_dimÚ	value_dimÚ
ValueErrorrF   Úconv_kernel_sizer>   Úlayer_norm_epsilonÚconv_dimÚnnÚConv1dÚconv1dÚLinearÚin_proj_qkvÚ	in_proj_zÚ	in_proj_bÚ	in_proj_aÚmxÚonesÚdt_biasÚrandomÚuniformÚlogÚA_logÚRMSNormGatedÚnormÚout_projÚsharding_group)r6   rb   ÚAÚ	__class__s   && €r'   rt   ÚGatedDeltaNet.__init__W   sK  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ!×8Ñ8ˆÔØ!×6Ñ6ˆÔØ ×4Ñ4ˆŒØ ×6Ñ6ˆŒØ—‘¨×)9Ñ)9Õ9ˆŒØŸ™¨4×+;Ñ+;Õ;ˆŒØ×Ñ˜d×.Ñ.Õ.°!Ô3ÜØ × 0Ñ 0Ð1Ð1UÐVZ×VfÑVfÐUgÐghÐióð ð !'× =Ñ =ˆÔØ"(×"5Ñ"5ˆÔàŸ™ qÕ(¨4¯>©>Õ9ˆŒÜ—i’iØŸ™ØŸ™ØØ×-Ñ-Ø—=‘=Øô
ˆŒô Ÿ9š9Ø×Ñ˜dŸl™l¨QÕ.°·±Õ?Àeô
ˆÔô Ÿš 4×#3Ñ#3°T·^±^È%ÔPˆŒÜŸš 4×#3Ñ#3°T×5EÑ5EÈEÔRˆŒÜŸš 4×#3Ñ#3°T×5EÑ5EÈEÔRˆŒä—w’w˜t×/Ñ/Ó0ˆŒä�I‰I×Ñ !¨"°T×5EÑ5EÐ4GÐÓHˆÜ—V’V˜A“YˆŒ
ä  §¡°d×6MÑ6MÔNˆŒ	äŸ	š	 $§.¡.°$×2BÑ2BÈÔOˆŒà"ˆÖr&   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r:   ÚinputsÚmaskÚcacheÚreturn©r‡   Úarrayr   r   )rT   rU   s   "€r'   rV   rd   „   sN   ø€ ÷ Jñ Já—‘ðJñ ‘r—x‘xÕ ðJñ ™�}ð	Jñ
 
�‰ñJr&   c                ó  € VP                   w  rEpV P                  e   \        V P                  4      ! V4      pV P                  V4      pV P	                  V4      P                  WEV P                  V P                  4      pV P                  V4      p	V P                  V4      p
Ve   V^ ,          e   V^ ,          pM?\        P                  ! W@P                  ^,
          V P                  3VP                  R7      pVe    \        P                  ! VR,          V^ 4      p\        P                   ! W·.^R7      pVe±   V P                  ^,
          pVP"                  ej   \        P$                  ! VP"                  ^ V4      pVR,          \        P&                  ! V4      ,           R,          p\        P(                  ! WÏ^R7      V^ &   M&\        P*                  ! VRV) R 1R3,          4      V^ &   \,        P.                  ! V P1                  V4      4      p\3        \        P4                  ! VV P6                  ^V P6                  ,          .R	4      V P8                  V P8                  V P                  .V P:                  V P:                  V P                  .4       UUUu. uF  w  pppVP                  WEVV4      NK  	  upppw  pppV'       d
   V^,          MR pVP                   R	,          R
,          pV^,          \        P<                  P?                  VR R4      ,          pV\        P<                  P?                  VR R4      ,          p\A        VVVV
V	V PB                  V PD                  VVV PF                  '       * R7
      w  ppVe   VV^&   VPI                  V4       V PK                  VV4      pV PM                  VP                  WER	4      4      pV P                  e,   \        PN                  PQ                  VV P                  R7      pV# u upppi )N)Údtype©ÚaxisºNNNr   )Ú
use_kernel©Úgroup).N)r    Néÿÿÿÿg      à¿))rp   r‘   r   rƒ   r„   Úreshaperu   rx   r…   r†   r‡   Úzerosr|   r~   r�   ÚwhereÚconcatenateÚlengthsÚclipÚarangeÚtake_along_axisÚ
contiguousr   Úsilur�   ÚzipÚsplitry   rv   rw   ÚfastÚrms_normr   r�   r‰   ÚtrainingÚadvancer�   r�   ÚdistributedÚall_sum)r6   r–   r—   r˜   ÚBÚSÚ_ÚqkvÚzÚbÚaÚ
conv_stateÚ
conv_inputÚn_keepÚendsÚ	positionsÚconv_outÚtÚhÚdÚqÚkÚvÚstateÚ	inv_scaleÚouts   &&&&                      r'   Ú__call__ÚGatedDeltaNet.__call__„   sE  € ð —,‘,‰ˆˆaà×ÑÒ*Ü" 4×#6Ñ#6Ô7¸Ó?ˆFà×Ñ˜vÓ&ˆØ�N‰N˜6Ó"×*Ñ*¨1°×1AÑ1AÀ4Ç?Á?ÓSˆØ�N‰N˜6Ó"ˆØ�N‰N˜6Ó"ˆàÒ  q¥Ò!5Ø˜q�‰JäŸšØ×)Ñ)¨AÕ-¨t¯}©}Ð=Ø—l‘lôˆJð
 ÒÜ—(’(˜4 	�?¨C°Ó3ˆCÜ—^’^ ZÐ$5¸AÔ>ˆ
ØÒØ×*Ñ*¨QÕ.ˆFØ�}‰}Ò(Ü—w’w˜uŸ}™}¨a°Ó3�Ø! '�]¬R¯YªY°vÓ->Õ>À	ÕJ�	Ü×-Ò-¨jÈ!ÔL��a’äŸ=š=¨°A¸°w±xÀ°NÕ)CÓD��a‘Ü—7’7˜4Ÿ;™; zÓ2Ó3ˆô Ü—’˜ D§L¡L°!°d·l±lÕ2BÐ#CÀRÓHØ×!Ñ! 4×#3Ñ#3°T×5EÑ5EÐFØ—‘ $§/¡/°4·?±?ÐCôõ
ñ‘��1�að �I‰I�a˜A˜qÖ!ñó
‰ˆˆ1ˆa÷ "��a– tˆØ—G‘G˜B•K 4Õ'ˆ	Ø˜�\œRŸW™W×-Ñ-¨a°°tÓ<Õ<ˆØœŸ™×(Ñ(¨¨D°$Ó7Õ7ˆä'ØØØØØØ�J‰JØ�L‰LØØØŸ=™=Ô(ô
‰
ˆˆUð ÒØˆE�!‰HØ�M‰M˜!Ôà�i‰i˜˜QÓˆØ�m‰m˜CŸK™K¨¨bÓ1Ó2ˆà×ÑÒ*Ü—.‘.×(Ñ(¨°D×4GÑ4GÐ(ÓHˆCàˆ
ùôK
s   Ê P)r�   r�   r~   r|   r‰   rw   rx   r/   r†   r…   rƒ   r„   ry   r}   r�   rv   ru   r�   r‘   rz   ©NN©	rX   rY   rZ   r[   rt   rÍ   r]   r^   Ú__classcell__©r“   rU   s   @@r'   r`   r`   V   s"   ù‡ € ÷+#ó +#÷ZJ÷ Jò Jr&   r`   c                   óT   a a€ ] tR t^Ñt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r:   ÚargsÚ	layer_idx)r   rQ   )rT   rU   s   "€r'   rV   ÚDecoderLayer.__annotate__Ò   s   ø€ ÷ Eñ E™]ð E±sñ Er&   c                ó"  <€ \         SV `  4        V^,           VP                  ,          ^ 8g  V n        V P                  '       d   \	        V4      V n        M\        V4      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        VP                  ^ 8”  d   \        V4      V n        R# \#        VP                  VP$                  4      V n        R# )é   rq   N)rs   rt   rI   Ú	is_linearr`   Úlinear_attnÚ	AttentionÚ	self_attnr   ÚRMSNormr/   r>   Úinput_layernormÚpost_attention_layernormrJ   ÚSparseMoeBlockÚmlpÚMLPr<   )r6   rÖ   r×   r“   s   &&&€r'   rt   ÚDecoderLayer.__init__Ò   sÁ   ø€ Ü‰ÑÔØ# a�-¨4×+GÑ+GÕGÈ1ÑLˆŒØ�>�>ˆ>Ü,¨TÓ2ˆDÕä& t›_ˆDŒNä!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð ×Ñ˜aÔÜ% dÓ+ˆDŽHä˜4×+Ñ+¨T×-CÑ-CÓDˆDŽHr&   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r:   Úxr—   r˜   r™   rš   )rT   rU   s   "€r'   rV   rØ   ä   sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr&   c                ó  € V P                   '       d#   V P                  V P                  V4      W#4      pM!V P                  V P                  V4      W#4      pW,           pWPP	                  V P                  V4      4      ,           pV# ©N)rÛ   rÜ   rà   rÞ   rã   rá   )r6   rç   r—   r˜   ÚrrÅ   rÌ   s   &&&&   r'   rÍ   ÚDecoderLayer.__call__ä   sm   € ð �>�>ˆ>Ø× Ñ  ×!5Ñ!5°aÓ!8¸$ÓF‰Aà—‘˜t×3Ñ3°AÓ6¸ÓDˆAØ�EˆØ—(‘(˜4×8Ñ8¸Ó;Ó<Õ<ˆØˆ
r&   )rà   rÛ   rÜ   rã   rá   rÞ   rÏ   rÐ   rÒ   s   @@r'   rÔ   rÔ   Ñ   s    ù‡ € ÷Eó E÷$÷ ò r&   rÔ   c                   óT   a a€ ] tR t^ót oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚQwen3_5TextModelc                ó    <€ V ^8„  d   QhRS[ /# ©r:   rÖ   rc   )rT   rU   s   "€r'   rV   ÚQwen3_5TextModel.__annotate__ô   s   ø€ ÷ 7ñ 7™]ñ 7r&   c                ó–  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        WR 7      NK  	  upV n
        \        P                  ! VP
                  VP                  R7      V n        ^ V n        VP                  ^,
          V n        R# u upi ))rÖ   r×   rq   N)rs   rt   r   Ú	Embeddingr?   r/   Úembed_tokensÚranger=   rÔ   Úlayersrß   r>   r�   Ússm_idxrI   Úfa_idx)r6   rÖ   Úir“   s   && €r'   rt   ÚQwen3_5TextModel.__init__ô   s•   ø€ Ü‰ÑÔÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔä:?À×@VÑ@VÔ:Wó
Ù:W°QŒL˜d×0Ñ:Wñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	ØˆŒØ×2Ñ2°QÕ6ˆŽùò
s   ÁCc                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[ P                  ,          RS[ P                  /# ©r:   r–   r˜   Úinput_embeddingsr™   rš   )rT   rU   s   "€r'   rV   rð   þ   sH   ø€ ÷ (ñ (á—‘ð(ñ ™�}ð(ñ #¡2§8¡8Õ,ð	(ñ
 
�‰ñ(r&   c                ó„  € Ve   TpMV P                  V4      pVf   R .\        V P                  4      ,          p\        WBV P                  ,          4      p\        WBV P                  ,          4      p\        V P                  V4       F%  w  rxVP                  '       d   TMTp	V! WIVR7      pK'  	  V P                  V4      # )N)r—   r˜   )
ró   Úlenrõ   r   r÷   r   rö   r¯   rÛ   r�   )
r6   r–   r˜   rü   Úhidden_statesÚfa_maskÚssm_maskÚlayerÚcr—   s
   &&&&      r'   rÍ   ÚQwen3_5TextModel.__call__þ   s¢   € ð Ò'Ø,‰Mà ×-Ñ-¨fÓ5ˆMàŠ=Ø�FœS §¡Ó-Õ-ˆEä'¨¸T¿[¹[Õ7IÓJˆÜ" =¸¿¹Õ2EÓFˆä˜DŸK™K¨Ö/‰HˆEØ$ŸŸ˜‘8°GˆDÙ! -À!ÔDŠMñ 0ð �y‰y˜Ó'Ð'r&   )ró   r÷   rõ   r�   rö   rÏ   rÐ   rÒ   s   @@r'   rí   rí   ó   s   ù‡ € ÷7ó 7÷(÷ (ò (r&   rí   c                   ó�   a a€ ] tR tRt oV3R lV 3R lltRV3R lR llt]R 4       tR tR t	]R	 4       t
]R
 4       tRtVtV ;t# )Ú	TextModeli  c                ó    <€ V ^8„  d   QhRS[ /# rï   rc   )rT   rU   s   "€r'   rV   ÚTextModel.__annotate__  s   ø€ ÷ Tñ T™]ñ Tr&   c                óþ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# )Frm   N)rs   rt   rÖ   r;   rí   ÚmodelrG   r   r‚   r/   r?   Úlm_head©r6   rÖ   r“   s   &&€r'   rt   ÚTextModel.__init__  sY   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ% dÓ+ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r&   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[ P                  ,          RS[ P                  /# rû   rš   )rT   rU   s   "€r'   rV   r    sH   ø€ ÷ ñ á—‘ðñ ™�}ðñ #¡2§8¡8Õ,ð	ñ
 
�‰ñr&   c                óÔ   € V P                  WVR 7      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# ))rü   )r
  rÖ   rG   ró   Ú	as_linearr  )r6   r–   r˜   rü   rÌ   s   &&&& r'   rÍ   ÚTextModel.__call__  s]   € ð �j‰j˜Ð9IˆjÓJˆØ�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r&   c                ó.   € V P                   P                  # ré   )r
  rõ   r5   s   &r'   rõ   ÚTextModel.layers,  s   € à�z‰z× Ñ Ð r&   c                óŽ   € V P                    Uu. uF*  qP                  '       d   \        ^R7      M	\        4       NK,  	  up# u upi )r:   )Úsize)rõ   rÛ   r   r   )r6   Úls   & r'   Ú
make_cacheÚTextModel.make_cache0  s0   € ØKOÏ;Ê;ÓWÉ;Àa§{§{ {” Õ#¼»	ÒAÉ;ÑWÐWùÒWs   �0Ac                ó^  a€ \         ;QJ d    R  V 4       F  '       g   K   RM	  RM! R  V 4       4      p\         ;QJ d*    R VP                  4        4       F  '       g   K   RM	  RM! R VP                  4        4       4      pT;'       g    TpVP                  4        UUu/ uF  w  rVRV9  g   K  WVbK  	  pppV P                  P                  '       d   VP	                  RR4       R
pVP                  4        F¥  w  opRS9   d.   VP
                  R,          ^8w  d   VP                  ^^4      VS&   V'       g   KD  \         ;QJ d    V3R lV 4       F  '       g   K   RM	  RM! V3R lV 4       4      '       g   K†  VP                  ^8X  g   K™  VR	,           VS&   K§  	  V# u uppi )c              3   ó,   "  € T F
  pR V9   x € K  	  R# 5i)úmtp.Nr%   )Ú.0rÈ   s   & r'   Ú	<genexpr>Ú%TextModel.sanitize.<locals>.<genexpr>4  s   é € Ð;±7¨a˜f¨žk³7ùs   ‚TFc              3   ól   "  € T F*  w  rR V9   ;'       d    VP                   R,          ^8g  x € K,  	  R# 5i)úconv1d.weightNr¤   )rp   )r  rÈ   rÉ   s   &  r'   r  r  5  s2   é € ð %
ÙBQ¹$¸!ˆO˜qÑ ×5Ð5 Q§W¡W¨R¥[°AÑ%5Ô5Ã/ùs   ‚4•4r  zlm_head.weightNr   c              3   óF   <"  € T F  pSP                  V4      x € K  	  R # 5iré   ©Úendswith)r  ÚsfxrÈ   s   & €r'   r  r  H  s   øé € Ð0VÉIÀS°·±¸C·°ËIùs   ƒ!g      ð?)z.input_layernorm.weightz .post_attention_layernorm.weightzmodel.norm.weightz.q_norm.weightz.k_norm.weightr¤   )ÚanyÚitemsrÖ   rG   r2   rp   ÚmoveaxisÚndim)r6   ÚweightsÚhas_mtp_weightsÚhas_unsanitized_conv1dÚshould_shift_norm_weightsrÈ   rÉ   Ú	norm_keyss   &&   `  r'   ÚsanitizeÚTextModel.sanitize3  sE  ø€ ß›#Ñ;±7Ó;Ÿ#Ÿ#š#Ñ;±7Ó;Ó;ˆß!$£ñ %
ØBIÇ-Á-Ä/ó%
§§¢ñ %
ØBIÇ-Á-Ä/ó%
ó "
Ðð %4×$MÐ$MÐ7MÐ!Ø$+§M¡M¤OÔG¡O™D˜A°vÀQ±”4�1’4¡OˆÑGà�9‰9×(×(Ð(Ø�K‰KÐ(¨$Ô/ð
ˆ	ð —M‘M–O‰DˆAˆqØ !Ô#¨¯©°­°qÔ(8ØŸZ™Z¨¨1Ó-�˜‘
ß(Ñ(¯S«SÔ0VÉIÓ0V¯S¯SªSÔ0VÉIÓ0V×-VÔ-VØ—6‘6˜Q–;Ø!" S¥�G˜A“Jñ $ð ˆùó% Hs   Â%F)Â5F)c                óF   € V P                   P                  ^ 8:  d   R# R pV# )r   Nc                 ón   € V P                  R 4      '       g   V P                  R4      '       d   R^@R^/# R# )zmlp.gateÚshared_expert_gateÚ
group_sizeÚbitsTr"  )Úpathr¹   s   &&r'   Ú	predicateÚ,TextModel.quant_predicate.<locals>.predicateR  s2   € Ø�}‰}˜Z×(Ò(¨D¯M©MÐ:N×,OÒ,OØ$ b¨&°!Ð4Ð4Ùr&   )rÖ   rJ   ©r6   r6  s   & r'   Úquant_predicateÚTextModel.quant_predicateM  s$   € à�9‰9× Ñ  AÔ%Ùò	ð
 Ðr&   c                ó   € R  R lpV# )c                ó$   € V ^8„  d   QhR\         /# )r:   r5  )rP   )rT   s   "r'   rV   Ú.TextModel.cast_predicate.<locals>.__annotate__[  s   € ÷ 	ñ 	œCñ 	r&   c                 ó8   € V P                  R 4      '       d   R# R# )r�   FTr"  )r5  s   &r'   r6  Ú+TextModel.cast_predicate.<locals>.predicate[  s   € Ø�}‰}˜W×%Ò%ÙÙr&   r%   r8  s   & r'   Úcast_predicateÚTextModel.cast_predicateY  s   € õ	ð
 Ðr&   )rÖ   r  r
  r;   rÏ   )rX   rY   rZ   r[   rt   rÍ   Úpropertyrõ   r  r.  r9  r@  r]   r^   rÑ   rÒ   s   @@r'   r  r    sg   ù‡ € ÷Tó T÷ò ð ñ!ó ð!òXòð4 ñ	ó ð	ð ñó ÷ð r&   r  c                   óH   a a€ ] tR tRt o]V 3R l4       tV3R ltRtVtV ;t	# )Ú	ModelArgsic  c                óR   <€ R V9  d   V ! VR,          VR7      # \         SV `  V4      # )Útext_configr;   )r;   rF  )rs   Ú	from_dict)ÚclsÚparamsr“   s   &&€r'   rG  ÚModelArgs.from_dicth  s-   ø€ à Ô&Ù &¨Õ"6ÀFÔKÐKÜ‰wÑ  Ó(Ð(r&   c                ó2   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   # )r:   r;   rF  )rP   Údict)rT   rU   s   "€r'   rV   ÚModelArgs.__annotate__c  s   ø‡ ‚ á�Oñ ñ Ñò r&   r%   )
rX   rY   rZ   r[   ÚclassmethodrG  r\   r]   r^   rÑ   rÒ   s   @@r'   rD  rD  c  s    ù‡ € ð
 ô)ó ð)÷ … r&   rD  c                   óš   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR tRR lt]R 4       t	R	 t
]R
 4       t]R 4       tRtVtV ;t# )ÚModelio  c                ó    <€ V ^8„  d   QhRS[ /# rï   )rD  )rT   rU   s   "€r'   rV   ÚModel.__annotate__p  s   ø€ ÷ Sñ S™Yñ Sr&   c                ó¬   <€ \         SV `  4        Wn        VP                  V n        \	        \
        P                  VP                  4      4      V n        R # ré   )	rs   rt   rÖ   r;   r  r   rG  rF  Úlanguage_modelr  s   &&€r'   rt   ÚModel.__init__p  s<   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ'¬×(?Ñ(?À×@PÑ@PÓ(QÓRˆÖr&   c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# )r:   r–   rü   )r‡   r›   r   )rT   rU   s   "€r'   rV   rR  v  s,   ø€ ÷ 
ñ 
á—‘ð
ñ #¡2§8¡8Õ,ñ	
r&   c                ó(   € V P                  WVR 7      # ))r˜   rü   )rT  )r6   r–   r˜   rü   s   &&&&r'   rÍ   ÚModel.__call__v  s"   € ð ×"Ñ"ØÐ2Bð #ó 
ð 	
r&   c                óž  € / pVP                  4        F�  w  r4VP                  R 4      '       g   VP                  R4      '       d   K5  VP                  R4      '       d   KN  VP                  R4      '       d   VP                  RR4      pM!VP                  R4      '       d   M	RV,           pWBV&   KŸ  	  V P                  P	                  V4      # )Úvision_towerzmodel.visualzmodel.language_modelzlanguage_model.modelzlanguage_model.)r&  Ú
startswithÚreplacerT  r.  )r6   r)  Ú	sanitizedÚkeyÚvalues   &&   r'   r.  ÚModel.sanitize€  sª   € Øˆ	Ø!Ÿ-™-ž/‰JˆCØ�~‰~˜n×-Ò-°·±À×1OÒ1OÙØ�~‰~˜n×-Ò-ÙØ�~‰~Ð4×5Ò5Ø—k‘kÐ"8Ð:PÓQ‘Ø—‘Ð 1×2Ò2Øà'¨#Õ-�Ø"�c‹Nñ *ð ×"Ñ"×+Ñ+¨IÓ6Ð6r&   c                ó¨  a€ T;'       g    \         P                  P                  4       pVP                  4       oVP	                  4       pR  pV3R lpV P
                   EFî  pVP                  '       Ed‹   VP                  P                  pWP                  n	        \        VP                  P                  V! V4      VR7       VP                  P                  ;P                  S,          un        \        VP                  P                  RV^V,          .VR7       \        VP                  P                  RVR7       \        VP                  P                  RVR7       \        VP                  P                   RVR7       \         P"                  ! \         P$                  ! VP                  P&                  S4      V,          4      VP                  n        \         P"                  ! \         P$                  ! VP                  P(                  S4      V,          4      VP                  n        \        VP                  P*                  RVR7       VP                  ;P,                  S,          un        VP                  ;P.                  S,          un        VP                  ;P                  S,          un        VP                  ;P0                  S,          un        VP                  ;P2                  S,          un        EM\5        VP6                  P8                  RVR7      VP6                  n        \5        VP6                  P:                  RVR7      VP6                  n        V! VP6                  P<                  VP6                  P>                  4       V! VP6                  P@                  VP6                  P>                  4       \5        VP6                  P<                  RVR7      VP6                  n        \5        VP6                  P@                  RVR7      VP6                  n         VP6                  ;PB                  S,          un!        \E        ^VP6                  P>                  S,          4      VP6                  n        \G        VPH                  \J        4      '       d—   \5        VPH                  PL                  RVR7      VPH                  n&        \5        VPH                  PN                  RVR7      VPH                  n'        \5        VPH                  PP                  RVR7      VPH                  n(        EKÖ  WPH                  n	        \        VPH                  PR                  PL                  RVR7       \        VPH                  PR                  PN                  RVR7       \        VPH                  PR                  PP                  RVR7       \        VPH                  PT                  PL                  RVR7       \        VPH                  PT                  PN                  RVR7       \        VPH                  PT                  PP                  RVR7       EKñ  	  R# )c                 ó   a € V 3R  l# )c                 ó   <€ ^ S^S,          .3# )r   r%   )ÚpÚwry   s   &&€r'   r(   Ú4Model.shard.<locals>.conv_sharding.<locals>.<lambda>—  s   ø€   W¨a°'­kÐ$:Ñ ;r&   r%   )ry   s   fr'   Úconv_shardingÚ"Model.shard.<locals>.conv_sharding–  s	   ø€ Ü;Ð;r&   c                 óz   <a€ SS8:  d   R # VV3R lpV P                  \        W P                  4       4      4       R # )Nc                 óê   <€ V P                   pV P                  ! SV^ ,          S,          .VR,          O5!  p \        P                  ! V SS,          ^ R7      p V P                  ! R.VR,          O5!  p V # )r   :rÚ   NNrž   r¤   )rp   r¥   r‡   Úrepeat)rd  ÚsÚNrÅ   s   & €€r'   Ú_repeatÚ=Model.shard.<locals>.repeat_kv_layer_inplace.<locals>._repeatŸ  s_   ø€ Ø—G‘G�Ø—I’I˜a  1¥¨¥Ð3¨Q¨r­UÓ3�Ü—I’I˜a  a¥¨aÔ0�Ø—I’I˜bÐ) 1 R¥5Ó)�Ø�r&   )Úupdater   Ú
parameters)r  rÅ   rn  rm  s   &f €r'   Úrepeat_kv_layer_inplaceÚ,Model.shard.<locals>.repeat_kv_layer_inplace™  s.   ù€ à�AŒvÙöð �L‰Lœ '×+;Ñ+;Ó+=Ó>Ö?r&   r¢   zall-to-sharded)Úsegmentsr£   zsharded-to-allN)+r‡   rµ   Úinitr  Úrankrõ   rÛ   rÜ   ry   r‘   r
   r�   rk   rƒ   r„   r…   r†   r­   r°   r‰   r�   r�   rv   ru   rz   r~   r   rÞ   Úo_projÚq_projÚk_projr@   Úv_projr0   ÚmaxÚ
isinstancerã   rä   Ú	gate_projÚ	down_projÚup_projÚshared_expertÚ
switch_mlp)r6   r£   rv  rg  rr  r  Úkdrm  s   &&     @r'   ÚshardÚModel.shard�  s¢  ø€ Ø×.Ð.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ�z‰z‹|ˆò	<õ	@ð —[•[ˆEà��‰Ø×&Ñ&×.Ñ.�Ø38×!Ñ!Ô0Ü˜e×/Ñ/×6Ñ6¹ÀbÓ8IÐQVÕWØ×!Ñ!×(Ñ(×/Ò/°AÕ5Õ/ÜØ×%Ñ%×1Ñ1Ø$Ø  ! b¥&˜\Øõ	ô Ø×%Ñ%×/Ñ/Ð1AÈõô Ø×%Ñ%×/Ñ/Ð1AÈõô Ø×%Ñ%×/Ñ/Ð1AÈõô -/¯MªMÜ—H’H˜U×.Ñ.×6Ñ6¸Ó:¸4Õ@ó-�×!Ñ!Ô)ô +-¯-ª-Ü—H’H˜U×.Ñ.×4Ñ4°aÓ8¸Õ>ó+�×!Ñ!Ô'ô ˜e×/Ñ/×8Ñ8Ð:JÐRWÕXØ×!Ñ!×-Ò-°!Õ3Õ-Ø×!Ñ!×-Ò-°!Õ3Õ-Ø×!Ñ!×)Ò)¨aÕ/Õ)Ø×!Ñ!×+Ò+°Õ1Õ+Ø×!Ñ!×*Ò*¨qÕ0×*ô *6Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Ô&ô *6Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Ô&ñ (Ø—O‘O×*Ñ*¨E¯O©O×,OÑ,Oôñ (Ø—O‘O×*Ñ*¨E¯O©O×,OÑ,Oôô *6Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Ô&ô *6Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Ô&ð —‘×3Ò3¸Õ9Õ3Ü69Ø�u—‘×:Ñ:¸aÕ?ó7�—‘Ô3ô
 ˜%Ÿ)™)¤S×)Ò)Ü&2Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô '3Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô %1Ø—I‘I×%Ñ%Ð'7¸uô%�—	‘	×!ð ,1—	‘	Ô(ÜØ—I‘I×+Ñ+×5Ñ5Ð7GÈuõô Ø—I‘I×+Ñ+×5Ñ5Ð7GÈuõô Ø—I‘I×+Ñ+×3Ñ3Ð5EÈUõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×0Ñ0Ð2BÈ%÷ð óu !r&   c                óB   € V P                   P                  P                  # ré   )rT  r
  rõ   r5   s   &r'   rõ   ÚModel.layers  s   € à×"Ñ"×(Ñ(×/Ñ/Ð/r&   c                ó6   € V P                   P                  4       # ré   )rT  r  r5   s   &r'   r  ÚModel.make_cache
  s   € Ø×"Ñ"×-Ñ-Ó/Ð/r&   c                ó.   € V P                   P                  # ré   )rT  r9  r5   s   &r'   r9  ÚModel.quant_predicate  s   € à×"Ñ"×2Ñ2Ð2r&   c                ó.   € V P                   P                  # ré   )rT  r@  r5   s   &r'   r@  ÚModel.cast_predicate  s   € à×"Ñ"×1Ñ1Ð1r&   )rÖ   rT  r;   rÏ   ré   )rX   rY   rZ   r[   rt   rÍ   r.  rƒ  rB  rõ   r  r9  r@  r]   r^   rÑ   rÒ   s   @@r'   rP  rP  o  sm   ù‡ € ÷Só S÷
ò 
ò7ô tðl ñ0ó ð0ò0ð ñ3ó ð3ð ñ2ó ÷2ð 2r&   rP  ).Údataclassesr   r   Útypingr   r   r   r   r	   Úmlx.coreÚcorer‡   Úmlx.nnr   Úmlx.nn.layers.distributedr
   r   r   Ú	mlx.utilsr   Úbaser   r   r   r˜   r   r   Úgated_deltar   Ú
qwen3_nextr   rÝ   r   rä   r   rŽ   r   râ   r   ÚModuler`   rÔ   rí   r  rD  rP  r%   r&   r'   Ú<module>r˜     sÎ   ð÷ )ß 3Õ 3å Ý ß PÑ PÝ ÷ñ ÷
 (Ý +Ý 7Ý +Ý =Ý Að ô:5�Mó :5ó ð:5ôzx�B—I‘Iô xôv�2—9‘9ô ôD (�r—y‘yô  (ôFJ�—	‘	ô JðZ ô)�ó )ó ð)ôd2ˆB�I‰Iö d2r&   