+
    TV-j3  ã                   ó   € ^ RI t ^ RIHt ^ RIHt ^ RIHt ^RIH	t	 R t
RR lt ! R R]P                  4      t ! R R	]P                  4      t ! R
 R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tR# )é    N)Úpartial©Úswigluc                 óæ   € VP                   Ev r#VP                  4       p\        P                  ! V4      p\        P                  ! V4      pV P                  ^ R4      WC,          ,          W,          V3# )r   éýÿÿÿ)ÚshapeÚflattenÚmxÚargsort)ÚxÚindicesÚ_ÚMÚorderÚ	inv_orders   &&    Úl/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/switch_layers.pyÚ_gather_sortr      sV   € Ø�M‰M�E€QØ�o‰oÓ€GÜ�JŠJ�wÓ€EÜ—
’
˜5Ó!€IØ�9‰9�Q˜Ó˜E�JÕ'¨­¸ÐBÐBó    c                 óN   € W,          p Ve   \         P                  ! V ^ V4      p V # ©N)r
   Ú	unflatten)r   r   r   s   &&&r   Ú_scatter_unsortr      s&   € Ø	�€AØÒÜ�LŠL˜˜A˜uÓ%ˆØ€Hr   c                   ó|   a a€ ] tR t^t oRV3R lV 3R lllt]R 4       t]R 4       t]R 4       tR	R lt	Rt
VtV ;t# )
ÚQuantizedSwitchLinearc                óD   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[ RS[ RS[/# )é   Ú
input_dimsÚoutput_dimsÚnum_expertsÚbiasÚ
group_sizeÚbitsÚmode)ÚintÚboolÚstr)ÚformatÚ__classdict__s   "€r   Ú__annotate__Ú"QuantizedSwitchLinear.__annotate__   sQ   ø€ ÷ !ñ !áð!ñ ð!ñ ð	!ñ
 ð!ñ ð!ñ ð!ñ ñ!r   c           	     ó®  <€ \         S
V `  4        \        P                  ! ^V,          4      p\        P
                  ! \        P                  P                  V) VW2V3R7      VVVR7      vV n        V n	        p	V	'       d
   V	^ ,          MRV n
        V'       d   \        P                  ! W234      V n        WPn        W`n        Wpn        V P!                  4        R# )é   ©ÚlowÚhighr   )r!   r"   r#   N)ÚsuperÚ__init__ÚmathÚsqrtr
   ÚquantizeÚrandomÚuniformÚweightÚscalesÚbiasesÚzerosr    r!   r"   r#   Úfreeze)Úselfr   r   r   r    r!   r"   r#   Úscaler9   Ú	__class__s   &&&&&&&&  €r   r1   ÚQuantizedSwitchLinear.__init__   s®   ø€ ô 	‰ÑÔä—	’	˜!˜j�.Ó)ˆÜ,.¯KªKÜ�I‰I×ÑØ�FØØ"°Ð<ð ó ð
 "ØØô	-
Ð)ˆŒ�T”[ 6÷ $*�f˜Q–i¨tˆŒçÜŸš +Ð!;Ó<ˆDŒIà$ŒØŒ	ØŒ	ð 	�‰Žr   c                ó^   € V P                   P                  ^,          V P                  ,          # ©r   )r8   r   r!   ©r<   s   &r   r   Ú QuantizedSwitchLinear.input_dims?   s    € à�{‰{× Ñ  Õ# d§o¡oÕ5Ð5r   c                ó<   € V P                   P                  ^,          # ©r,   ©r7   r   rB   s   &r   r   Ú!QuantizedSwitchLinear.output_dimsC   ó   € à�{‰{× Ñ  Õ#Ð#r   c                ó<   € V P                   P                  ^ ,          # ©r   rF   rB   s   &r   r   Ú!QuantizedSwitchLinear.num_expertsG   rH   r   c                ó"  € \         P                  ! VV R ,          V R,          V P                  R4      VRV P                  V P                  V P
                  VR7
      pRV 9   d-   V\         P                  ! V R,          V,          R4      ,           pV# )r7   r8   r9   T)Úrhs_indicesÚ	transposer!   r"   r#   Úsorted_indicesr    éþÿÿÿ)r
   Ú
gather_qmmÚgetr!   r"   r#   Úexpand_dims©r<   r   r   rO   s   &&&&r   Ú__call__ÚQuantizedSwitchLinear.__call__K   sw   € Ü�MŠMØØ��NØ��NØ�H‰H�XÓØØØ—‘Ø—‘Ø—‘Ø)ô
ˆð �TŒ>Ø”B—N’N 4¨¥<°Õ#8¸"Ó=Õ=ˆAØˆr   )r    r9   r"   r!   r#   r8   r7   )Té@   é   Úaffine©F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r1   Úpropertyr   r   r   rU   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r>   r(   s   @@r   r   r      sV   ù‡ € ÷!õ !ðF ñ6ó ð6ð ñ$ó ð$ð ñ$ó ð$÷ô r   r   c                   ó’   a a€ ] tR t^]t oR
V3R lV 3R lllt]R 4       t]R 4       t]R 4       tRR lt	RV3R lR llt
R	tVtV ;t# )ÚSwitchLinearc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )r   r   r   r   r    ©r$   r%   )r'   r(   s   "€r   r)   ÚSwitchLinear.__annotate__^   s-   ø€ ÷ =ñ =Ùð=Ù,/ð=Ù>Að=ÙIMñ=r   c                óþ   <€ \         SV `  4        \        P                  ! ^V,          4      p\        P
                  P                  V) VW2V3R7      V n        V'       d   \        P                  ! W234      V n	        R# R# )r,   r-   N)
r0   r1   r2   r3   r
   r5   r6   r7   r:   r    )r<   r   r   r   r    r=   r>   s   &&&&& €r   r1   ÚSwitchLinear.__init__^   sj   ø€ ô 	‰ÑÔÜ—	’	˜!˜j�.Ó)ˆÜ—i‘i×'Ñ'Ø�ØØ¨ZÐ8ð (ó 
ˆŒ÷ ÜŸš +Ð!;Ó<ˆDŽIñ r   c                ó<   € V P                   P                  ^,          # rA   rF   rB   s   &r   r   ÚSwitchLinear.input_dimsl   rH   r   c                ó<   € V P                   P                  ^,          # rE   rF   rB   s   &r   r   ÚSwitchLinear.output_dimsp   rH   r   c                ó<   € V P                   P                  ^ ,          # rJ   rF   rB   s   &r   r   ÚSwitchLinear.num_expertst   rH   r   c                óÎ   € \         P                  ! VV R ,          P                  RR4      VVR7      pRV 9   d-   V\         P                  ! V R,          V,          R4      ,           pV# )r7   )rM   rO   r    éÿÿÿÿrP   )r
   Ú	gather_mmÚswapaxesrS   rT   s   &&&&r   rU   ÚSwitchLinear.__call__x   sY   € Ü�LŠLØØ��N×#Ñ# B¨Ó+ØØ)ô	
ˆð �TŒ>Ø”B—N’N 4¨¥<°Õ#8¸"Ó=Õ=ˆAØˆr   c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )r   r!   r"   r#   )r$   r&   )r'   r(   s   "€r   r)   rh   ƒ   s"   ø€ ÷ ñ ¡sð ±sð Ácñ r   c           
     ó  € V P                   P                  w  rEp\        VVVR VVVR7      p\        P                  ! V P                   WVR7      vVn         Vn        pV'       d
   V^ ,          MRVn        RV 9   d   V P                  Vn        V# )F)r#   Nr    )r7   r   r   r
   r4   r8   r9   r    )	r<   r!   r"   r#   r   r   r   Úqlr9   s	   &&&&     r   Úto_quantizedÚSwitchLinear.to_quantizedƒ   sƒ   € Ø/3¯{©{×/@Ñ/@Ñ,ˆ *Ü"ØØØØØØØô
ˆô )+¯ªØ�K‰K˜°ô)
Ð%ˆŒ	�2”9˜v÷ "(�F˜1–I¨TˆŒ	à�TŒ>Ø—i‘iˆBŒGØˆ	r   )r    r7   )TrZ   )rW   rX   rY   )r[   r\   r]   r^   r1   r_   r   r   r   rU   ry   r`   ra   rb   rc   s   @@r   re   re   ]   s_   ù‡ € ÷=õ =ð ñ$ó ð$ð ñ$ó ð$ð ñ$ó ð$ô	÷÷ ò r   re   c                   ó8   a a€ ] tR t^˜t oV 3R ltR tRtVtV ;t# )ÚSwiGLUc                ó$   <€ \         SV `  4        R # r   )r0   r1   )r<   r>   s   &€r   r1   ÚSwiGLU.__init__™   s   ø€ Ü‰ÑÖr   c                ó   € \        W!4      # r   r   )r<   r   Úgates   &&&r   rU   ÚSwiGLU.__call__œ   s   € Ü�d‹Ðr   © )	r[   r\   r]   r^   r1   rU   r`   ra   rb   rc   s   @@r   r|   r|   ˜   s   ù‡ € õ÷ò r   r|   c                   ób   a a€ ] tR t^ t o]! 4       R3V3R lV 3R llltV3R lR ltRtVtV ;t	# )Ú	SwitchGLUFc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# ©r   r   Úhidden_dimsr   r    rg   )r'   r(   s   "€r   r)   ÚSwitchGLU.__annotate__¡   s3   ø€ ÷ %ñ %áð%ñ ð%ñ ð	%ñ ñ%r   c                óœ   <€ \         SV `  4        \        WW5R 7      V n        \        WW5R 7      V n        \        W!W5R 7      V n        W@n        R# ©)r    N)r0   r1   re   Ú	gate_projÚup_projÚ	down_projÚ
activation©r<   r   r‡   r   rŽ   r    r>   s   &&&&&&€r   r1   ÚSwitchGLU.__init__¡   s?   ø€ ô 	‰ÑÔä% j¸{ÔVˆŒÜ# J¸[ÔTˆŒÜ% k¸{ÔVˆŒØ$Žr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   Úreturn©r
   Úarray)r'   r(   s   "€r   r)   rˆ   °   s   ø€ ÷ ñ ¡b§h¡hñ r   c                óÂ  € \         P                  ! VR4      pVP                  ^@8¬  pTpRpV'       d   \        W4      w  rpV P                  '       d   \         P
                  ! V4      pV P                  WVR7      pV P                  WVR7      pV P                  V P                  Wg4      VVR7      pV'       d   \        WVP                  4      pVP                  R4      # ©r   N)rO   rP   )rP   r   )r
   rS   Úsizer   ÚtrainingÚstop_gradientrŒ   r‹   r�   rŽ   r   r   Úsqueeze)r<   r   r   Údo_sortÚidxr   Úx_upÚx_gates   &&&     r   rU   ÚSwitchGLU.__call__°   sÃ   € Ü�NŠN˜1˜hÓ'ˆð —,‘, "Ñ$ˆØˆØˆ	ßÜ ,¨QÓ 8ÑˆA�IØ�=�=ˆ=Ü×"Ò" 3Ó'ˆCØ�|‰|˜A°7ˆ|Ó;ˆØ—‘ °w�Ó?ˆØ�N‰NØ�O‰O˜DÓ)ØØ"ð ó 
ˆ÷ Ü ¨g¯m©mÓ<ˆAà�y‰y˜‹}Ðr   )rŽ   r�   r‹   rŒ   )
r[   r\   r]   r^   r|   r1   rU   r`   ra   rb   rc   s   @@r   r„   r„       s(   ù‡ € ñ “8Ø÷%õ %÷÷ ð r   r„   c                   óz   a a€ ] tR t^Êt o]P
                  ! RR7      R3V3R lV 3R llltV3R lR ltRtVt	V ;t
# )	Ú	SwitchMLPÚprecise)ÚapproxFc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# r†   rg   )r'   r(   s   "€r   r)   ÚSwitchMLP.__annotate__Ë   s3   ø€ ÷ %ñ %áð%ñ ð%ñ ð	%ñ ñ%r   c                óx   <€ \         SV `  4        \        WW5R 7      V n        \        W!W5R 7      V n        W@n        R# rŠ   )r0   r1   re   Úfc1Úfc2rŽ   r�   s   &&&&&&€r   r1   ÚSwitchMLP.__init__Ë   s1   ø€ ô 	‰ÑÔä 
¸ÔPˆŒÜ ¸ÔPˆŒØ$Žr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r’   r”   )r'   r(   s   "€r   r)   r¦   Ù   s   ø€ ÷ ñ ¡b§h¡hñ r   c                óž  € \         P                  ! VR4      pVP                  ^@8¬  pTpRpV'       d   \        W4      w  rpV P                  '       d   \         P
                  ! V4      pV P                  WVR7      pV P                  V4      pV P                  WVR7      pV'       d   \        WVP                  4      pVP                  R4      # r—   )r
   rS   r˜   r   r™   rš   r¨   rŽ   r©   r   r   r›   )r<   r   r   rœ   r�   r   s   &&&   r   rU   ÚSwitchMLP.__call__Ù   sª   € Ü�NŠN˜1˜hÓ'ˆð —,‘, "Ñ$ˆØˆØˆ	ßÜ ,¨QÓ 8ÑˆA�IØ�=�=ˆ=Ü×"Ò" 3Ó'ˆCØ�H‰H�Q¨GˆHÓ4ˆØ�O‰O˜AÓˆØ�H‰H�Q¨GˆHÓ4ˆçÜ ¨g¯m©mÓ<ˆAà�y‰y˜‹}Ðr   )rŽ   r¨   r©   )r[   r\   r]   r^   ÚnnÚGELUr1   rU   r`   ra   rb   rc   s   @@r   r¢   r¢   Ê   s/   ù‡ € ð —7’7 )Ô,Ø÷%õ %÷÷ ð r   r¢   r   )r2   Ú	functoolsr   Úmlx.coreÚcorer
   Úmlx.nnr®   Úactivationsr   r   r   ÚModuler   re   r|   r„   r¢   r‚   r   r   Ú<module>r¶      st   ðó Ý å Ý å òCôô?˜BŸI™Iô ?ôD8�2—9‘9ô 8ôvˆR�Y‰Yô ô'�—	‘	ô 'ôT"�—	‘	ö "r   