Ë
    îÿæiâ2  ã                   ó"  — d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z d dlmZ ddlmZ ddlmZmZmZ ddlmZ dd	lmZ e G d
„ de«      «       Z eej2                  d¬«      d„ «       Z G d„ dej6                  «      Z G d„ dej6                  «      Zej2                  d„ «       Z G d„ dej6                  «      Z G d„ dej6                  «      Z  G d„ dej6                  «      Z! G d„ dej6                  «      Z" G d„ dej6                  «      Z#y)é    )Ú	dataclass)Úpartial)ÚAnyÚDictÚOptionalÚUnionNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Úinitialize_rope)Ú	SwitchGLUc                   óÂ  — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   dZe	e
eeeef   f      ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZe	e   ed<   dZeed<   dZeed<   dZeed <   d!Zeed"<   d#Zeed$<   d%Zeed&<   dZe	e   ed'<   dZeed(<   y))Ú	ModelArgsÚ
model_typeÚhidden_sizeÚintermediate_sizeÚmax_position_embeddingsÚmoe_intermediate_sizeÚnum_expertsÚnum_shared_expertsÚnorm_topk_probÚnum_attention_headsÚnum_experts_per_tokÚnum_hidden_layersÚnum_key_value_headsÚrms_norm_epsÚ
rope_thetaÚ
vocab_sizeÚfirst_k_dense_replaceNÚrope_scalingFÚuse_biasÚuse_qkv_biasÚ	norm_headÚnorm_softmaxÚuse_qk_normÚtie_word_embeddingsg      ð?Úpartial_rotary_factorÚ
rotary_dimÚmoe_router_enable_expert_biasTÚ moe_router_enable_routed_scalingÚrouted_scaling_factorÚsoftmaxÚscore_functionr	   Ún_groupé   Ú
topk_groupÚ#moe_shared_expert_intermediate_sizeÚmoe_router_enable_shared_expert)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚboolÚfloatr"   r   r   r   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r/   r0   r2   r3   r4   © ó    ún/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/bailing_moe.pyr   r      s1  … àƒOØÓØÓØ Ó ØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØƒOØÓØ;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?Ø€HˆdÓØ€L�$ÓØ€IˆtÓØ€L�$ÓØ€K�ÓØ %Ð˜Ó%Ø#&Ð˜5Ó&Ø $€J�˜‘Ó$Ø*/Ð! 4Ó/Ø-1Ð$ dÓ1Ø#&Ð˜5Ó&Ø#€N�CÓ#Ø€GˆSÓØ€J�ÓØ9=Ð'¨°#©Ó=Ø,0Ð# TÔ0r>   r   T)Ú	shapelessc                 ód   — | |d   z  j                  d¬«      j                  | j                  «      S )N).Néþÿÿÿ©Úaxis)ÚsumÚastypeÚdtype)Úexpert_outputsÚscoress     r?   Úaggregate_expert_outputsrJ   5   s5   € ð 
˜& Ñ+Ñ	+×0Ñ0°bÐ0Ó9×@Ñ@À×AUÑAUÓVðr>   c                   óP   ‡ — e Zd Zddedee   fˆ fd„Zdej                  fd„Z	ˆ xZ
S )ÚBailingMoeMLPÚargsr   c                 ó®  •— t         ‰| �  «        |�|n|j                  | _        t        j                  |j
                  | j                  |j                  ¬«      | _        t        j                  | j                  |j
                  |j                  ¬«      | _        t        j                  |j
                  | j                  |j                  ¬«      | _	        y )N©Úbias)
ÚsuperÚ__init__r   ÚnnÚLinearr   r#   Ú	gate_projÚ	down_projÚup_proj)ÚselfrM   r   Ú	__class__s      €r?   rR   zBailingMoeMLP.__init__=   s§   ø€ Ü‰ÑÔð !Ð,ñ à×'Ñ'ð 	Ôô Ÿ™Ø×Ñ˜d×4Ñ4¸4¿=¹=ô
ˆŒô Ÿ™Ø×"Ñ" D×$4Ñ$4¸4¿=¹=ô
ˆŒô —y‘yØ×Ñ˜d×4Ñ4¸4¿=¹=ô
ˆ�r>   Úreturnc                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)rV   r
   rU   rW   ©rX   Úxs     r?   Ú__call__zBailingMoeMLP.__call__O   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr>   r\   )r5   r6   r7   r   r   r:   rR   ÚmxÚarrayr_   Ú__classcell__©rY   s   @r?   rL   rL   <   s,   ø„ ñ
˜Yð 
¸8ÀC¹=õ 
ð$J˜RŸX™X÷ Jr>   rL   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚBailingMoeAttentionrM   c                 óÊ  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j
                  | j                  z  | _        | j                  dz  | _        t        j                  |j
                  | j                  d| j                  z  z   | j                  z  |j                  ¬«      | _        t        j                  | j                  | j                  z  |j
                  |j                  ¬«      | _        |j                  r`t        j                  | j                  |j                  ¬«      | _        t        j                  | j                  |j                  ¬«      | _        |j$                  x}€"t'        | j                  |j(                  z  «      }t+        ||j,                  d|j.                  |j0                  ¬«      | _        y )Ng      à¿é   rO   ©ÚepsF)ÚtraditionalÚscaling_configr   )rQ   rR   r'   r   r   r   Úhead_dimÚscalerS   rT   r$   Úquery_key_valuer#   ÚdenseÚRMSNormr   Úkey_layernormÚquery_layernormr*   r:   r)   r   r   r"   r   Úrope)rX   rM   Úrope_dimrY   s      €r?   rR   zBailingMoeAttention.__init__T   sq  ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ#'×#;Ñ#;ˆÔ Ø#'×#;Ñ#;ˆÔ Ø×(Ñ(¨D×,DÑ,DÑDˆŒØ—]‘] DÑ(ˆŒ
ä!Ÿy™yØ×ÑØ×%Ñ%¨¨D×,DÑ,DÑ(DÑDÈÏÉÑUØ×"Ñ"ô 
ˆÔô
 —Y‘YØ×$Ñ$ t§}¡}Ñ4Ø×ÑØ—‘ô
ˆŒ
ð ×ÒÜ!#§¡¨D¯M©M¸t×?PÑ?PÔ!QˆDÔÜ#%§:¡:¨d¯m©mÀ×ARÑARÔ#SˆDÔ àŸ™Ð'ˆHÐ0Ü˜4Ÿ=™=¨4×+EÑ+EÑEÓFˆHÜ#ØØ�O‰OØØ×,Ñ,Ø$(×$@Ñ$@ô
ˆ�	r>   r^   ÚmaskÚcacherZ   c                 ó,  — |j                   \  }}}| j                  |«      }| j                  | j                  z  }| j                  | j                  z  }	t        j                  ||||	z   gd¬«      \  }
}}|
j                  ||| j                  | j                  «      j                  dddd«      }|j                  ||| j                  | j                  «      j                  dddd«      }|j                  ||| j                  | j                  «      j                  dddd«      }| j                  r"| j                  |«      }| j                  |«      }|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||«      \  }}n"| j                  |«      }| j                  |«      }t        ||||| j                   |¬«      }|j                  dddd«      j                  ||d«      }| j#                  |«      S )	NéÿÿÿÿrC   r   rg   r	   é   )Úoffset)rv   rm   ru   )Úshapern   r   rl   r   r`   ÚsplitÚreshapeÚ	transposer'   rr   rq   rs   rz   Úupdate_and_fetchr   rm   ro   )rX   r^   ru   rv   ÚBÚLÚDÚqkvÚq_sizeÚkv_sizeÚqÚkÚvÚqueriesÚkeysÚvaluesÚoutputs                    r?   r_   zBailingMoeAttention.__call__u   sæ  € ð —'‘'‰ˆˆ1ˆaà×"Ñ" 1Ó%ˆà×)Ñ)¨D¯M©MÑ9ˆØ×*Ñ*¨T¯]©]Ñ:ˆÜ—(‘(˜3 ¨°'Ñ)9Ð :ÀÔD‰ˆˆ1ˆaà—)‘)˜A˜q $×":Ñ":¸D¿M¹MÓJ×TÑTØˆq�!�Qó
ˆð �y‰y˜˜A˜t×7Ñ7¸¿¹ÓG×QÑQØˆq�!�Qó
ˆð —‘˜1˜a ×!9Ñ!9¸4¿=¹=ÓI×SÑSØˆq�!�Qó
ˆð ×ÒØ×*Ñ*¨7Ó3ˆGØ×%Ñ% dÓ+ˆDàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�z‰z˜&Ó!Ð!r>   ©NN©r5   r6   r7   r   rR   r`   ra   r   r   r_   rb   rc   s   @r?   re   re   S   sX   ø„ ð
˜Yõ 
ðH $(Ø#ñ	)"à�8‰8ð)"ð �r—x‘xÑ ð)"ð ˜‰}ð	)"ð
 
�‰÷)"r>   re   c                 ó”  — | j                   }|dk(  r3t        j                  | j                  t        j                  «      «      }	n4t        j
                  | j                  t        j                  «      d¬«      }	|	}
|�|	|z   }	|dkD  rÎt        j                  |	d|df¬«      }	t        j                  |	dd¬«      j                  dd¬«      }||z
  }t        j                  ||dz
  d	¬
«      dd |…d d …f   }t        j                  |	t        j                  |«      t        j                  d|	j                   «      d	¬«      }	t        j                  |	d	d«      }	|}t        j                  |	| d¬
«      d| d …f   }t        j                  |
|d¬«      }	|dkD  r|r|	j                  dd¬«      dz   }|	|z  }	|	|z  }	||	j                  |«      fS )NÚsigmoidrx   rC   r	   )rD   r{   rg   T©rD   ÚkeepdimsrB   )ÚkthrD   .g        g#B’¡œÇ;)rG   r`   r�   rF   Úfloat32r.   Ú	unflattenÚtopkrE   ÚargpartitionÚput_along_axisÚstop_gradientra   ÚflattenÚtake_along_axis)ÚgatesÚe_score_correction_biasÚtop_kr0   r2   r-   r   r/   Úin_typerI   Úorig_scoresÚgroup_scoresr‡   Ú	group_idxÚindsÚdenominators                   r?   Úgroup_expert_selectr¥   ¡   sœ  € ð �k‰k€GØ˜Ò"Ü—‘˜EŸL™L¬¯©Ó4Ó5‰ä—‘˜EŸL™L¬¯©Ó4¸2Ô>ˆØ€KØÐ*ØÐ1Ñ1ˆØ�‚{Ü—‘˜f¨2°g¸r°]ÔCˆÜ—w‘w˜v q¨rÔ2×6Ñ6¸BÈÐ6ÓNˆØ�jÑ ˆÜ—O‘O L°a¸!±eÀ"ÔEÀcÈ2ÈAÈ2ÊqÀjÑQˆ	Ü×"Ñ"Ø”B×$Ñ$ YÓ/´·±¸#¸v¿|¹|Ó1LÐSUô
ˆô —‘˜F B¨Ó+ˆà€AÜ�?‰?˜6¨ r°Ô3°C¸!¸¹°HÑ=€DÜ×Ñ ¨T¸Ô;€FØˆq‚y‘^Ø—j‘j b°4�jÓ8¸5Ñ@ˆØ˜+Ñ%ˆØÐ+Ñ+€Fà�—‘˜wÓ'Ð'Ð'r>   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚBailingMoeGaterM   c                 óØ  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  |j                  |j                  d¬«      | _        |j                  r t!        j"                  |j                  f«      nd | _        |j&                  | _        y ©NFrO   )rQ   rR   r   r   rž   r0   r2   r-   r,   Úenable_routed_scalingrS   rT   r   r   rU   r+   r`   ÚzerosÚexpert_biasr/   ©rX   rM   rY   s     €r?   rR   zBailingMoeGate.__init__Ë   s¸   ø€ Ü‰ÑÔØ"×1Ñ1ˆÔà×-Ñ-ˆŒ
Ø—|‘|ˆŒØŸ/™/ˆŒØ%)×%?Ñ%?ˆÔ"Ø%)×%JÑ%JˆÔ"äŸ™ 4×#3Ñ#3°T×5EÑ5EÈEÔRˆŒð ×1Ò1ô �H‰H�d×&Ñ&Ð(Ô)àð 	Ôð
 #×1Ñ1ˆÕr>   c           
      óÐ   — t        | j                  |«      | j                  | j                  | j                  | j
                  | j                  | j                  | j                  «      S r\   )	r¥   rU   r¬   rž   r0   r2   r-   r   r/   r]   s     r?   r_   zBailingMoeGate.__call__Ý   sT   € Ü"Ø�N‰N˜1ÓØ×ÑØ�J‰JØ�L‰LØ�O‰OØ×&Ñ&Ø×ÑØ×Ñó	
ð 		
r>   ©r5   r6   r7   r   rR   r_   rb   rc   s   @r?   r§   r§   Ê   s   ø„ ð2˜Yõ 2ö$

r>   r§   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚBailingMoeSparseMoeBlockrM   c                 ó¢  •— t         ‰| �  «        || _        |j                  | _        t	        |j
                  |j                  |j                  |j                  ¬«      | _	        t        |«      | _        |j                  xs |j                  }|j                  dkD  r,|j                  r t        |||j                  z  ¬«      | _        y d | _        y )NrO   r   )rM   r   )rQ   rR   rM   r   r   r   r   r   r#   Ú
switch_mlpr§   Úgater3   r   r4   rL   Úshared_experts)rX   rM   Ú
shared_dimrY   s      €r?   rR   z!BailingMoeSparseMoeBlock.__init__ë   sÀ   ø€ Ü‰ÑÔØˆŒ	Ø#'×#;Ñ#;ˆÔ Ü#Ø×ÑØ×&Ñ&Ø×ÑØ—‘ô	
ˆŒô # 4Ó(ˆŒ	à×4Ñ4ÒR¸×8RÑ8Rð 	ð ×&Ñ&¨Ò*¨t×/SÒ/Sô	 ØØ",¨t×/FÑ/FÑ"Fôð 	Õð ð 	Õr>   c                 óª   — | j                  |«      \  }}| j                  ||«      }t        ||«      }| j                  �|| j                  |«      z   }|S r\   )r´   r³   rJ   rµ   )rX   r^   Útopk_idxÚtopk_weightÚouts        r?   r_   z!BailingMoeSparseMoeBlock.__call__  sV   € Ø $§	¡	¨!£Ñˆ�+Ø�o‰o˜a Ó*ˆÜ& s¨KÓ8ˆØ×ÑÐ*Ø˜×+Ñ+¨AÓ.Ñ.ˆCØˆ
r>   r¯   rc   s   @r?   r±   r±   ê   s   ø„ ð
˜Yõ 
ö.r>   r±   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚBailingMoeDecoderLayerrM   Ú	layer_idxc                 ón  •— t         ‰| �  «        t        |«      | _        |j                  �||j
                  k\  rt        |«      n
t        |«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        y )Nrh   )rQ   rR   re   Ú	attentionr   r!   r±   rL   ÚmlprS   rp   r   r   Úinput_layernormÚpost_attention_layernorm)rX   rM   r½   rY   s      €r?   rR   zBailingMoeDecoderLayer.__init__  s“   ø€ Ü‰ÑÔÜ,¨TÓ2ˆŒð
 × Ñ Ð,°¸d×>XÑ>XÒ1Xô % TÔ*ô ˜tÓ$ð 	Œô  "Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÕ%r>   r^   ru   rv   rZ   c                 óš   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   S r\   )r¿   rÁ   rÀ   rÂ   )rX   r^   ru   rv   ÚrÚhs         r?   r_   zBailingMoeDecoderLayer.__call__  sL   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�1‰uˆr>   r�   )r5   r6   r7   r   r:   rR   r`   ra   r   r   r_   rb   rc   s   @r?   r¼   r¼     s^   ø„ ð
˜Yð 
°3õ 
ð& $(Ø#ñ		à�8‰8ð	ð �r—x‘xÑ ð	ð ˜‰}ð		ð
 
�‰÷	r>   r¼   c                   óR   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   fd„Z	ˆ xZ
S )ÚBailingMoeModelrM   c                 óP  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}| _
        t        j                  |j
                  |j                  ¬«      | _        y c c}w )N)r½   rh   )rQ   rR   rS   Ú	Embeddingr    r   Úword_embeddingsÚranger   r¼   Úlayersrp   r   Únorm)rX   rM   ÚirY   s      €r?   rR   zBailingMoeModel.__init__)  sƒ   ø€ Ü‰ÑÔÜ!Ÿ|™|¨D¯O©O¸T×=MÑ=MÓNˆÔô ˜4×1Ñ1Ô2ó
á2�ô # 4°1Ö5Ø2ñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆ�	ùò	
s   ÁB#Úinputsrv   c                 óê   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||«      }Œ | j                  |«      S )Nr   )rÊ   ÚlenrÌ   r   ÚziprÍ   )rX   rÏ   rv   rÅ   ru   ÚlayerÚcs          r?   r_   zBailingMoeModel.__call__2  su   € ð
 × Ñ  Ó(ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä$ Q¨¨a©Ó1ˆä˜DŸK™K¨Ö/‰HˆE�1Ù�a˜˜qÓ!‰Að 0ð �y‰y˜‹|Ðr>   r\   rŽ   rc   s   @r?   rÇ   rÇ   (  s5   ø„ ðH˜Yõ Hð  $ñà—‘ðð ˜‰}÷r>   rÇ   c                   ó~   ‡ — e Zd Zdefˆ fd„Z	 d	dej                  fd„Zd„ Ze	d„ «       Z
e	d„ «       Ze	d„ «       Zˆ xZS )
ÚModelrM   c                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y r©   )rQ   rR   rM   r%   r   rÇ   Úmodelr(   rS   rT   r   r    Úlm_headr­   s     €r?   rR   zModel.__init__E  sc   ø€ Ü‰ÑÔØˆŒ	ØŸ™ˆŒØŸ/™/ˆŒÜ$ TÓ*ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r>   rÏ   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S r\   )rØ   rM   r(   rÊ   Ú	as_linearrÙ   )rX   rÏ   rv   rº   s       r?   r_   zModel.__call__N  sW   € ð
 �j‰j˜ Ó'ˆØ�9‰9×(Ò(Ø—*‘*×,Ñ,×6Ñ6°sÓ;ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r>   c                 ób  — | j                   j                  r|j                  dd «       | j                  rj|d   }|j                  }t
        j                  j                  |j                  t
        j                  «      dd¬«      dz   }||z  j                  |«      |d<   t        | j                   j                  «      D ]é  }d|› �}|| j                   j                  k\  sŒ"dD ]|  }dD ]u  }|› d	|› d
|› �|v sŒt        | j                   j                  «      D �	cg c]  }	|j                  |› d|	› d
|› d
|› �«      ‘Œ  }
}	t        j                  |
«      ||› d|› d
|› �<   Œw Œ~ |› d�|v r|j                  |› d�«      }|||› d�<   |› d�|v sŒÎ|j                  |› d�«      }|||› d�<   Œë |S c c}	w )Nzlm_head.weightr   Tr‘   gH¯¼šò×z>zmodel.layers.)rU   rV   rW   )ÚweightÚscalesÚbiasesz.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.z.mlp.gate.weightz.mlp.gate.gate_proj.weightz.mlp.gate.biasz.mlp.gate.gate_proj.bias)rM   r(   Úpopr%   rG   r`   ÚlinalgrÍ   rF   r”   rË   r   r!   r   Ústack)rX   ÚweightsÚwrG   Úweight_normÚlÚprefixÚmr‡   ÚeÚto_joinÚgate_weightÚ	gate_biass                r?   ÚsanitizezModel.sanitizeZ  sà  € Ø�9‰9×(Ò(Ø�K‰KÐ(¨$Ô/à�>Š>ØÐ(Ñ)ˆAØ—G‘GˆEä—	‘	—‘˜qŸx™x¬¯
©
Ó3¸!Àd�ÓKÈdÑRð ð *+¨[©×(@Ñ(@ÀÓ(GˆGÐ$Ñ%ä�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFà�D—I‘I×3Ñ3Ó3Û>�AÛ;˜Ø$˜X _°Q°C°q¸¸Ð<ÀÒGô */¨t¯y©y×/DÑ/DÔ)Eó'á)E Að !(§¡¨v¨h°mÀAÀ3ÀaÈÀsÈ!ÈAÈ3Ð,OÕ PØ)Eð $ð 'ô KMÏ(É(Ø 'óK˜G v hÐ.>¸q¸cÀÀ1À#Ð$FÒGñ <ð ?ð �XÐ-Ð.°'Ñ9Ø")§+¡+°°Ð8HÐ.IÓ"J�KØEP�G˜v˜hÐ&@ÐAÑBà�X˜^Ð,°Ò7Ø '§¡¨v¨h°nÐ,EÓ F�IØCL�G˜v˜hÐ&>Ð?Ò@ð+ 4ð. ˆùò!'s   Ä#F,
c                 ó   — d„ }|S )Nc                 ó0   — | j                  d«      rdddœS y)Nzmlp.gate.gate_projé@   é   )Ú
group_sizeÚbitsT)Úendswith)ÚpathÚ_s     r?   Ú	predicatez(Model.quant_predicate.<locals>.predicate�  s   € Ø�}‰}Ð1Ô2Ø&(°!Ñ4Ð4Ør>   r=   ©rX   rø   s     r?   Úquant_predicatezModel.quant_predicate  s   € ò	ð
 Ðr>   c                 ó   — d„ }|S )Nc                 ó
   — d| vS )Nr¬   r=   )r‡   s    r?   rø   z'Model.cast_predicate.<locals>.predicateŠ  s   € Ø ¨Ð)Ð)r>   r=   rù   s     r?   Úcast_predicatezModel.cast_predicateˆ  s   € ò	*ð Ðr>   c                 ó.   — | j                   j                  S r\   )rØ   rÌ   )rX   s    r?   rÌ   zModel.layers�  s   € à�z‰z× Ñ Ð r>   r\   )r5   r6   r7   r   rR   r`   ra   r_   rî   Úpropertyrú   rý   rÌ   rb   rc   s   @r?   rÖ   rÖ   D  si   ø„ ðT˜Yõ Tð ñ
à—‘ó
ò#ðJ ñó ðð ñó ðð ñ!ó ô!r>   rÖ   )$Údataclassesr   Ú	functoolsr   Útypingr   r   r   r   Úmlx.coreÚcorer`   Úmlx.nnrS   Úactivationsr
   Úbaser   r   r   Ú
rope_utilsr   Úswitch_layersr   r   ÚcompilerJ   ÚModulerL   re   r¥   r§   r±   r¼   rÇ   rÖ   r=   r>   r?   Ú<module>r     sõ   ðõ "Ý ß -Ó -å Ý å ß TÑ TÝ 'Ý $ð ô!1�ó !1ó ð!1ñH 	ˆ�‰˜tÔ$ñó %ðôJ�B—I‘Iô Jô.K"˜"Ÿ)™)ô K"ð\ ‡�ñ%(ó ð%(ôP
�R—Y‘Yô 
ô@˜rŸy™yô ôB˜RŸY™Yô ô:�b—i‘iô ô8M!ˆB�I‰Iõ M!r>   