Ë
    îÿæiö2  ã                   ó¨  — d dl mZ d dlmZ d dlmZmZmZ d dlm	Z
 d dlmZ d dlmZmZmZ ddlmZmZmZ ddlmZ e G d	„ d
e«      «       Zed„ «       Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Zy)é    )Ú	dataclass)Ú	lru_cache)ÚAnyÚListÚOptionalN)Úshard_inplaceÚshard_linearÚsum_gradientsé   )ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	SwitchGLUc                   óØ   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   dZeed<   dZ	eed<   dZ
ee   ed<   dZeed<   y)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚintermediate_sizeÚnum_attention_headsÚnum_key_value_headsÚmax_position_embeddingsÚnum_experts_per_tokÚnum_local_expertsÚshared_intermediate_sizeÚnum_hidden_layersÚrms_norm_epsÚ
rope_thetaÚ
rotary_dimÚ
vocab_sizeFÚtie_word_embeddingsÚsigmoidÚscoring_funcNÚhead_dimTÚuse_qk_norm)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr    Úboolr"   r#   r   r$   © ó    új/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/minimax.pyr   r      s€   … àƒOØÓØÓØÓØÓØ Ó ØÓØÓØ!Ó!ØÓØÓØÓØƒOØƒOØ %Ð˜Ó%Ø!€L�#Ó!Ø"€Hˆh�s‰mÓ"Ø€K�Ôr.   r   c                 ót   ‡ ‡‡— t         j                  d„ «       Št         j                  ˆ fd„«       Šˆˆfd„}|S )Nc                 ó~   — | j                  t        j                  «      j                  «       j	                  dd¬«      S )NéÿÿÿÿT)Úkeepdims)ÚastypeÚmxÚfloat32ÚsquareÚsum)Úxs    r/   Ú_cast_square_sumz*sharded_rms_norm.<locals>._cast_square_sum'   s.   € à�x‰xœŸ
™
Ó#×*Ñ*Ó,×0Ñ0°¸dÐ0ÓCÐCr.   c                 ó4  •— t         j                  j                  |‰¬«      }t        j                  || j                  d   ‰j                  «       z  z  |z   «      }| j                  t         j                  «      |z  |z  j                  | j                  «      S )N©Úgroupr2   )	r5   ÚdistributedÚall_sumÚrsqrtÚshapeÚsizer4   r6   Údtype)r9   Únorm2ÚwÚepsÚnormr=   s        €r/   Ú
_normalizez$sharded_rms_norm.<locals>._normalize+   ss   ø€ ä—‘×&Ñ& u°EÐ&Ó:ˆÜ�x‰x˜ §¡¨¡¨u¯z©z«|Ñ!;Ñ<¸sÑBÓCˆØ—‘œŸ™Ó$ tÑ+¨aÑ/×7Ñ7¸¿¹Ó@Ð@r.   c                 ó&   •—  ‰|  ‰| «      ||«      S ©Nr-   )r9   rE   rF   r:   rH   s      €€r/   Ú_inner_sharded_rms_normz1sharded_rms_norm.<locals>._inner_sharded_rms_norm4   s   ø€ Ù˜!Ñ-¨aÓ0°!°SÓ9Ð9r.   )r5   Úcompile)r=   rK   r:   rH   s   ` @@r/   Úsharded_rms_normrM   %   sB   ú€ ä‡Z�ZñDó ðDô ‡Z�ZóAó ðAõ:ð #Ð"r.   c                   ó´   ‡ — e Zd Z	 d
dededeej                  j                     fˆ fd„Z	d„ Z
d„ Zeddœdeej                  j                     fd	„«       Zˆ xZS )ÚShardedRMSNormNÚdimsrF   r=   c                 óÚ   •— t         ‰| �  «        |xs t        j                  j	                  «       }t        j
                  ||j                  «       z  f«      | _        || _        || _	        y rJ   )
ÚsuperÚ__init__r5   r>   ÚinitÚonesrB   Úweightr=   rF   )ÚselfrP   rF   r=   Ú	__class__s       €r/   rS   zShardedRMSNorm.__init__;   sS   ø€ ô 	‰ÑÔØÒ.œŸ™×,Ñ,Ó.ˆÜ—g‘g˜t u§z¡z£|Ñ3Ð5Ó6ˆŒØˆŒ
Øˆ�r.   c                 óˆ   — | j                   j                  d   | j                  j                  «       z  › d| j                  › �S )Nr   z, eps=)rV   rA   r=   rB   rF   ©rW   s    r/   Ú_extra_reprzShardedRMSNorm._extra_reprD   s7   € Ø—+‘+×#Ñ# AÑ&¨¯©¯©Ó):Ñ:Ð;¸6À$Ç(Á(ÀÐLÐLr.   c                 óV   —  t        | j                  «      || d   | j                  «      S )NrV   )rM   r=   rF   )rW   r9   s     r/   Ú__call__zShardedRMSNorm.__call__G   s%   € Ø+Ô §
¡
Ó+¨A¨t°H©~¸t¿x¹xÓHÐHr.   r<   c                ó  —  | |j                   j                  d   |j                  |¬«      }t        j                  t        j
                  |j                   |j                  «       d¬«      |j                  «          «      |_         |S )Nr   r<   r2   ©Úaxis)rV   rA   rF   r5   Ú
contiguousÚsplitrB   Úrank)ÚclsÚnorm_moduler=   Úsns       r/   Úfrom_rms_normzShardedRMSNorm.from_rms_normJ   se   € ñ �×#Ñ#×)Ñ)¨!Ñ,¨k¯o©oÀUÔKˆÜ—M‘MÜ�H‰H�[×'Ñ'¨¯©«¸BÔ?ÀÇ
Á
ÃÑMó
ˆŒ	ð ˆ	r.   )gñhãˆµøä>N)r%   r&   r'   r*   r+   r   r5   r>   ÚGrouprS   r[   r]   Úclassmethodrg   Ú__classcell__©rX   s   @r/   rO   rO   :   sl   ø„ àTXñØðØ#ðØ3;¸B¿N¹N×<PÑ<PÑ3QõòMòIð àEIòØ$,¨R¯^©^×-AÑ-AÑ$Bòó ôr.   rO   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚMiniMaxAttentionÚargsc                 ó  •— t         ‰| �  «        |j                  x| _        }|j                  | _        |j
                  | _        |j                  xs ||j                  z  x| _        }|dz  | _        t        j                  |j                  | j                  |z  d¬«      | _
        t        j                  |j                  | j
                  |z  d¬«      | _        t        j                  |j                  | j
                  |z  d¬«      | _        t        j                  | j                  |z  |j                  d¬«      | _        t        |d«      r|j                  nd| _        | j                  rft        j                   || j                  z  |j"                  ¬«      | _        t        j                   || j
                  z  |j"                  ¬«      | _        t        j(                  |j*                  d|j,                  ¬«      | _        y )Ng      à¿F©Úbiasr$   ©rF   )ÚtraditionalÚbase)rR   rS   r   Ú
hidden_dimr   r   r#   ÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚhasattrr$   ÚRMSNormr   Úq_normÚk_normÚRoPEr   r   Úrope)rW   rn   r   r#   rX   s       €r/   rS   zMiniMaxAttention.__init__W   s¢  ø€ Ü‰ÑÔà(,×(8Ñ(8Ð8ˆŒ˜+à#'×#;Ñ#;ˆÔ Ø#'×#;Ñ#;ˆÔ à�M‰MÒD˜[¨D×,DÑ,DÑDð	
ˆŒ˜ð ˜t‘^ˆŒ
ä—i‘iØ×Ñ˜d×6Ñ6¸ÑAÈô
ˆŒô —i‘iØ×Ñ˜d×6Ñ6¸ÑAÈô
ˆŒô —i‘iØ×Ñ˜d×6Ñ6¸ÑAÈô
ˆŒô —i‘iØ×$Ñ$ xÑ/°×1AÑ1AÈô
ˆŒô 07°t¸]Ô/K˜4×+Ò+ÐQVˆÔØ×ÒÜŸ*™*Ø˜4×3Ñ3Ñ3¸×9JÑ9JôˆDŒKô Ÿ*™*Ø˜4×3Ñ3Ñ3¸×9JÑ9JôˆDŒKô —G‘G˜DŸO™O¸ÀTÇ_Á_ÔUˆ�	r.   r9   ÚmaskÚcacheÚreturnc                 óŽ  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}| j                  r"| j                  |«      }| j                  |«      }|j                  ||| j                  d«      j                  dddd«      }|j                  ||| j                  d«      j                  dddd«      }|	j                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j                  ||d«      }
| j!                  |
«      S )Nr2   r   é   r   é   )Úoffset)r„   rv   rƒ   )rA   ry   rz   r{   r$   r   r€   Úreshaper   Ú	transposer   r‚   r‰   Úupdate_and_fetchr   rv   r|   )rW   r9   rƒ   r„   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r/   r]   zMiniMaxAttention.__call__{   sž  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆà×ÒØ—k‘k 'Ó*ˆGØ—;‘;˜tÓ$ˆDà—/‘/ ! Q¨×(@Ñ(@À"ÓE×OÑOØˆq�!�Qó
ˆð �|‰|˜A˜q $×":Ñ":¸BÓ?×IÑIÈ!ÈQÐPQÐSTÓUˆØ—‘  1 d×&>Ñ&>ÀÓC×MÑMØˆq�!�Qó
ˆð ÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆà�{‰{˜6Ó"Ð"r.   ©NN©r%   r&   r'   r   rS   r5   Úarrayr   r   r]   rj   rk   s   @r/   rm   rm   V   sZ   ø„ ð"V˜Yõ "VðN $(Ø#ñ	$#à�8‰8ð$#ð �r—x‘xÑ ð$#ð ˜‰}ð	$#ð
 
�‰÷$#r.   rm   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚMiniMaxSparseMoeBlockrn   c                 ó^  •— t         ‰| �  «        |j                  | _        t        j                  |j
                  |j                  d¬«      | _        t        |j
                  |j                  |j                  «      | _
        t        j                  |j                  f«      | _        d | _        y ©NFrp   )rR   rS   r   rw   rx   r   r   Úgater   r   Ú
switch_mlpr5   ÚzerosÚe_score_correction_biasÚsharding_group©rW   rn   rX   s     €r/   rS   zMiniMaxSparseMoeBlock.__init__£   s…   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ ä—I‘I˜d×.Ñ.°×0FÑ0FÈUÔSˆŒ	Ü#Ø×Ñ˜d×4Ñ4°d×6LÑ6Ló
ˆŒô (*§x¡x°×1GÑ1GÐ0IÓ'JˆÔ$Ø"ˆÕr.   r9   r…   c                 ó¾  — | j                   � t        | j                   «      |«      }| j                  |j                  t        j
                  «      «      }t	        j                  |«      }|}|| j                  z   }| j                  }t	        j                  | |dz
  d¬«      dd |…f   }t	        j                  ||d¬«      }|t	        j                  |dd¬«      dz   z  }|j                  |j                  «      }| j                  ||«      }||d	   z  j                  d
¬«      }| j                   �+t        j                  j                  || j                   ¬«      }|S )Nr   r2   )Úkthr`   .r_   T)r`   r3   g#B’¡œÇ;).Néþÿÿÿr<   )rŸ   r
   r›   r4   r5   r6   r!   rž   r   ÚargpartitionÚtake_along_axisr8   rC   rœ   r>   r?   )rW   r9   ÚgatesÚscoresÚorig_scoresÚkÚindsÚys           r/   r]   zMiniMaxSparseMoeBlock.__call__®   s2  € Ø×ÑÐ*Ø2”˜d×1Ñ1Ó2°1Ó5ˆAà—	‘	˜!Ÿ(™(¤2§:¡:Ó.Ó/ˆä—‘˜EÓ"ˆØˆØ˜$×6Ñ6Ñ6ˆà×$Ñ$ˆÜ�‰ ˜w¨A°©E¸Ô;¸CÀÀ!À¸GÑDˆÜ×#Ñ# K°¸BÔ?ˆàœ2Ÿ6™6 &¨r¸DÔAÀEÑIÑJˆØ—‘˜qŸw™wÓ'ˆà�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0ˆà×ÑÐ*Ü—‘×&Ñ& q°×0CÑ0CÐ&ÓDˆAàˆr.   )	r%   r&   r'   r   rS   r5   r–   r]   rj   rk   s   @r/   r˜   r˜   ¢   s*   ø„ ð	#˜Yõ 	#ð˜"Ÿ(™(ð  r§x¡x÷ r.   r˜   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚMiniMaxDecoderLayerrn   c                 ó"  •— t         ‰| �  «        t        |«      | _        t	        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  ¬«      | _        y )Nrr   )rR   rS   rm   Ú	self_attnr˜   Úblock_sparse_moerw   r~   r   r   Úinput_layernormÚpost_attention_layernormr    s     €r/   rS   zMiniMaxDecoderLayer.__init__É   sh   ø€ Ü‰ÑÔä)¨$Ó/ˆŒä 5°dÓ ;ˆÔä!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÕ%r.   r9   rƒ   r„   r…   c                 ó–   — || j                  | j                  |«      ||«      z   }|| j                  | j                  |«      «      z   }|S rJ   )r¯   r±   r°   r²   )rW   r9   rƒ   r„   Úrs        r/   r]   zMiniMaxDecoderLayer.__call__Õ   sK   € ð �—‘˜t×3Ñ3°AÓ6¸¸eÓDÑDˆØ�×%Ñ% d×&CÑ&CÀAÓ&FÓGÑGˆØˆr.   r”   r•   rk   s   @r/   r­   r­   È   sW   ø„ ð

˜Yõ 

ð $(Ø#ñ	à�8‰8ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷r.   r­   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚMiniMaxModelrn   c                 óN  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        |¬«      ‘Œ c}| _
        t        j                  |j
                  |j                  ¬«      | _        y c c}w )N)rn   rr   )rR   rS   rw   Ú	Embeddingr   r   Úembed_tokensÚranger   r­   Úlayersr~   r   rG   )rW   rn   Ú_rX   s      €r/   rS   zMiniMaxModel.__init__á   s   ø€ Ü‰ÑÔÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔô 5:¸$×:PÑ:PÔ4Qó
Ù4Q¨qÔ TÖ*Ð4Qñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆ�	ùò	
s   ÁB"Úinputsrƒ   r„   r…   c                 óê   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||«      }Œ | j                  |«      S )Nr   )r¹   Úlenr»   r   ÚziprG   )rW   r½   rƒ   r„   ÚhÚlayerÚcs          r/   r]   zMiniMaxModel.__call__ë   su   € ð ×Ñ˜fÓ%ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä$ Q¨¨a©Ó1ˆä˜DŸK™K¨Ö/‰HˆE�1Ù�a˜˜qÓ!‰Að 0ð �y‰y˜‹|Ðr.   r”   r•   rk   s   @r/   r¶   r¶   à   sY   ø„ ðH˜Yõ Hð $(Ø#ñ	à—‘ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷r.   r¶   c                   óä   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   fd„Z	d„ Z
ddeej                  j                     fd	„Zed
„ «       Zed„ «       Zed„ «       Zˆ xZS )ÚModelrn   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y rš   )rR   rS   rn   r   r¶   Úmodelr    rw   rx   r   r   Úlm_headr    s     €r/   rS   zModel.__init__ÿ   sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ! $Ó'ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r.   r½   rƒ   r„   c                 óÊ   — | j                  |||¬«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S )N)r½   rƒ   r„   )rÇ   rn   r    r¹   Ú	as_linearrÈ   )rW   r½   rƒ   r„   Úouts        r/   r]   zModel.__call__  s[   € ð �j‰j ¨T¸ˆjÓ?ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r.   c                 ó.  — d„ }i }|j                  «       D ];  \  }}d|v r(|}|j                  dd«      }||   } |||«      }|||<   Œ2||vsŒ7|||<   Œ= |}d|vr|S t        | j                  j                  «      D ]’  }	d|	› �}
ddd	d
œ}|j                  «       D ]r  \  }}|
› d|› d�|v sŒt        | j                  j
                  «      D �cg c]  }|j                  |
› d|› d|› d�«      ‘Œ }}t        j                  |«      ||
› d|› d�<   Œt Œ” |S c c}w )z3Dequantize FP8 weights and restructure MoE experts.c                 ó   — t         j                  }t        j                  | t         j                  ¬«      } d}| j                  \  }}| |z  }| |z  }t        j                  | d|fd|ff«      } | j                  ||z   |z  |||z   |z  |f«      } | |d d …d d d …d f   z  j                  ||z   ||z   «      } | d |…d |…f   j                  |«      S )N)rC   é€   r   )r5   Úbfloat16Úfrom_fp8rA   ÚpadrŠ   r4   )rV   Ú	scale_invrC   ÚbsÚmÚnÚ
pad_bottomÚpad_sides           r/   ÚdequantzModel.sanitize.<locals>.dequant  så   € Ü—K‘KˆEÜ—[‘[ ¬r¯{©{Ô;ˆFØˆBØ—<‘<‰DˆAˆqØ˜" ™ˆJØ˜˜b‘yˆHÜ—V‘V˜F a¨ _°q¸(°mÐ$DÓEˆFØ—^‘^Ø�j‘. RÑ'¨¨a°(©l¸rÑ-AÀ2ÐFóˆFð ˜yª¨D²!°TÐ)9Ñ:Ñ:×CÑCØ�J‘  H¡óˆFð ˜"˜1˜"˜b˜q˜b˜&‘>×(Ñ(¨Ó/Ð/r.   Úweight_scale_invÚ
_scale_invÚ z3model.layers.0.block_sparse_moe.experts.0.w1.weightzmodel.layers.Ú	gate_projÚ	down_projÚup_proj)Úw1Úw2Úw3z.block_sparse_moe.experts.0.z.weightz.block_sparse_moe.experts.Ú.z.block_sparse_moe.switch_mlp.)	ÚitemsÚreplacerº   rn   r   r   Úpopr5   Ústack)rW   ÚweightsrØ   Únew_weightsr©   ÚvrÒ   ÚwkrV   ÚlÚprefixÚmappingÚ	orig_nameÚnew_nameÚeÚto_joins                   r/   ÚsanitizezModel.sanitize  sn  € ò	0ð" ˆØ—M‘M–O‰DˆAˆqØ! QÑ&Ø�	Ø—Y‘Y˜|¨RÓ0�Ø  ™�Ù  ¨Ó3�Ø"(�˜B’Ø˜+Ò%Ø!"�˜A’ð $ð ˆð AÈÑOØˆNä�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFØ(°À9ÑMˆGØ'.§}¡}¦Ñ#�	˜8Ø�XÐ9¸)¸ÀGÐLÐPWÒWô
 "' t§y¡y×'BÑ'BÔ!Có	ñ "D˜Að  Ÿ™Ø%˜hÐ&@ÀÀÀ1ÀYÀKÈwÐWõð "Dð	 ð ô Ÿ™ Ó)ð Ø!˜(Ð"?À¸zÈÐQòñ (7ð 4ð ˆùòs   Ã!Dr=   c                 ó  — |xs t         j                  j                  «       }|j                  «       }|j	                  «       }| j
                  j                  D �]$  }t        |j                  j                  d|¬«      |j                  _	        t        |j                  j                  d|¬«      |j                  _
        t        |j                  j                  d|¬«      |j                  _        t        |j                  j                  d|¬«      |j                  _        |j                  j                  rtt        j                  |j                  j                   |¬«      |j                  _        t        j                  |j                  j"                  |¬«      |j                  _        |j                  xj$                  |z  c_        |j                  xj&                  |z  c_        t)        |j*                  j,                  j.                  d|¬«       t)        |j*                  j,                  j0                  d|¬«       t)        |j*                  j,                  j2                  d|¬«       ||j*                  _        �Œ' y )Nzall-to-shardedr<   zsharded-to-all)r5   r>   rT   rB   rc   rÇ   r»   r	   r¯   ry   rz   r{   r|   r$   rO   rg   r   r€   r   r   r   r°   rœ   rÜ   rÝ   rÞ   rŸ   )rW   r=   ÚNrc   rÂ   s        r/   ÚshardzModel.shardI  sê  € ØÒ.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ�z‰z‹|ˆØ—Z‘Z×&Õ&ˆEä%1Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ð �‰×*Ò*Ü)7×)EÑ)EØ—O‘O×*Ñ*°%ð *Fó *�—‘Ô&ô *8×)EÑ)EØ—O‘O×*Ñ*°%ð *Fó *�—‘Ô&ð �O‰O×/Ò/°AÑ5Õ/Ø�O‰O×/Ò/°AÑ5Õ/ô Ø×&Ñ&×1Ñ1×;Ñ;Ø Øõô
 Ø×&Ñ&×1Ñ1×;Ñ;Ø Øõô
 Ø×&Ñ&×1Ñ1×9Ñ9Ø Øõð
 5:ˆE×"Ñ"Ö1ñS 'r.   c                 ó.   — | j                   j                  S rJ   )rÇ   r»   rZ   s    r/   r»   zModel.layersx  s   € à�z‰z× Ñ Ð r.   c                 ó   — d„ }|S )Nc                 ó
   — d| vS )Nrž   r-   )r©   s    r/   Ú	predicatez'Model.cast_predicate.<locals>.predicate~  s   € Ø,°AÐ5Ð5r.   r-   ©rW   rù   s     r/   Úcast_predicatezModel.cast_predicate|  s   € ò	6ð Ðr.   c                 ó   — d„ }|S )Nc                 ó0   — | j                  d«      rdddœS y)Nzblock_sparse_moe.gateé@   é   )Ú
group_sizeÚbitsT)Úendswith)Úpathr¼   s     r/   rù   z(Model.quant_predicate.<locals>.predicate…  s   € Ø�}‰}Ð4Ô5Ø&(°!Ñ4Ð4Ør.   r-   rú   s     r/   Úquant_predicatezModel.quant_predicateƒ  s   € ò	ð
 Ðr.   r”   rJ   )r%   r&   r'   r   rS   r5   r–   r   r   r]   rò   r>   rh   rõ   Úpropertyr»   rû   r  rj   rk   s   @r/   rÅ   rÅ   þ   s©   ø„ ðT˜Yõ Tð $(Ø#ñ	à—‘ðð �r—x‘xÑ ðð ˜‰}ó	ò3ñj-:˜8 B§N¡N×$8Ñ$8Ñ9ó -:ð^ ñ!ó ð!ð ñó ðð ñó ôr.   rÅ   ) Údataclassesr   Ú	functoolsr   Útypingr   r   r   Úmlx.coreÚcorer5   Úmlx.nnrw   Úmlx.nn.layers.distributedr   r	   r
   rt   r   r   r   Úswitch_layersr   r   rM   ÚModulerO   rm   r˜   r­   r¶   rÅ   r-   r.   r/   Ú<module>r     s½   ðõ "Ý ß &Ñ &å Ý ß PÑ Pç TÑ TÝ $ð ô�ó ó ðð* ñ#ó ð#ô(�R—Y‘Yô ô8I#�r—y‘yô I#ôX#˜BŸI™Iô #ôL˜"Ÿ)™)ô ô0�2—9‘9ô ô<LˆB�I‰Iõ Lr.   