Ë
    îÿæiÅ5  ã                   ó  — d dl Z d dlmZ d dlmZ d dlmZmZmZ d dl	m
Z d dlmZ d dlmZmZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ e G d„ de«      «       Z G d„ dej:                  «      Z G d„ dej:                  «      Zej@                  d„ «       Z! G d„ dej:                  «      Z" G d„ dej:                  «      Z# G d„ dej:                  «      Z$ G d„ deej:                  «      Z% G d„ dej:                  «      Z&y)é    N)Ú	dataclass)Úpartial)ÚAnyÚDictÚOptional)Úshard_inplaceÚshard_linearÚsum_gradientsé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚPipelineMixin)Ú	SwitchGLUc                   ó*  — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   ee	   ed<   eed<   eed<   eed<   eed<   dZ
eed<   dZeed<   y)Ú	ModelArgsÚ
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚmax_position_embeddingsÚmoe_intermediate_sizeÚnorm_topk_probÚnum_attention_headsÚn_groupÚhead_dimÚ
topk_groupÚn_shared_expertsÚn_routed_expertsÚrouted_scaling_factorÚnum_experts_per_tokÚfirst_k_dense_replaceÚnum_hidden_layersÚnum_key_value_headsÚrms_norm_epsÚ
rope_thetaÚrope_scalingÚuse_qk_normÚtie_word_embeddingsÚattention_biasÚpartial_rotary_factorÚsigmoidÚscoring_funcÚnoaux_tcÚtopk_methodN)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚboolÚfloatr   r   r.   r0   © ó    úk/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/glm4_moe.pyr   r      s«   … àƒOØƒOØÓØÓØ Ó ØÓØÓØÓØƒLØƒMØƒOØÓØÓØ Ó ØÓØÓØÓØÓØÓØÓØ˜4‘.Ó ØÓØÓØÓØ Ó Ø!€L�#Ó!Ø!€K�Ô!r:   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	Ú	AttentionÚargsc                 ó:  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  x| _        }|j                  }|dz  | _        t        j                  |||z  |j                  ¬«      | _        t        j                  |||z  |j                  ¬«      | _        t        j                  |||z  |j                  ¬«      | _        t        j                  ||z  |d¬«      | _        |j                   | _        | j                   rLt        j"                  ||j$                  ¬«      | _        t        j"                  ||j$                  ¬«      | _        t        j*                  t-        ||j.                  z  «      d|j0                  ¬«      | _        y )Ng      à¿©ÚbiasF©Úeps)ÚtraditionalÚbase)ÚsuperÚ__init__r   r   Ún_headsr%   Ú
n_kv_headsr   ÚscaleÚnnÚLinearr+   Úq_projÚk_projÚv_projÚo_projr)   ÚRMSNormr&   Úq_normÚk_normÚRoPEr6   r,   r'   Úrope)Úselfr>   ÚdimrH   rI   r   Ú	__class__s         €r;   rG   zAttention.__init__2   s;  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à—=‘=ˆØ˜t‘^ˆŒ
ä—i‘i  W¨xÑ%7¸d×>QÑ>QÔRˆŒÜ—i‘i  Z°(Ñ%:À×ATÑATÔUˆŒÜ—i‘i  Z°(Ñ%:À×ATÑATÔUˆŒÜ—i‘i ¨(Ñ 2°C¸eÔDˆŒà×+Ñ+ˆÔØ×ÒÜŸ*™* X°4×3DÑ3DÔEˆDŒKÜŸ*™* X°4×3DÑ3DÔEˆDŒKä—G‘GÜ�˜4×5Ñ5Ñ5Ó6ØØ—‘ô
ˆ�	r:   ÚxÚmaskÚcacheÚreturnc                 ó–  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      }|j	                  ||| j                  d«      }| j                  r"| j                  |«      }| j                  |«      }|j                  dddd«      }|j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j	                  ||d«      }
| j!                  |
«      S )Néÿÿÿÿr   é   r   é   )Úoffset)r[   rJ   rZ   )ÚshaperM   rN   rO   ÚreshaperH   rI   r)   rR   rS   Ú	transposerU   ra   Úupdate_and_fetchr   rJ   rP   )rV   rY   rZ   r[   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r;   Ú__call__zAttention.__call__L   s”  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆà—/‘/ ! Q¨¯©°bÓ9ˆØ�|‰|˜A˜q $§/¡/°2Ó6ˆà×ÒØ—k‘k 'Ó*ˆGØ—;‘;˜tÓ$ˆDà×#Ñ# A q¨!¨QÓ/ˆØ�~‰~˜a  A qÓ)ˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆØÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r:   ©NN©r1   r2   r3   r   rG   ÚmxÚarrayr   r   rm   Ú__classcell__©rX   s   @r;   r=   r=   1   sW   ø„ ð
˜Yõ 
ð: $(Ø#ñ	 #à�8‰8ð #ð �r—x‘xÑ ð #ð ˜‰}ð	 #ð
 
�‰÷ #r:   r=   c                   ó6   ‡ — e Zd Z	 ddededefˆ fd„Zd„ Zˆ xZS )ÚMLPÚconfigr   r   c                 óª  •— t         ‰| �  «        || _        |€|j                  n|| _        |€|j                  n|| _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        y ©NFr@   )
rF   rG   rv   r   r   rK   rL   Ú	gate_projÚup_projÚ	down_proj)rV   rv   r   r   rX   s       €r;   rG   zMLP.__init__p   sª   ø€ ô 	‰ÑÔØˆŒØ1<Ð1D˜6×-Ò-È+ˆÔà(9Ð(AˆF×$Ò$ÐGXð 	Ôô Ÿ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆ�r:   c                 óx   — | j                  t        | j                  |«      | j                  |«      «      «      }|S ©N)r{   r   ry   rz   )rV   rY   r{   s      r;   rm   zMLP.__call__~   s.   € Ø—N‘N¤6¨$¯.©.¸Ó*;¸T¿\¹\È!»_Ó#MÓNˆ	ØÐr:   rn   )r1   r2   r3   r   r6   rG   rm   rr   rs   s   @r;   ru   ru   o   s,   ø„ àSWñYØðYØ.1ðYØMPõYör:   ru   c                 óÎ  — t        j                  | j                  t         j                  «      «      }|}||z   }|dkD  rÃt        j                  |d|df¬«      }t        j
                  |dd¬«      j                  dd¬«      }	||z
  }
t        j                  |	|
dz
  d¬	«      d
d |
…d d …f   }t        j                  |t        j                  |«      t        j                  d«      d¬«      }t        j                  |dd«      }|}
t        j                  | |
dz
  d¬	«      d
d |
…f   }t        j                  ||d¬«      }|dkD  r|r|j                  dd¬«      }||z  }||z  }||fS )Nr   r^   )Úaxisrb   r_   ©r   T)r   Úkeepdimséþÿÿÿ)Úkthr   .g        )rp   r-   ÚastypeÚfloat32Ú	unflattenÚtopkÚsumÚargpartitionÚput_along_axisÚstop_gradientrq   ÚflattenÚtake_along_axis)ÚgatesÚe_score_correction_biasÚtop_kr   r   r!   r   ÚscoresÚorig_scoresÚgroup_scoresÚkÚ	group_idxÚindsÚdenominators                 r;   Úgroup_expert_selectr˜   ƒ   sU  € ô �Z‰Z˜Ÿ™¤R§Z¡ZÓ0Ó1€FØ€KØÐ-Ñ-€FØ�‚{Ü—‘˜f¨2°g¸r°]ÔCˆÜ—w‘w˜v q¨rÔ2×6Ñ6¸BÈÐ6ÓNˆØ�jÑ ˆÜ—O‘O L°a¸!±eÀ"ÔEÀcÈ2ÈAÈ2ÊqÀjÑQˆ	Ü×"Ñ"Ø”B×$Ñ$ YÓ/´·±¸#³ÀRô
ˆô —‘˜F B¨Ó+ˆà€AÜ�?‰?˜F˜7¨¨A©°BÔ7¸¸R¸a¸R¸Ñ@€DÜ×Ñ ¨T¸Ô;€FØˆq‚y‘^Ø—j‘j b°4�jÓ8ˆØ˜+Ñ%ˆØÐ+Ñ+€Fà�ˆ<Ðr:   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚMoEGaterv   c                 óÒ  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  |j                  f«      | _        t        j                  | j                  f«      | _        |j                  dk(  sJ d«       ‚y )Nr/   zUnsupported topk method.)rF   rG   rv   r"   r�   r   r    r!   r   r   rp   Úzerosr   Úweightr�   r0   ©rV   rv   rX   s     €r;   rG   zMoEGate.__init__§   s¹   ø€ Ü‰ÑÔØˆŒØ×/Ñ/ˆŒ
Ø$×3Ñ3ˆÔØ &× 7Ñ 7ˆÔØ%+×%AÑ%AˆÔ"Ø—~‘~ˆŒØ ×+Ñ+ˆŒÜ—h‘h × 5Ñ 5°v×7IÑ7IÐJÓKˆŒÜ')§x¡x°×1FÑ1FÐ0HÓ'IˆÔ$Ø×!Ñ! ZÒ/ÐKÐ1KÓKÑ/r:   c           	      óÊ   — t        || j                  j                  z  | j                  | j                  | j
                  | j                  | j                  | j                  «      S r}   )	r˜   r�   ÚTr�   r�   r   r   r!   r   )rV   rY   s     r;   rm   zMoEGate.__call__´   sO   € Ü"Ø�—‘—‘ÑØ×(Ñ(Ø�J‰JØ�L‰LØ�O‰OØ×&Ñ&Ø×Ñó
ð 	
r:   ©r1   r2   r3   r   rG   rm   rr   rs   s   @r;   rš   rš   ¦   s   ø„ ðL˜yõ Lö	
r:   rš   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚMoErv   c                 óN  •— t         ‰| �  «        || _        |j                  | _        t	        |j
                  |j                  |j                  «      | _        t        |«      | _
        |j                  �+|j                  |j                  z  }t        ||¬«      | _        d | _        y )N)rv   r   )rF   rG   rv   r"   r   r   r   r    Ú
switch_mlprš   Úgater   ru   Úshared_expertsÚsharding_group)rV   rv   r   rX   s      €r;   rG   zMoE.__init__Á   s—   ø€ Ü‰ÑÔØˆŒØ#)×#=Ñ#=ˆÔ Ü#Ø×ÑØ×(Ñ(Ø×#Ñ#ó
ˆŒô ˜F“OˆŒ	Ø×"Ñ"Ð.Ø &× <Ñ <¸v×?VÑ?VÑ VÐÜ"%ØÐ1Bô#ˆDÔð #ˆÕr:   c                 óÄ  — | j                   � t        | j                   «      |«      }| j                  |«      \  }}| j                  ||«      }||d   z  j	                  d¬«      j                  |j                  «      }| j                  j                  �|| j                  |«      z   }| j                   �+t        j                  j                  || j                   ¬«      }|S )N).Nr‚   r€   ©Úgroup)r¨   r
   r¦   r¥   rˆ   r„   Údtyperv   r   r§   rp   ÚdistributedÚall_sum)rV   rY   r–   r‘   Úys        r;   rm   zMoE.__call__Ô   sÇ   € Ø×ÑÐ*Ø2”˜d×1Ñ1Ó2°1Ó5ˆAà—y‘y “|‰ˆˆfØ�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0×7Ñ7¸¿¹Ó@ˆØ�;‰;×'Ñ'Ð3Ø�D×'Ñ'¨Ó*Ñ*ˆAà×ÑÐ*Ü—‘×&Ñ& q°×0CÑ0CÐ&ÓDˆAàˆr:   r¡   rs   s   @r;   r£   r£   À   s   ø„ ð#˜yõ #ö&r:   r£   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚDecoderLayerrv   Ú	layer_idxc                 ón  •— t         ‰| �  «        t        |«      | _        |j                  �||j
                  k\  rt        |«      n
t        |«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        y ©NrB   )rF   rG   r=   Ú	self_attnr    r#   r£   ru   ÚmlprK   rQ   r   r&   Úinput_layernormÚpost_attention_layernorm)rV   rv   r²   rX   s      €r;   rG   zDecoderLayer.__init__å   s’   ø€ Ü‰ÑÔÜ" 6Ó*ˆŒð ×'Ñ'Ð3Ø ×!=Ñ!=Ò=ô �ŒKô
 �V“ð 	Œô  "Ÿz™z¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
©
Ø×Ñ F×$7Ñ$7ô)
ˆÕ%r:   rY   rZ   r[   r\   c                 óš   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   S r}   )rµ   r·   r¶   r¸   )rV   rY   rZ   r[   ÚrÚhs         r;   rm   zDecoderLayer.__call__õ   sL   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�1‰uˆr:   rn   )r1   r2   r3   r   r6   rG   rp   rq   r   r   rm   rr   rs   s   @r;   r±   r±   ä   s^   ø„ ð
˜yð 
°Sõ 
ð& $(Ø#ñ		à�8‰8ð	ð �r—x‘xÑ ð	ð ˜‰}ð		ð
 
�‰÷	r:   r±   c                   ój   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   dej                  fd„Z	ˆ xZ
S )ÚLanguageModelrv   c                 óp  •— t         ‰| �  «        |j                  | _        t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}| _
        t        j                  |j
                  |j                  ¬«      | _        y c c}w r´   )rF   rG   r   rK   Ú	Embeddingr   Úembed_tokensÚranger$   r±   ÚlayersrQ   r&   Únorm)rV   rv   ÚidxrX   s      €r;   rG   zLanguageModel.__init__  s�   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜŸL™L¨×):Ñ):¸F×<NÑ<NÓOˆÔä16°v×7OÑ7OÔ1Pó
Ù1P¨#ŒL˜ Õ%Ð1Pñ
ˆŒô —J‘J˜v×1Ñ1°v×7JÑ7JÔKˆ�	ùò
s   Á'B3rY   r[   r\   c                 ó�  — | j                  |«      }| j                  }| j                  }|€d gt        | j                  «      z  }t        ||d   «      }||dz
  k  r#t        j                  j                  ||dz   «      }t        | j                  |«      D ]  \  }} ||||¬«      }Œ |dk7  rVt        j                  j                  ||dz
  |z  «      }|d   �+t        j                  |d   j                  |«      |d   _        |dkD  r/t        j                  j                  |«      d |j                  d    }| j                  |«      S )Nr   r   )r[   r^   )rÀ   Úpipeline_rankÚpipeline_sizeÚlenÚpipeline_layersr   rp   r­   Ú	recv_likeÚzipÚsendÚdependsrj   Ú
all_gatherrb   rÃ   )	rV   rY   r[   r»   rÆ   rÇ   rZ   ÚlÚcs	            r;   rm   zLanguageModel.__call__  s6  € ð
 ×Ñ˜aÓ ˆà×*Ñ*ˆØ×*Ñ*ˆàˆ=Ø�FœS ×!5Ñ!5Ó6Ñ6ˆEÜ$ Q¨¨a©Ó1ˆð ˜=¨1Ñ,Ò,Ü—‘×(Ñ(¨¨]¸QÑ->Ó@ˆAä˜×,Ñ,¨eÖ4‰DˆAˆqÙ�!�T Ô#‰Að 5ð ˜AÒÜ—‘×#Ñ# A¨¸Ñ(9¸]Ñ'JÓKˆAØ�R‰yÐ$Ü!#§¡¨E°"©I¯N©N¸AÓ!>��b‘	”ð ˜1ÒÜ—‘×)Ñ)¨!Ó,¨\¨q¯w©w°q©zÐ:ˆAà�y‰y˜‹|Ðr:   r}   ro   rs   s   @r;   r½   r½     sC   ø„ ðL˜yõ Lð  $ñà�8‰8ðð ˜‰}ðð 
�‰÷	r:   r½   c                   ó´   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   fd„Z	d„ Z
ddeej                  j                     fd„Zed	„ «       Zed
„ «       Zˆ xZS )ÚModelrv   c                 óÔ   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        t        j                  |j                  |j                  d¬«      | _
        y rx   )rF   rG   r>   r   r½   ÚmodelrK   rL   r   r   Úlm_headrž   s     €r;   rG   zModel.__init__.  sN   ø€ Ü‰ÑÔØˆŒ	Ø ×+Ñ+ˆŒÜ" 6Ó*ˆŒ
Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆ�r:   Úinputsr[   c                 óH   — | j                  ||«      }| j                  |«      S r}   )rÔ   rÕ   )rV   rÖ   r[   Úouts       r;   rm   zModel.__call__5  s#   € ð
 �j‰j˜ Ó'ˆØ�|‰|˜CÓ Ð r:   c                 ó  — | j                   j                  }t        | j                   j                  «      D ]‹  }d|› �}dD ]  \  }}dD ]u  }|› d|› d|› �|v sŒt        | j                   j                  «      D �cg c]  }|j	                  |› d|› d|› d|› �«      ‘Œ  }	}t        j                  |	«      ||› d|› d|› �<   Œw Œ� Œ� |j                  «       D ��
ci c]  \  }}
|j                  d|› �«      s||
“Œ c}
}S c c}w c c}
}w )Nzmodel.layers.))Úw1ry   )Úw2r{   )Úw3rz   )r�   ÚscalesÚbiasesz.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.)	r>   r$   rÁ   r    Úpoprp   ÚstackÚitemsÚ
startswith)rV   ÚweightsÚ	mpt_layerrÏ   ÚprefixÚnÚmr”   ÚeÚto_joinÚvs              r;   ÚsanitizezModel.sanitize=  s1  € Ø—I‘I×/Ñ/ˆ	ô �t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛU‘��1Û7�AØ ˜ °°°1°Q°CÐ8¸GÒCô &+¨4¯9©9×+EÑ+EÔ%Fó#á%F ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÕLØ%Fð  ð #ô GIÇhÁhÈwÓFW˜ 6 (Ð*:¸1¸#¸Q¸q¸cÐ BÒCñ 8ñ Vð 4ð  Ÿ™œô
á'‘��1Ø—<‘< -°	¨{Ð ;Ô<ð ˆq‰DØ'ò
ð 	
ùò#ùó
s   Á9#C<
Ã!Dr«   c                 ó6  — |xs t         j                  j                  «       }|j                  «       }| j                  j
                  D �]Í  }t        |j                  j                  d|¬«      |j                  _        t        |j                  j                  d|¬«      |j                  _	        t        |j                  j                  d|¬«      |j                  _
        t        |j                  j                  d|¬«      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        t        |j                  t         «      r•t        |j                  j"                  d|¬«      |j                  _        t        |j                  j$                  d|¬«      |j                  _        t        |j                  j&                  d|¬«      |j                  _        �Œµ||j                  _        t+        |j                  j,                  j"                  d|¬«       t+        |j                  j,                  j$                  d|¬«       t+        |j                  j,                  j&                  d|¬«       t+        |j                  j.                  j"                  d|¬«       t+        |j                  j.                  j$                  d|¬«       t+        |j                  j.                  j&                  d|¬«       �ŒÐ y )Nzall-to-shardedrª   zsharded-to-all)rp   r­   ÚinitÚsizerÔ   rÂ   r	   rµ   rM   rN   rO   rP   rH   rI   Ú
isinstancer¶   ru   ry   r{   rz   r¨   r   r§   r¥   )rV   r«   ÚNÚlayers       r;   ÚshardzModel.shardS  sN  € ØÒ.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ—Z‘Z×&Õ&ˆEä%1Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ð �O‰O×#Ò#¨Ñ)Õ#Ø�O‰O×&Ò&¨1Ñ,Õ&ô ˜%Ÿ)™)¤SÔ)Ü&2Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô '3Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô %1Ø—I‘I×%Ñ%Ð'7¸uô%�—	‘	Ö!ð ,1�—	‘	Ô(ÜØ—I‘I×,Ñ,×6Ñ6Ð8HÐPUõô Ø—I‘I×,Ñ,×6Ñ6Ð8HÐPUõô Ø—I‘I×,Ñ,×4Ñ4Ð6FÈeõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×0Ñ0Ð2BÈ%÷ña 'r:   c                 ó.   — | j                   j                  S r}   )rÔ   rÉ   )rV   s    r;   rÂ   zModel.layersŠ  s   € à�z‰z×)Ñ)Ð)r:   c                 ó   — d„ }|S )Nc                 ó
   — d| vS )Nr�   r9   )r”   s    r;   Ú	predicatez'Model.cast_predicate.<locals>.predicate�  s   € Ø,°AÐ5Ð5r:   r9   )rV   r÷   s     r;   Úcast_predicatezModel.cast_predicateŽ  s   € ò	6ð Ðr:   r}   )r1   r2   r3   r   rG   rp   rq   r   r   rm   rì   r­   ÚGroupró   ÚpropertyrÂ   rø   rr   rs   s   @r;   rÒ   rÒ   -  s~   ø„ ðT˜yõ Tð  $ñ!à—‘ð!ð ˜‰}ó!ò
ñ,5˜8 B§N¡N×$8Ñ$8Ñ9ó 5ðn ñ*ó ð*ð ñó ôr:   rÒ   )'ÚmathÚdataclassesr   Ú	functoolsr   Útypingr   r   r   Úmlx.coreÚcorerp   Úmlx.nnrK   Úmlx.nn.layers.distributedr   r	   r
   Úactivationsr   rE   r   r   r   Úpipeliner   Úswitch_layersr   r   ÚModuler=   ru   Úcompiler˜   rš   r£   r±   r½   rÒ   r9   r:   r;   Ú<module>r     sÜ   ðó Ý !Ý ß &Ñ &å Ý ß PÑ På ß TÑ TÝ #Ý $ð ô"�ó "ó ð"ô<;#�—	‘	ô ;#ô|ˆ"�)‰)ô ð( ‡�ñó ðôD
ˆb�i‰iô 
ô4!ˆ"�)‰)ô !ôH�2—9‘9ô ô:)�M 2§9¡9ô )ôXfˆB�I‰Iõ fr:   