Ë
    îÿæiž2  ã                   óÚ  — d dl Z d dlmZ d dlmZmZmZmZmZ d dl	m
Z d dlmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ dd	lmZ e G d
„ de«      «       Z G d„ dej8                  «      Z G d„ dej8                  «      Z G d„ dej8                  «      Z G d„ dej8                  «      Z  G d„ dej8                  «      Z! G d„ dej8                  «      Z" G d„ dej8                  «      Z#y)é    N)Ú	dataclass)ÚAnyÚDictÚListÚOptionalÚUnioné   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache)Úinitialize_rope)Ú	SwitchGLUc                   ó’  — e Zd ZU eed<   ee   ed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   dZeed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d&Zeed'<   d(Zeed)<   d Z eed*<   d Z!eed+<   dZ"eed,<   d$Z#eed-<   y).Ú	ModelArgsÚ
model_typeÚlayer_typesi  Ú
vocab_sizei   Úhidden_sizei   Úintermediate_sizei   Úmoe_intermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsé@   Úhead_dimi   Úmax_position_embeddingsgñhãˆµøä>Úrms_norm_epsi'  Ú
rope_thetaNÚrope_scalingFÚtie_word_embeddingsé€   Únum_expertsé   Únum_experts_per_tokr	   Únum_shared_expertsé   Únum_dense_layersTÚ
route_normgÏ÷Sã¥›@Úroute_scaleÚsigmoidÚ
score_funcÚn_groupÚ
topk_groupÚsliding_windowÚmup_enabled)$Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__r   r   Úintr   r   r   r   r   r   r    r!   r"   Úfloatr#   r$   r   r   r   r%   Úboolr'   r)   r*   r,   r-   r.   r0   r1   r2   r3   r4   © ó    úh/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/afmoe.pyr   r      s#  … àƒOØ�c‘ÓØ€J�ÓØ€K�ÓØ!Ð�sÓ!Ø!%Ð˜3Ó%ØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø€HˆcÓØ#)Ð˜SÓ)Ø€L�%ÓØ€J�ÓØ;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?Ø %Ð˜Ó%à€K�ÓØ Ð˜Ó ØÐ˜ÓØÐ�cÓØ€J�ÓØ€K�ÓØ€J�ÓØ€GˆSÓØ€J�ÓØ€N�CÓØ€K�Ôr>   r   c            	       ó�   ‡ — e Zd Zd	dedefˆ fd„Z	 	 d
dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )Ú	AttentionÚargsÚis_local_attentionc                 ó˜  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        || _        | j                  dz  | _	        t        j                  | j                  | j                  | j                  z  d¬«      | _        t        j                  | j                  | j                  | j                  z  d¬«      | _        t        j                  | j                  | j                  | j                  z  d¬«      | _        t        j                  | j                  | j                  z  | j                  d¬«      | _        t        j                   | j                  |j"                  ¬«      | _        t        j                   | j                  |j"                  ¬«      | _        t        j                  | j                  | j                  | j                  z  d¬«      | _        |r=t+        | j                  |j,                  d|j.                  |j0                  «      | _        y d | _        y )Ng      à¿F©Úbias©Úeps)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsr    rC   ÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚRMSNormr"   Úq_normÚk_normÚ	gate_projr   r#   r$   r!   Úrope)ÚselfrB   rC   Ú	__class__s      €r?   rJ   zAttention.__init__1   s¨  ø€ Ü‰ÑÔà×+Ñ+ˆÔØ×/Ñ/ˆŒØ×2Ñ2ˆŒØŸ™ˆŒØ"4ˆÔà—]‘] DÑ(ˆŒ
ä—i‘iØ×Ñ˜dŸl™l¨T¯]©]Ñ:Àô
ˆŒô —i‘iØ×Ñ˜dŸo™o°·±Ñ=ÀEô
ˆŒô —i‘iØ×Ñ˜dŸo™o°·±Ñ=ÀEô
ˆŒô —i‘iØ�L‰L˜4Ÿ=™=Ñ(¨$×*:Ñ*:Àô
ˆŒô —j‘j §¡°D×4EÑ4EÔFˆŒÜ—j‘j §¡°D×4EÑ4EÔFˆŒäŸ™Ø×Ñ˜dŸl™l¨T¯]©]Ñ:Àô
ˆŒñ Ü'Ø—‘Ø—‘ØØ×!Ñ!Ø×,Ñ,óˆD�Ið ˆD�Ir>   ÚxÚmaskÚcacheÚreturnc                 ó8  — |j                   \  }}}| j                  |«      }| j                  |«      }| j                  |«      }	|j	                  ||| j
                  | j                  «      j                  dddd«      }|j	                  ||| j                  | j                  «      j                  dddd«      }|	j	                  ||| j                  | j                  «      j                  dddd«      }	| j                  |«      }| j                  |«      }| j                  rk| j                  �_|�;| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }n"| j                  |«      }| j                  |«      }|�|j                  ||	«      \  }}	t        |||	|| j                   |¬«      }
|
j                  dddd«      j	                  ||d«      }
t#        j$                  | j'                  |«      «      }|
|z  }
| j)                  |
«      S )Nr   r+   r	   é   )Úoffset)r]   rM   r\   éÿÿÿÿ)ÚshaperP   rQ   rR   ÚreshaperK   r    Ú	transposerL   rU   rV   rC   rX   ra   Úupdate_and_fetchr   rM   Úmxr/   rW   rS   )rY   r[   r\   r]   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputÚgates               r?   Ú__call__zAttention.__call__[   sÕ  € ð —'‘'‰ˆˆ1ˆaà—+‘+˜a“.ˆØ�{‰{˜1‹~ˆØ—‘˜Q“ˆà—/‘/ ! Q¨¯©°d·m±mÓD×NÑNØˆq�!�Qó
ˆð �|‰|˜A˜q $§/¡/°4·=±=ÓA×KÑKÈAÈqÐRSÐUVÓWˆØ—‘  1 d§o¡o°t·}±}ÓE×OÑOØˆq�!�Qó
ˆð —+‘+˜gÓ&ˆØ�{‰{˜4Ó ˆà×"Ò" t§y¡yÐ'<ØÐ ØŸ)™) G°E·L±L˜)ÓA�Ø—y‘y ¨e¯l©l�yÓ;‘àŸ)™) GÓ,�Ø—y‘y “�àÐØ ×1Ñ1°$¸Ó?‰LˆD�&ä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆä�z‰z˜$Ÿ.™.¨Ó+Ó,ˆØ˜$‘ˆà�{‰{˜6Ó"Ð"r>   ©F©NN)r5   r6   r7   r   r<   rJ   rg   Úarrayr   r   rp   Ú__classcell__©rZ   s   @r?   rA   rA   0   s_   ø„ ñ(˜Yð (¸Dõ (ðZ $(Ø#ñ	+#à�8‰8ð+#ð �r—x‘xÑ ð+#ð ˜‰}ð	+#ð
 
�‰÷+#r>   rA   c                   óP   ‡ — e Zd Zddedee   fˆ fd„Zdej                  fd„Z	ˆ xZ
S )ÚMLPrB   r   c                 ó  •— t         ‰| �  «        |j                  }|�|n|j                  }t	        j
                  ||d¬«      | _        t	        j
                  ||d¬«      | _        t	        j
                  ||d¬«      | _        y ©NFrE   )	rI   rJ   r   r   rN   rO   rW   Ú	down_projÚup_proj)rY   rB   r   ÚdimÚ
hidden_dimrZ   s        €r?   rJ   zMLP.__init__Š   sr   ø€ Ü‰ÑÔà×Ñˆð !Ð,ñ à×'Ñ'ð 	ô Ÿ™ 3¨
¸Ô?ˆŒÜŸ™ :¨s¸Ô?ˆŒÜ—y‘y  j°uÔ=ˆ�r>   r^   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)rz   r
   rW   r{   ©rY   r[   s     r?   rp   zMLP.__call__˜   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr>   r   )r5   r6   r7   r   r   r:   rJ   rg   rs   rp   rt   ru   s   @r?   rw   rw   ‰   s,   ø„ ñ>˜Yð >¸8ÀC¹=õ >ðJ˜RŸX™X÷ Jr>   rw   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )Ú	MoERouterz;Router module that wraps the gate for proper weight naming.rB   c                 ó„   •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        y ry   )rI   rJ   rN   rO   r   r'   ro   ©rY   rB   rZ   s     €r?   rJ   zMoERouter.__init__Ÿ   s.   ø€ Ü‰ÑÔÜ—I‘I˜d×.Ñ.°×0@Ñ0@ÀuÔMˆ�	r>   r[   r^   c                 ó$   — | j                  |«      S r   )ro   r€   s     r?   rp   zMoERouter.__call__£   s   € Ø�y‰y˜‹|Ðr>   )
r5   r6   r7   Ú__doc__r   rJ   rg   rs   rp   rt   ru   s   @r?   r‚   r‚   œ   s/   ø„ ÙEðN˜Yõ Nð˜"Ÿ(™(ð  r§x¡x÷ r>   r‚   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚAfmoeMoErB   c                 ó^  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        |«      | _        t        j                  |j                  f«      | _        t        |j                   |j"                  |j                  «      | _        |j&                  dkD  r,|j"                  |j&                  z  }t)        ||¬«      | _        y y )Nr   )r   )rI   rJ   rB   r'   r)   r-   r.   r0   r1   r2   r‚   Úrouterrg   ÚzerosÚexpert_biasr   r   r   Úexpertsr*   rw   Úshared_experts)rY   rB   Úshared_intermediate_sizerZ   s      €r?   rJ   zAfmoeMoE.__init__¨   s÷   ø€ Ü‰ÑÔØˆŒ	Ø×+Ñ+ˆÔØ#'×#;Ñ#;ˆÔ ØŸ/™/ˆŒØ×+Ñ+ˆÔØŸ/™/ˆŒØ—|‘|ˆŒØŸ/™/ˆŒä “oˆŒäŸ8™8 T×%5Ñ%5Ð$7Ó8ˆÔä Ø×ÑØ×&Ñ&Ø×Ñó
ˆŒð ×"Ñ" QÒ&à×*Ñ*¨T×-DÑ-DÑDð %ô #& dÐ>VÔ"WˆDÕð	 'r>   r[   r^   c                 ó  — | j                  |«      }| j                  dk(  r3t        j                  |j	                  t        j
                  «      «      }n4t        j                  |j	                  t        j
                  «      d¬«      }|| j                  z   }| j                  dkD  rát        j                  |d| j                  df¬«      }t        j                  |dd¬«      j                  dd¬«      }| j                  | j                  z
  }t        j                  ||dz
  d	¬
«      dd |…d d …f   }t        j                  |t        j                  |«      t        j                   d«      d	¬«      }t        j"                  |d	d«      }| j$                  }t        j                  | |dz
  d¬
«      dd |…f   }t        j&                  ||d¬«      }	| j(                  r'| j$                  dkD  r|	j                  dd¬«      }
|	|
z  }	|	| j*                  z  }	| j-                  ||«      }||	d   z  j                  d	¬«      j	                  |j.                  «      }| j0                  j2                  dkD  r|| j5                  |«      z   }|S )Nr/   rb   )Úaxisr	   )r‘   rc   r+   T)r‘   Úkeepdimséþÿÿÿ)Úkthr‘   .g        ).Nr   )rŠ   r0   rg   r/   ÚastypeÚfloat32ÚsoftmaxrŒ   r1   Ú	unflattenÚtopkÚsumr2   ÚargpartitionÚput_along_axisÚstop_gradientrs   Úflattenr)   Útake_along_axisr-   r.   r�   ÚdtyperB   r*   rŽ   )rY   r[   ÚgatesÚscoresÚselection_scoresÚgroup_scoresÚkÚ	group_idxÚindsÚselected_scoresÚdenominatorÚys               r?   rp   zAfmoeMoE.__call__Ã   s0  € Ø—‘˜A“ˆà�?‰?˜iÒ'Ü—Z‘Z §¡¬R¯Z©ZÓ 8Ó9‰Fä—Z‘Z §¡¬R¯Z©ZÓ 8¸rÔBˆFð " D×$4Ñ$4Ñ4Ðð �<‰<˜!ÒÜ!Ÿ|™|Ø  r°$·,±,ÀÐ1Cô Ðô Ÿ7™7Ð#3°Q¸RÔ@×DÑDØ $ð Eó ˆLð —‘˜tŸ™Ñ.ˆAÜŸ™¨¸!¸a¹%ÀbÔIÈ#ÈrÐPQÈrÒSTÈ*ÑUˆIÜ!×0Ñ0Ø ¤"×"2Ñ"2°9Ó"=¼r¿x¹xÈ»}ÐSUô Ðô  "Ÿz™zÐ*:¸BÀÓCÐð ×$Ñ$ˆÜ�‰Ð 0Ð0°a¸!±eÀ"ÔEÀcÈ2ÈAÈ2ÀgÑNˆä×,Ñ,¨V°TÀÔCˆà�?Š?˜t×7Ñ7¸!Ò;Ø)×-Ñ-°2ÀÐ-ÓEˆKØ-°Ñ;ˆOà)¨D×,<Ñ,<Ñ<ˆà�L‰L˜˜DÓ!ˆØ� Ñ+Ñ+×0Ñ0°bÐ0Ó9×@Ñ@ÀÇÁÓIˆà�9‰9×'Ñ'¨!Ò+Ø�D×'Ñ'¨Ó*Ñ*ˆAàˆr>   ©	r5   r6   r7   r   rJ   rg   rs   rp   rt   ru   s   @r?   rˆ   rˆ   §   s,   ø„ ðX˜Yõ Xð6,˜"Ÿ(™(ð , r§x¡x÷ ,r>   rˆ   c            	       ó”   ‡ — e Zd Zd
dededefˆ fd„Z	 	 ddej                  de	ej                     de	e
   dej                  fd	„Zˆ xZS )ÚDecoderLayerrB   Ú	layer_idxÚuse_slidingc                 ód  •— t         ‰| �  «        |j                  | _        || _        || _        t        ||¬«      | _        ||j                  k  rt        |«      | _	        nt        |«      | _	        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        y )N)rC   rG   )rI   rJ   r   r¯   r®   rA   Ú	self_attnr,   rw   Úmlprˆ   rN   rT   r"   Úinput_layernormÚpost_attention_layernormÚpre_mlp_layernormÚpost_mlp_layernorm)rY   rB   r®   r¯   rZ   s       €r?   rJ   zDecoderLayer.__init__ó   sÞ   ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ&ˆÔØ"ˆŒä" 4¸KÔHˆŒà�t×,Ñ,Ò,Ü˜4“yˆD�Hä “~ˆDŒHä!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ô "$§¡¨D×,<Ñ,<À$×BSÑBSÔ!TˆÔÜ"$§*¡*¨T×-=Ñ-=À4×CTÑCTÔ"UˆÕr>   r[   r\   r]   r^   c                 óÞ   — | j                  | j                  |«      ||«      }| j                  |«      }||z   }| j                  | j	                  |«      «      }| j                  |«      }||z   S r   )r±   r³   r´   r²   rµ   r¶   )rY   r[   r\   r]   ÚrÚhs         r?   rp   zDecoderLayer.__call__  sl   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ×)Ñ)¨!Ó,ˆØ�‰Eˆà�H‰H�T×+Ñ+¨AÓ.Ó/ˆØ×#Ñ# AÓ&ˆØ�1‰uˆr>   rq   rr   )r5   r6   r7   r   r:   r<   rJ   rg   rs   r   r   rp   rt   ru   s   @r?   r­   r­   ò   si   ø„ ñV˜Yð V°3ð VÀTõ Vð. $(Ø#ñ	à�8‰8ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷r>   r­   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )Ú
AfmoeModelrB   c           
      óð  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        t        j                  |j                  |j                  «      | _        t        | j
                  «      D ��cg c]  \  }}t        |||dk(  ¬«      ‘Œ c}}| _        t        j                  |j                  |j                   ¬«      | _        | j
                  j%                  d«      | _        d | _        t        | j                  «      D ]  \  }}|j*                  sŒ|| _         y  y c c}}w )NÚsliding_attention)rB   r®   r¯   rG   Úfull_attention)rI   rJ   rB   r   r   r   r3   r4   r   rN   Ú	EmbeddingÚembed_tokensÚ	enumerater­   ÚlayersrT   r"   ÚnormÚindexÚfa_idxÚswa_idxr¯   )rY   rB   ÚidxÚ
layer_typeÚlayerrZ   s        €r?   rJ   zAfmoeModel.__init__  s8  ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ×+Ñ+ˆÔØ"×1Ñ1ˆÔØ×+Ñ+ˆÔØ×+Ñ+ˆÔäŸL™L¨¯©¸$×:JÑ:JÓKˆÔô
 $-¨T×-=Ñ-=Ô#>ô	
ñ $?‘��Zô Ø S°jÐDWÑ6Wöð $?ò	
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	à×&Ñ&×,Ñ,Ð-=Ó>ˆŒØˆŒÜ# D§K¡KÖ0‰JˆC�Ø× Ó Ø"�”Ùñ 1ùó
s   ÃE2Úinputsc                 óâ  — | j                  |«      }| j                  r"|t        j                  | j                  «      z  }|€d gt        | j                  «      z  }t        ||| j                     «      }d }| j                  �%t        ||| j                     | j                  ¬«      }t        | j                  |«      D ]   \  }}|j                  r|n|} ||||¬«      }Œ" | j                  |«      S )N)Úwindow_size)r]   )rÀ   r4   ÚmathÚsqrtr   ÚlenrÂ   r   rÅ   rÆ   r3   Úzipr¯   rÃ   )	rY   rÊ   r]   r¹   Úfa_maskÚswa_maskrÉ   Úcr\   s	            r?   rp   zAfmoeModel.__call__1  sÜ   € ð
 ×Ñ˜fÓ%ˆà×ÒØ”D—I‘I˜d×.Ñ.Ó/Ñ/ˆAàˆ=Ø�FœS §¡Ó-Ñ-ˆEä'¨¨5°·±Ñ+=Ó>ˆØˆØ�<‰<Ð#Ü,Ø�5˜Ÿ™Ñ&°D×4GÑ4GôˆHô ˜DŸK™K¨Ö/‰HˆE�1Ø$×0Ò0‘8°gˆDÙ�a˜ QÔ'‰Að 0ð �y‰y˜‹|Ðr>   r   r«   ru   s   @r?   r»   r»     s%   ø„ ð˜Yõ ð: ñà—‘÷r>   r»   c                   ó„   ‡ — e Zd Zdefˆ fd„Z	 d
dej                  fd„Zd„ Ze	d„ «       Z
d„ Ze	d„ «       Ze	d	„ «       Zˆ xZS )ÚModelrB   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y ry   )rI   rJ   rB   r   r»   Úmodelr%   rN   rO   r   r   Úlm_headr„   s     €r?   rJ   zModel.__init__M  sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ Ó%ˆŒ
à×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r>   rÊ   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S r   )r×   rB   r%   rÀ   Ú	as_linearrØ   )rY   rÊ   r]   Úouts       r?   rp   zModel.__call__V  sW   € ð
 �j‰j˜ Ó'ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r>   c                 óD  — |j                  «       D ��ci c]  \  }}d|vsŒ||“Œ }}}| j                  j                  r|j                  dd «       t	        | j                  j
                  «      D ]¢  }|| j                  j                  k  rŒd|› �}dD ]|  }dD ]u  }|› d|› d|› �|v sŒt	        | j                  j                  «      D �cg c]  }|j                  |› d|› d|› d|› �«      ‘Œ  }}t        j                  |«      ||› d|› d|› �<   Œw Œ~ Œ¤ |S c c}}w c c}w )	Nzrotary_emb.inv_freqzlm_head.weightzmodel.layers.)r{   rz   rW   )ÚweightÚscalesÚbiasesz.mlp.experts.0.Ú.z.mlp.experts.)
ÚitemsrB   r%   ÚpopÚranger   r,   r'   rg   Ústack)	rY   Úweightsr¥   ÚvÚlÚprefixÚnÚeÚto_joins	            r?   ÚsanitizezModel.sanitizeb  s=  € à$+§M¡M¤OÔV¡O™D˜A˜qÐ7LÐTUÒ7U�1�a‘4 OˆÑVà�9‰9×(Ò(Ø�K‰KÐ(¨$Ô/ô �t—y‘y×2Ñ2Ö3ˆAØ�4—9‘9×-Ñ-Ò-ØØ$ Q CÐ(ˆFÛ:�Û7�AØ ˜ °°°1°Q°CÐ8¸GÒCô &+¨4¯9©9×+@Ñ+@Ô%Aó#á%A ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÕLØ%Að  ð #ô DFÇ8Á8ÈGÓCT˜ 6 (¨-¸°s¸!¸A¸3Ð ?Ò@ñ 8ñ ;ð	 4ð ˆùó' Wùò#s   ”D¡DÃ
#D
c                 ó.   — | j                   j                  S r   )r×   rÂ   )rY   s    r?   rÂ   zModel.layersy  s   € à�z‰z× Ñ Ð r>   c                 ó¨   — | j                   D �cg c]8  }|j                  r t        | j                  j                  ¬«      n	t        «       ‘Œ: c}S c c}w )N)Úmax_size)rÂ   r¯   r   r×   r3   r   )rY   rÉ   s     r?   Ú
make_cachezModel.make_cache}  sU   € ð Ÿšó
ñ %�ð ×$Ò$ô  ¨¯©×)BÑ)BÕCä“Yñð %ñ
ð 	
ùò 
s   �=Ac                 ó   — d„ }|S )Nc                 ó
   — d| vS )NrŒ   r=   )r¥   s    r?   Ú	predicatez'Model.cast_predicate.<locals>.predicate‰  s   € Ø ¨Ð)Ð)r>   r=   ©rY   ró   s     r?   Úcast_predicatezModel.cast_predicate‡  s   € ò	*ð Ðr>   c                 ó   — d„ }|S )Nc                 ó   — d| v rdddœS y)Nzrouter.gater   r(   )Ú
group_sizeÚbitsTr=   )ÚpathÚ_s     r?   ró   z(Model.quant_predicate.<locals>.predicate�  s   € Ø Ñ$Ø&(°!Ñ4Ð4Ør>   r=   rô   s     r?   Úquant_predicatezModel.quant_predicateŽ  s   € ò	ð
 Ðr>   r   )r5   r6   r7   r   rJ   rg   rs   rp   rì   ÚpropertyrÂ   rð   rõ   rü   rt   ru   s   @r?   rÕ   rÕ   L  sm   ø„ ðT˜Yõ Tð ñ
à—‘ó
òð. ñ!ó ð!ò
ð ñó ðð ñó ôr>   rÕ   )$rÍ   Údataclassesr   Útypingr   r   r   r   r   Úmlx.coreÚcorerg   Úmlx.nnrN   Úactivationsr
   Úbaser   r   r   r]   r   r   Ú
rope_utilsr   Úswitch_layersr   r   ÚModulerA   rw   r‚   rˆ   r­   r»   rÕ   r=   r>   r?   Ú<module>r     sÂ   ðó Ý !ß 3Õ 3å Ý å ß TÑ Tß +Ý 'Ý $ð ô�ó ó ðô<V#�—	‘	ô V#ôrJˆ"�)‰)ô Jô&�—	‘	ô ôHˆr�y‰yô HôV!�2—9‘9ô !ôH3�—‘ô 3ôlIˆB�I‰Iõ Ir>   