Ë
    îÿæiQ2  ã                   óò  — d dl Z d dlmZ d dlmZmZmZmZ d dlm	Z
 d dlmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZ e G d	„ d
e«      «       Z G d„ dej4                  «      Z G d„ dej4                  «      Ze
j:                  d„ «       Z G d„ dej4                  «      Z G d„ dej4                  «      Z  G d„ dej4                  «      Z! G d„ dej4                  «      Z" G d„ dej4                  «      Z#y)é    N)Ú	dataclass)ÚAnyÚListÚOptionalÚUnioné   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_maskÚscaled_dot_product_attention)ÚArraysCacheÚKVCache)Ú	SwitchGLUc                   ó$  — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeeef   ed<   dZ	e
ed<   dZe
ed<   dZeee      ed<   dZe
ed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚattn_layer_offsetÚattn_layer_periodÚexpert_layer_offsetÚexpert_layer_periodÚmamba_d_convÚmamba_d_stateÚmamba_expandÚnum_expertsÚnum_experts_per_tokÚrms_norm_epsÚmax_position_embeddingsÚ
vocab_sizeÚautoÚmamba_dt_rankFÚmamba_proj_biasTÚmamba_conv_biasNÚlayers_block_typeÚtie_word_embeddingsc                 ó   — | j                   dk(  r't        j                  | j                  dz  «      | _         | j                  €Gt        | j                  «      D �cg c]"  }|| j                  z  | j                  k(  rdnd‘Œ$ c}| _        y y c c}w )Nr%   é   Ú	attentionÚmamba)	r&   ÚmathÚceilr   r)   Úranger   r   r   )ÚselfÚis     úh/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/jamba.pyÚ__post_init__zModelArgs.__post_init__/   s–   € Ø×Ñ Ò'Ü!%§¡¨4×+;Ñ+;¸bÑ+@Ó!AˆDÔØ×!Ñ!Ð)ô ˜t×5Ñ5Ô6ó&ñ 7�Að ˜4×1Ñ1Ñ1°T×5KÑ5KÒKñ  à ñ!ð 7ñ&ˆDÕ"ð *ùò&s   Á'B)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr&   r   r'   Úboolr(   r)   r   r   r*   r5   © ó    r4   r   r      s¶   … àƒOØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØ Ó ØƒOØ%+€M�5˜˜c˜‘?Ó+Ø!€O�TÓ!Ø €O�TÓ Ø-1Ð�x  S¡	Ñ*Ó1Ø $Ð˜Ó$ór?   r   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚJambaMLPÚargsc                 óH  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        y ©NF©Úbias)	ÚsuperÚ__init__ÚnnÚLinearr   r   Ú	gate_projÚup_projÚ	down_proj©r2   rB   Ú	__class__s     €r4   rH   zJambaMLP.__init__>   sq   ø€ Ü‰ÑÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆ�r?   ÚxÚreturnc                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)rM   r	   rK   rL   )r2   rP   s     r4   Ú__call__zJambaMLP.__call__D   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr?   ©	r6   r7   r8   r   rH   ÚmxÚarrayrT   Ú__classcell__©rO   s   @r4   rA   rA   =   s/   ø„ ðY˜Yõ YðJ˜"Ÿ(™(ð J r§x¡x÷ Jr?   rA   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚJambaAttentionrB   c                 óº  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  |j                  z  | _        | j
                  dz  | _        t        j                  |j                  |j                  | j
                  z  d¬«      | _	        t        j                  |j                  |j                  | j
                  z  d¬«      | _
        t        j                  |j                  |j                  | j
                  z  d¬«      | _        t        j                  |j                  | j
                  z  |j                  d¬«      | _        y )Ng      à¿FrE   )rG   rH   r   r   r   Úhead_dimÚscalerI   rJ   Úq_projÚk_projÚv_projÚo_projrN   s     €r4   rH   zJambaAttention.__init__I   s  ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø#'×#;Ñ#;ˆÔ Ø×(Ñ(¨D×,DÑ,DÑDˆŒØ—]‘] DÑ(ˆŒ
ä—i‘iØ×Ñ˜d×6Ñ6¸¿¹ÑFÈUô
ˆŒô —i‘iØ×Ñ˜d×6Ñ6¸¿¹ÑFÈUô
ˆŒô —i‘iØ×Ñ˜d×6Ñ6¸¿¹ÑFÈUô
ˆŒô —i‘iØ×$Ñ$ t§}¡}Ñ4°d×6FÑ6FÈUô
ˆ�r?   rP   ÚmaskÚcacherQ   c                 óx  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      j                  dddd«      }|j	                  ||| j                  d«      j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	|�|j                  ||	«      \  }}	t        |||	|| j                  |¬«      }
|
j                  dddd«      j	                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   é   r   é   )rd   r^   rc   )Úshaper_   r`   ra   Úreshaper   Ú	transposer   Úupdate_and_fetchr   r^   rb   )r2   rP   rc   rd   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r4   rT   zJambaAttention.__call__]   s2  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆà—/‘/ ! Q¨×(@Ñ(@À"ÓE×OÑOØˆq�!�Qó
ˆð �|‰|˜A˜q $×":Ñ":¸BÓ?×IÑIÈ!ÈQÐPQÐSTÓUˆØ—‘  1 d×&>Ñ&>ÀÓC×MÑMØˆq�!�Qó
ˆð ÐØ ×1Ñ1°$¸Ó?‰LˆD�&ä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r?   ©NN©r6   r7   r8   r   rH   rV   rW   r   r   rT   rX   rY   s   @r4   r[   r[   H   sW   ø„ ð
˜Yõ 
ð. $(Ø#ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜‰}ð	#ð
 
�‰÷#r?   r[   c                 ó   — | |z  |z   S rS   r>   )ÚaÚbÚcs      r4   Úfmarz   z   s   € àˆq‰5�1‰9Ðr?   c                   ó8   ‡ — e Zd Zdefˆ fd„Zdd„Zd„ Zd„ Zˆ xZS )ÚJambaMambaMixerrB   c                 óþ  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  |j                  z  | _        |j                  | _
        |j                  | _        |j                  | _        t        j                   | j                  | j                  dz  | j                  ¬«      | _        t        j$                  | j                  | j                  | j                  | j                  | j                  d¬«      | _        t        j                   | j                  | j                  | j                  dz  z   d¬«      | _        t        j                   | j                  | j                  d¬«      | _        t-        j.                  t-        j0                  d| j                  dz   «      j3                  d| j                  g«      | j                  d¬	«      }t-        j4                  |«      | _        t-        j8                  | j                  g«      | _        t        j                   | j                  | j                  | j                  ¬«      | _        t        j>                  | j                  |j@                  ¬
«      | _!        t        j>                  | j                  |j@                  ¬
«      | _"        t        j>                  | j                  |j@                  ¬
«      | _#        y )Nrg   rE   r   )Úin_channelsÚout_channelsÚkernel_sizeÚgroupsrF   ÚpaddingFTg      ð?r   )ÚrepeatsÚaxis©Úeps)$rG   rH   r   r   Ússm_state_sizer   Úconv_kernel_sizer   r   r&   Útime_step_rankr(   Úuse_conv_biasr'   Úuse_biasrI   rJ   Úin_projÚConv1dÚconv1dÚx_projÚdt_projrV   ÚrepeatÚarangerj   ÚlogÚA_logÚonesro   Úout_projÚRMSNormr"   Údt_layernormÚb_layernormÚc_layernorm)r2   rB   ÚArO   s      €r4   rH   zJambaMambaMixer.__init__€   s;  ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ"×0Ñ0ˆÔØ $× 1Ñ 1ˆÔØ!%×!2Ñ!2°T×5EÑ5EÑ!EˆÔØ"×0Ñ0ˆÔØ!×1Ñ1ˆÔØ×,Ñ,ˆŒä—y‘yØ×Ñ˜d×4Ñ4°qÑ8¸t¿}¹}ô
ˆŒô —i‘iØ×.Ñ.Ø×/Ñ/Ø×-Ñ-Ø×)Ñ)Ø×#Ñ#Øô
ˆŒô —i‘iØ×"Ñ"Ø×Ñ $×"5Ñ"5¸Ñ"9Ñ9Øô
ˆŒô
 —y‘y ×!4Ñ!4°d×6LÑ6LÐSWÔXˆŒä�I‰IÜ�I‰I�c˜4×.Ñ.°Ñ4Ó5×=Ñ=¸qÀ$×BUÑBUÐ>VÓWØ×*Ñ*Øô
ˆô
 —V‘V˜A“YˆŒ
Ü—‘˜$×0Ñ0Ð1Ó2ˆŒäŸ	™	Ø×"Ñ" D×$4Ñ$4¸4¿=¹=ô
ˆŒô ŸJ™J t×':Ñ':À×@QÑ@QÔRˆÔÜŸ:™: d×&9Ñ&9¸t×?PÑ?PÔQˆÔÜŸ:™: d×&9Ñ&9¸t×?PÑ?PÔQˆÕr?   c                 ó.  — |j                   d   }| j                  }| j                  |«      }t        j                  || j
                  | j
                  | j                  z   gd¬«      \  }}}	| j                  |«      | j                  |«      | j                  |	«      }	}}t        j                  | j                  |«      «      }t        j                  ||z  d«      t        j                  |d«      z  }
t        j                  t        j                  |d«      |z  «      }t        |«      D ]/  }|�"t!        ||d d …|f   |
d d …|f   «      |
d d …|f<   |
d d …|f   }Œ1 |
t        j                  |	d«      z  j#                  d«      }|||z  z   }||
d d …df   fS )Nr   rf   ©r„   éþÿÿÿ)ri   ro   r�   rV   Úsplitr‰   r‡   r˜   r™   rš   rI   Úsoftplusr�   Úexpand_dimsÚexpr1   rz   Úsqueeze)r2   rP   r›   ÚstateÚTro   ÚdeltaBCÚdeltarm   ÚCÚ	new_stateÚdtAÚtÚys                 r4   Ússm_stepzJambaMambaMixer.ssm_step­   st  € Ø�G‰G�A‰JˆØ�F‰FˆØ—+‘+˜a“.ˆÜ—h‘hØØ× Ñ  $×"5Ñ"5¸×8KÑ8KÑ"KÐLØô
‰ˆˆq�!ð
 ×'Ñ'¨Ó.°×0@Ñ0@ÀÓ0CÀT×EUÑEUÐVWÓEX�!ˆqˆÜ—‘˜DŸL™L¨Ó/Ó0ˆÜ—N‘N 5¨1¡9¨bÓ1´B·N±NÀ1ÀbÓ4IÑIˆ	Ü�f‰f”R—^‘^ E¨2Ó.°Ñ2Ó3ˆô �q–ˆAØÐ Ü"% e¨S²°A°©Y¸	Â!ÀQÀ$¹Ó"H�	š!˜Q˜$‘Øša ˜d‘O‰Eð ð œŸ™¨¨2Ó.Ñ.×7Ñ7¸Ó;ˆØ��A‘‰IˆØ�)šA˜r˜EÑ"Ð"Ð"r?   c                 óú  — | j                  |«      }|j                  dd¬«      \  }}| j                  }|�t        j                  ||gd¬«      }nt        j
                  |d|dz
  dfdg«      }| j                  |«      }|d d …|dz
   d …d d …f   }t        j                  |«      }t        j                  | j                  «       }	| j                  ||	|«      \  }
}| j                  t        ||
«      «      }|||ffS )Nrg   rf   )Úindices_or_sectionsr„   r   r�   )r   r   r   )rŒ   rŸ   rˆ   rV   ÚconcatenateÚpadrŽ   rI   Úsilur¢   r”   r­   r–   r	   )r2   rP   Ú
conv_stateÚ	ssm_stateÚxzÚzÚKÚx_fullÚconv_outr›   r¬   s              r4   Ú_process_sequencez!JambaMambaMixer._process_sequenceÄ   sí   € Ø�\‰\˜!‹_ˆØ�x‰x¨A°BˆxÓ7‰ˆˆ1Ø×!Ñ!ˆØÐ!Ü—^‘^ Z° O¸!Ô<‰Fä—V‘V˜A ¨¨Q©°¨
°FÐ;Ó<ˆFØ—;‘;˜vÓ&ˆØšA  Q¡˜x™zª1Ð,Ñ-ˆ
Ü�G‰G�HÓˆÜ�V‰V�D—J‘JÓÐˆØ—}‘} Q¨¨9Ó5‰ˆˆ9Ø�M‰Mœ&  A›,Ó'ˆØ�:˜yÐ)Ð)Ð)r?   c                 ót   — |€d\  }}n
|d   |d   }}| j                  |||«      \  }\  }}|�
||d<   ||d<   |S )Nrt   r   r   )rº   )r2   rP   rd   r³   r´   rs   s         r4   rT   zJambaMambaMixer.__call__Ô   sb   € Øˆ=Ø$.Ñ!ˆJ™	à$)¨!¡H¨e°A©h˜	ˆJà*.×*@Ñ*@Øˆz˜9ó+
Ñ'ˆÑ'�˜Yð ÐØ!ˆE�!‰HØ ˆE�!‰Hàˆr?   rS   )	r6   r7   r8   r   rH   r­   rº   rT   rX   rY   s   @r4   r|   r|      s!   ø„ ð+R˜Yõ +RóZ#ò.*ö r?   r|   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚJambaSparseMoeBlockrB   c                 ó  •— t         ‰| �  «        |j                  | _        t        j                  |j
                  |j                  d¬«      | _        t        |j
                  |j                  |j                  «      | _
        y rD   )rG   rH   r!   rI   rJ   r   r    Úrouterr   r   Ú
switch_mlprN   s     €r4   rH   zJambaSparseMoeBlock.__init__æ   sa   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ ä—i‘i × 0Ñ 0°$×2BÑ2BÈÔOˆŒÜ#Ø×Ñ˜d×4Ñ4°d×6FÑ6Fó
ˆ�r?   rP   rQ   c                 ó`  — | j                  |«      }| j                  }t        j                  t        j                  | |dz
  d¬«      dd |…f   «      }t        j
                  ||d¬«      }t        j                  |dd¬«      }| j                  ||«      }||d   z  j                  d	¬«      }|S )
Nr   rf   )Úkthr„   .r�   T)r„   Úprecise).Nrž   )	r¿   r!   rV   Ústop_gradientÚargpartitionÚtake_along_axisÚsoftmaxrÀ   Úsum)r2   rP   ÚgatesÚkÚindsÚscoresr¬   s          r4   rT   zJambaSparseMoeBlock.__call__ï   s¥   € Ø—‘˜A“ˆØ×$Ñ$ˆÜ×Ñ¤§¡°°¸AÀ¹EÈÔ KÈCÐQSÐRSÐQSÈGÑ TÓUˆÜ×#Ñ# E¨4°bÔ9ˆÜ—‘˜F¨°TÔ:ˆØ�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0ˆØˆr?   rU   rY   s   @r4   r½   r½   å   s*   ø„ ð
˜Yõ 
ð˜"Ÿ(™(ð  r§x¡x÷ r?   r½   c            	       ó’   ‡ — e Zd Zdededefˆ fd„Z	 	 d
dej                  de	ej                     de	e
   dej                  fd	„Zˆ xZS )ÚJambaDecoderLayerrB   Ú
layer_typeÚ	layer_idxc                 óà  •— t         ‰| �  «        |dk(  | _        | j                  rt        |«      | _        nt        |«      | _        |j                  dkD  r&||j                  z   |j                  z  dk(  rt        }nt        } ||«      | _        t        j                  |j                  |j                   ¬«      | _        t        j                  |j                  |j                   ¬«      | _        y )Nr-   r   r   r…   )rG   rH   Úis_attnr[   Ú	self_attnr|   r.   r    r   r   r½   rA   Úfeed_forwardrI   r—   r   r"   Úinput_layernormÚpre_ff_layernorm)r2   rB   rÏ   rÐ   Úffn_layer_classrO   s        €r4   rH   zJambaDecoderLayer.__init__û   s»   ø€ Ü‰ÑÔØ! [Ñ0ˆŒØ�<Š<Ü+¨DÓ1ˆD�Nä(¨Ó.ˆDŒJà×Ñ˜qÒ Ø˜T×5Ñ5Ñ5¸×9QÑ9QÑQÐUVÒVä1‰Oä&ˆOÙ+¨DÓ1ˆÔÜ!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ "§
¡
¨4×+;Ñ+;À×ARÑARÔ SˆÕr?   rP   rc   rd   rQ   c                 óö   — | j                   r#| j                  | j                  |«      ||«      }n!| j                  | j                  |«      |«      }||z   }|| j	                  | j                  |«      «      z   }|S rS   )rÒ   rÓ   rÕ   r.   rÔ   rÖ   )r2   rP   rc   rd   ÚhÚrÚouts          r4   rT   zJambaDecoderLayer.__call__  sq   € ð �<Š<Ø—‘˜t×3Ñ3°AÓ6¸¸eÓD‰Aà—
‘
˜4×/Ñ/°Ó2°EÓ:ˆAØ�‰EˆØ�$×#Ñ# D×$9Ñ$9¸!Ó$<Ó=Ñ=ˆØˆ
r?   rt   )r6   r7   r8   r   r9   r;   rH   rV   rW   r   r   rT   rX   rY   s   @r4   rÎ   rÎ   ú   si   ø„ ðT˜Yð T°Cð TÀCõ Tð* $(Ø#ñ	à�8‰8ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷r?   rÎ   c                   ój   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   dej                  fd„Z	ˆ xZ
S )Ú
JambaModelrB   c           	      óÜ  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  «      D ��cg c]  \  }}t        |||«      ‘Œ c}}| _
        t        j                  |j
                  |j                  ¬«      | _        |j                  j                  d«      | _        |j                  j                  d«      | _        y c c}}w )Nr…   r-   r.   )rG   rH   rI   Ú	Embeddingr$   r   Úembed_tokensÚ	enumerater)   rÎ   Úlayersr—   r"   Úfinal_layernormÚindexÚattn_idxÚssm_idx)r2   rB   Úidxr«   rO   s       €r4   rH   zJambaModel.__init__  sº   ø€ Ü‰ÑÔÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔô $ D×$:Ñ$:Ô;ô
á;‘��Qô ˜d A sÕ+Ø;ò
ˆŒô  "Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔØ×.Ñ.×4Ñ4°[ÓAˆŒØ×-Ñ-×3Ñ3°GÓ<ˆ�ùó
s   ÁC(Úinputsrd   rQ   c                 óR  — | j                  |«      }|€d gt        | j                  «      z  }t        ||| j                     «      }t        ||| j                     «      }t        | j                  |«      D ]   \  }}|j                  r|n|} ||||¬«      }Œ" | j                  |«      S )N)rc   rd   )
rà   Úlenrâ   r   rå   r   ræ   ÚziprÒ   rã   )	r2   rè   rd   rÙ   Ú	attn_maskÚssm_maskÚlayerry   rc   s	            r4   rT   zJambaModel.__call__)  sŸ   € ð
 ×Ñ˜fÓ%ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä)¨!¨U°4·=±=Ñ-AÓBˆ	Ü" 1 e¨D¯L©LÑ&9Ó:ˆä˜DŸK™K¨Ö/‰HˆE�1Ø %§¢‘9°8ˆDÙ�a˜d¨!Ô,‰Að 0ð ×#Ñ# AÓ&Ð&r?   rS   ru   rY   s   @r4   rÝ   rÝ     sA   ø„ ð
=˜Yõ 
=ð  $ñ'à—‘ð'ð ˜‰}ð'ð 
�‰÷	'r?   rÝ   c                   ó–   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   dej                  fd„Z	d„ Z
d„ Zed	„ «       Zed
„ «       Zˆ xZS )ÚModelrB   c                 óî   •— t         ‰| �  «        |j                  | _        || _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y rD   )rG   rH   r   rB   rÝ   Úmodelr*   rI   rJ   r   r$   Úlm_headrN   s     €r4   rH   zModel.__init__>  sX   ø€ Ü‰ÑÔØŸ/™/ˆŒØˆŒ	Ü Ó%ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r?   rè   rd   rQ   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S rS   )rò   rB   r*   rà   Ú	as_linearró   )r2   rè   rd   rÛ   s       r4   rT   zModel.__call__F  sW   € ð
 �j‰j˜ Ó'ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r?   c                 óÂ   — g }| j                   j                  D ]C  }|j                  r|j                  t	        «       «       Œ)|j                  t        d¬«      «       ŒE |S )Nrg   )Úsize)rò   râ   rÒ   Úappendr   r   )r2   Úcachesrî   s      r4   Ú
make_cachezModel.make_cacheR  sH   € ØˆØ—Z‘Z×&Ô&ˆEØ�}Š}Ø—‘œg›iÕ(à—‘œk¨qÔ1Õ2ð	 'ð
 ˆr?   c                 óŽ  ‡	— t        |j                  «       «      D ]2  \  }}d|v sŒ|j                  d   dk7  sŒ|j                  dd«      ||<   Œ4 | j                  j
                  r|j                  dd «       t        | j                  j                  «      D ]¦  }d|› d�Š	t        ˆ	fd„|j                  «       D «       «      sŒ,d	D ]v  }d
D ]o  }t        t        |«      «      D �cg c]-  }‰	› d|› d|› d|› �|v r|j                  ‰	› d|› d|› d|› �«      ‘Œ/ }}|sŒPt        j                  |«      |‰	› d|› d|› �<   Œq Œx Œ¨ |S c c}w )Nzconv1d.weightrf   r   rg   zlm_head.weightzmodel.layers.z.feed_forwardc              3   óF   •K  — | ]  }|j                  ‰› d �«      –— Œ y­w)ú	.experts.N)Ú
startswith)Ú.0ÚkeyÚbases     €r4   Ú	<genexpr>z!Model.sanitize.<locals>.<genexpr>e  s"   øè ø€ ÐTÁ^¸c�s—~‘~¨¨¨iÐ&8×9Á^ùs   ƒ!)rK   rM   rL   )ÚweightrF   ÚscalesÚbiasesrý   Ú.z.switch_mlp.)ÚlistÚitemsri   ÚmoveaxisrB   r*   Úpopr1   r   Úanyrq   rê   rV   Ústack)
r2   ÚweightsrÊ   ÚvÚlÚprojÚnameÚeÚexpert_tensorsr  s
            @r4   ÚsanitizezModel.sanitize[  s[  ø€ Ü˜Ÿ™›Ö)‰DˆAˆqØ !Ò#¨¯©°©°qÓ(8ØŸZ™Z¨¨1Ó-�˜’
ð *ð �9‰9×(Ò(Ø�K‰KÐ(¨$Ô/ä�t—y‘y×2Ñ2Ö3ˆAØ" 1 # ]Ð3ˆDÜÓTÀWÇ\Á\Ä^ÓTÔTØã=�ÛB�Dô "'¤s¨7£|Ô!4ó&á!4˜AØ"˜V 9¨Q¨C¨q°°°a¸°vÐ>À'ÑIð  Ÿ™ t f¨I°a°S¸¸$¸¸qÀÀÐ$GÕHØ!4ð #ð &ò
 &ÜFHÇhÁhØ*óG˜ 4 &¨°T°F¸!¸D¸6Ð BÒCñ Cñ >ð 4ð" ˆùò&s   Ã#2E
c                 ó.   — | j                   j                  S rS   )rò   râ   )r2   s    r4   râ   zModel.layersv  s   € à�z‰z× Ñ Ð r?   c                 ó   — d„ }|S )Nc                 ó0   — | j                  d«      rdddœS y)Nr¿   é@   é   )Ú
group_sizeÚbitsT)Úendswith)ÚpathÚ_s     r4   Ú	predicatez(Model.quant_predicate.<locals>.predicate|  s   € Ø�}‰}˜XÔ&Ø&(°!Ñ4Ð4Ør?   r>   )r2   r  s     r4   Úquant_predicatezModel.quant_predicatez  s   € ò	ð
 Ðr?   rS   )r6   r7   r8   r   rH   rV   rW   r   r   rT   rú   r  Úpropertyrâ   r   rX   rY   s   @r4   rð   rð   =  su   ø„ ðT˜Yõ Tð  $ñ
à—‘ð
ð ˜‰}ð
ð 
�‰ó	
òòð6 ñ!ó ð!ð ñó ôr?   rð   )$r/   Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorerV   Úmlx.nnrI   Úactivationsr	   r  r
   r   r   r   rd   r   r   Úswitch_layersr   r   ÚModulerA   r[   Úcompilerz   r|   r½   rÎ   rÝ   rð   r>   r?   r4   Ú<module>r+     sÜ   ðó Ý !ß -Ó -å Ý å ÷ó ÷ (Ý $ð ô$�ó $ó ð$ôNJˆr�y‰yô Jô/#�R—Y‘Yô /#ðd ‡�ñó ðôc�b—i‘iô côL˜"Ÿ)™)ô ô*˜Ÿ	™	ô ôD'�—‘ô 'ôBDˆB�I‰Iõ Dr?   