Ë
    îÿæi  ã                   ó^  — d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZmZmZ ddlmZ ddlmZ e G d„ d	e«      «       Z G d
„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚOptionalÚUnionNé   )ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Úinitialize_rope)Ú	SwitchGLUc                   ó&  — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   dZ	e
e   ed<   dZe
e   ed<   dZe
e   ed<   d
Zeed<   d
Zeed<   dZeed<   d
Zeed<   dZe
eeeeef   f      ed<   dZeed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚrms_norm_epsÚ
vocab_sizeÚnum_expertsÚnum_experts_per_tokFÚnorm_topk_probNÚhead_dimÚmax_position_embeddingsÚnum_key_value_headsÚattention_biasÚmlp_biasi'  Ú
rope_thetaÚrope_traditionalÚrope_scalingTÚtie_word_embeddingsc                 ó@   — | j                   €| j                  | _         y y ©N)r   r   ©Úselfs    úh/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/olmoe.pyÚ__post_init__zModelArgs.__post_init__$   s!   € Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÕ$ð ,ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr   Úboolr   r   r   r   r   r   r   r    r!   r   r   r"   r(   © r)   r'   r   r      sÈ   … àƒOØÓØÓØÓØÓØÓØƒOØÓØÓØ €N�DÓ Ø"€Hˆh�s‰mÓ"Ø-1Ð˜X c™]Ó1Ø)-Ð˜ #™Ó-Ø €N�DÓ Ø€HˆdÓØ€J�ÓØ"Ð�dÓ"Ø;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?Ø $Ð˜Ó$ó@r)   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	Ú	AttentionÚargsc                 ó^  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs |j                  |z  x| _        }|dz  | _        t        j                  |||z  |j                  ¬«      | _        t        j                  |||z  |j                  ¬«      | _        t        j                  |||z  |j                  ¬«      | _        t        j                  ||z  ||j                  ¬«      | _        t!        | j                  |j"                  |j$                  |j&                  |j(                  «      | _        t        j,                  ||z  |j.                  «      | _        t        j,                  ||z  |j.                  «      | _        y )Ng      à¿©Úbias)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsr   ÚscaleÚnnÚLinearr   Úq_projÚk_projÚv_projÚo_projr   r   r    r!   r   ÚropeÚRMSNormr   Úq_normÚk_norm)r&   r5   Údimr;   r<   r   Ú	__class__s         €r'   r:   zAttention.__init__*   sU  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à#'§=¡=Ò#O°D×4DÑ4DÈÑ4OÐOˆŒ˜à˜t‘^ˆŒ
ä—i‘i  W¨xÑ%7¸d×>QÑ>QÔRˆŒÜ—i‘i  Z°(Ñ%:À×ATÑATÔUˆŒÜ—i‘i  Z°(Ñ%:À×ATÑATÔUˆŒÜ—i‘i ¨(Ñ 2°C¸d×>QÑ>QÔRˆŒä#Ø�M‰MØ�O‰OØ×!Ñ!Ø×ÑØ×(Ñ(ó
ˆŒ	ô —j‘j ¨8Ñ!3°T×5FÑ5FÓGˆŒÜ—j‘j ¨hÑ!6¸×8IÑ8IÓJˆ�r)   ÚxÚmaskÚcacheÚreturnc                 óv  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}| j	                  |«      }| j                  |«      }|j                  ||| j                  d«      j                  dddd«      }|j                  ||| j                  d«      j                  dddd«      }|	j                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   é   r   é   )Úoffset)rL   r=   rK   )Úshaper@   rA   rB   rF   rG   Úreshaper;   Ú	transposer<   rD   rR   Úupdate_and_fetchr   r=   rC   )r&   rJ   rK   rL   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r'   Ú__call__zAttention.__call__E   sƒ  € ð —'‘'‰ˆˆ1ˆaØ $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆØ—+‘+˜gÓ&ˆØ�{‰{˜4Ó ˆØ—/‘/ ! Q¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A˜q $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆØÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDÜ-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r)   ©NN©r*   r+   r,   r   r:   ÚmxÚarrayr   r   r^   Ú__classcell__©rI   s   @r'   r4   r4   )   sY   ø„ ðK˜Yõ Kð< $(Ø#ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜‰}ð	#ð
 
�‰÷#r)   r4   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚOlmoeSparseMoeBlockr5   c                 ób  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        t        j                  |j                  | j                  d¬«      | _	        t        |j                  |j                  | j                  |j                  ¬«      | _        y ©NFr7   )r9   r:   r   r   Útop_kr   r>   r?   r   Úgater   r   r   Ú
switch_mlp©r&   r5   rI   s     €r'   r:   zOlmoeSparseMoeBlock.__init__a   s…   ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ×-Ñ-ˆŒ
Ø"×1Ñ1ˆÔä—I‘I˜d×.Ñ.°×0@Ñ0@ÀuÔMˆŒ	Ü#Ø×ÑØ×"Ñ"Ø×ÑØ—‘ô	
ˆ�r)   rJ   rM   c                 ó
  — |j                   \  }}}|j                  d|«      }| j                  |«      }t        j                  |dd¬«      }| j
                  }t        j                  t        j                  | |dz
  d¬«      dd |…f   «      }	t        j                  ||	d¬«      }
| j                  r|
|
j                  dd¬«      z  }
| j                  ||	«      }||
d	   z  j                  d
¬«      }|j                  |||«      S )NrO   r   T)ÚaxisÚprecise)Úkthrn   .)rn   )rn   Úkeepdims).Néþÿÿÿ)rS   rT   rj   ra   Úsoftmaxri   Ústop_gradientÚargpartitionÚtake_along_axisr   Úsumrk   )r&   rJ   rW   rX   rY   Úx_flatÚrouter_logitsÚrouting_weightsÚkÚindicesÚscoresÚys               r'   r^   zOlmoeSparseMoeBlock.__call__o   sô   € Ø—'‘'‰ˆˆ1ˆaØ—‘˜2˜qÓ!ˆØŸ	™	 &Ó)ˆÜŸ*™* ]¸ÀDÔIˆØ�J‰JˆÜ×"Ñ"Ü�O‰O˜_Ð,°!°a±%¸bÔAÀ#ÀrÈÀrÀ'ÑJó
ˆô ×#Ñ# O°WÀ2ÔFˆØ×ÒØ˜fŸj™j¨b¸4˜jÓ@Ñ@ˆFØ�O‰O˜F GÓ,ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0ˆØ�y‰y˜˜A˜qÓ!Ð!r)   ©	r*   r+   r,   r   r:   ra   rb   r^   rc   rd   s   @r'   rf   rf   `   s*   ø„ ð
˜Yõ 
ð"˜"Ÿ(™(ð " r§x¡x÷ "r)   rf   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚTransformerBlockr5   c                 ó"  •— t         ‰| �  «        t        |«      | _        t	        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  ¬«      | _        y )N©Úeps)r9   r:   r4   Ú	self_attnrf   Úmlpr>   rE   r   r   Úinput_layernormÚpost_attention_layernormrl   s     €r'   r:   zTransformerBlock.__init__�   sf   ø€ Ü‰ÑÔÜ" 4›ˆŒÜ& tÓ,ˆŒÜ!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÕ%r)   rJ   rK   rL   rM   c                 ó–   — || j                  | j                  |«      ||«      z   }|| j                  | j                  |«      «      z   }|S r$   )r…   r‡   r†   rˆ   )r&   rJ   rK   rL   s       r'   r^   zTransformerBlock.__call__Š   sI   € ð �—‘˜t×3Ñ3°AÓ6¸¸eÓDÑDˆØ�—‘˜×6Ñ6°qÓ9Ó:Ñ:ˆØˆr)   r_   r`   rd   s   @r'   r�   r�   €   sW   ø„ ð
˜Yõ 
ð $(Ø#ñ	à�8‰8ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷r)   r�   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )Ú
OlmoeModelr5   c                 óÂ  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        | j                  dkD  sJ ‚t        j                  |j                  |j                  «      | _        t        |j                  «      D �cg c]  }t        |¬«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        y c c}w )Nr   )r5   rƒ   )r9   r:   r5   r   r   r>   Ú	Embeddingr   Úembed_tokensÚranger�   ÚlayersrE   r   Únorm)r&   r5   Ú_rI   s      €r'   r:   zOlmoeModel.__init__–   s¯   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ�‰ Ò"Ð"Ð"ÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔä16°t×7MÑ7MÔ1Nó
Ù1N¨AÔ $Ö'Ð1Nñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆ�	ùò
s   ÂCÚinputsc                 óì   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||¬«      }Œ | j                  |«      S )Nr   )rL   )rŽ   Úlenr�   r
   Úzipr‘   )r&   r“   rL   ÚhrK   ÚlayerÚcs          r'   r^   zOlmoeModel.__call__¢   ss   € ð
 ×Ñ˜fÓ%ˆØˆ=Ø�FœS §¡Ó-Ñ-ˆEÜ$ Q¨¨a©Ó1ˆÜ˜DŸK™K¨Ö/‰HˆE�1Ù�a˜ QÔ'‰Að 0à�y‰y˜‹|Ðr)   r$   r   rd   s   @r'   r‹   r‹   •   s'   ø„ ð
H˜Yõ 
Hð ñà—‘÷r)   r‹   c                   ó^   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zd„ Ze	d„ «       Z
ˆ xZS )ÚModelr5   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y rh   )r9   r:   r5   r   r‹   Úmodelr"   r>   r?   r   r   Úlm_headrl   s     €r'   r:   zModel.__init__±   sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ Ó%ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r)   r“   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S r$   )r�   r5   r"   rŽ   Ú	as_linearrž   )r&   r“   rL   Úouts       r'   r^   zModel.__call__¹   sW   € ð
 �j‰j˜ Ó'ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r)   c                 óp  — d|vr|S t        | j                  j                  «      D ]ˆ  }d|› �}dD ]|  }dD ]u  }|› d|› d|› �|v sŒt        | j                  j                  «      D �cg c]  }|j	                  |› d|› d|› d|› �«      ‘Œ  }}t        j                  |«      ||› d|› d|› �<   Œw Œ~ ŒŠ |S c c}w )	Nz+model.layers.0.mlp.experts.0.up_proj.weightzmodel.layers.)Úup_projÚ	down_projÚ	gate_proj)ÚweightÚscalesÚbiasesz.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.)r�   r5   r   r   Úpopra   Ústack)r&   ÚweightsÚlÚprefixÚnr{   ÚeÚto_joins           r'   ÚsanitizezModel.sanitizeÅ   sç   € Ø8ÀÑGØˆNÜ�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛ:�Û7�AØ ˜ °°°1°Q°CÐ8¸GÒCô &+¨4¯9©9×+@Ñ+@Ô%Aó#á%A ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÕLØ%Að  ð #ô GIÇhÁhÈwÓFW˜ 6 (Ð*:¸1¸#¸Q¸q¸cÐ BÒCñ 8ñ ;ð 4ð ˆùò#s   Á&#B3
c                 ó.   — | j                   j                  S r$   )r�   r�   r%   s    r'   r�   zModel.layersÔ   s   € à�z‰z× Ñ Ð r)   r$   )r*   r+   r,   r   r:   ra   rb   r^   r²   Úpropertyr�   rc   rd   s   @r'   r›   r›   °   s@   ø„ ðT˜Yõ Tð ñ
à—‘ó
òð ñ!ó ô!r)   r›   )Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorera   Úmlx.nnr>   Úbaser	   r
   r   Ú
rope_utilsr   Úswitch_layersr   r   ÚModuler4   rf   r�   r‹   r›   r2   r)   r'   Ú<module>r¾      s’   ðõ "ß -Ó -å Ý ç TÑ TÝ 'Ý $ð ô@�ó @ó ð@ô44#�—	‘	ô 4#ôn"˜"Ÿ)™)ô "ô@�r—y‘yô ô*�—‘ô ô6&!ˆB�I‰Iõ &!r)   