Ë
    îÿæiL,  ã                   óœ  — d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ d dlmZ ddlmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZ e G d„ de«      «       Zdedefd„Z G d„ dej@                  «      Z! G d„ dej@                  «      Z" G d„ dej@                  «      Z# G d„ deej@                  «      Z$ G d„ dej@                  «      Z%y)é    )Ú	dataclass)ÚAnyÚDictÚListÚOptionalÚUnionN)Úshard_linearé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache)ÚPipelineMixin)Úinitialize_ropec                   óú   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZee   ed	<   dZ	ee   ed
<   dZ
ee   ed<   dZeeeeeef   f      ed<   dZeed<   dZeee      ed<   dZee   ed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚrms_norm_epsÚ
vocab_sizeNÚhead_dimÚmax_position_embeddingsÚnum_key_value_headsÚrope_parametersTÚtie_word_embeddingsÚlayer_typesÚsliding_windowc                 ó‚   — | j                   €| j                  | _         | j                  €dg| j                  z  | _        y y )NÚfull_attention)r   r   r!   r   ©Úselfs    úm/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/ministral3.pyÚ__post_init__zModelArgs.__post_init__"   sC   € Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÑÐ#Ø 0Ð1°D×4JÑ4JÑJˆDÕð $ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr   r   r   r   r   r   r   r    Úboolr!   r   r"   r(   © r)   r'   r   r      sª   … àƒOØÓØÓØÓØÓØÓØƒOØ"€Hˆh�s‰mÓ"Ø-1Ð˜X c™]Ó1Ø)-Ð˜ #™Ó-Ø>B€O�X˜d 3¨¨e°S¨jÑ(9Ð#9Ñ:Ñ;ÓBØ $Ð˜Ó$Ø'+€K�˜$˜s™)Ñ$Ó+Ø$(€N�H˜S‘MÓ(óKr)   r   Úbetar   c           
      óD  — t        |t        j                  «      r|j                  dkD  r	|d d …d f   }d|t        j                  dt        j
                  t        j                  | «      |z   |z  «      z   «      z  z   }|j                  dk(  r|d d …d d d …d f   S |d d …d f   S )Nr   r
   é   )Ú
isinstanceÚmxÚarrayÚndimÚlogÚfloorÚarange)ÚsizeÚoffsetr3   r   Úscalings        r'   Ú_get_llama_4_attn_scaler@   *   s˜   € Ü�&œ"Ÿ(™(Ô#¨¯©°aªØš˜4˜‘ˆà�$œŸ™Ø	ŒB�H‰H”b—i‘i “o¨Ñ.Ð2IÑIÓJÑJóñ ñ €Gð ‡|�|�qÒØ’q˜$¢ 4Ð'Ñ(Ð(à’q˜$�wÑÐr)   c                   ó¢   ‡ — e Zd Zdefˆ fd„Z	 	 d	dej                  dej                  deej                     dee   dej                  f
d„Z	ˆ xZ
S )
Ú	AttentionÚargsc                 ó`  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs |j                  |z  x| _        }|dz  | _        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  ||z  |d¬«      | _        t        | j                  |j                   d   d|j                   |j"                  «      | _        y )Ng      à¿F©ÚbiasÚ
rope_theta)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsr   ÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projr   r   r   Úrope)r&   rC   ÚdimrJ   rK   r   Ú	__class__s         €r'   rI   zAttention.__init__8   s  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à#'§=¡=Ò#O°D×4DÑ4DÈÑ4OÐOˆŒ˜à˜t‘^ˆŒ
ä—i‘i  W¨xÑ%7¸eÔDˆŒÜ—i‘i  Z°(Ñ%:ÀÔGˆŒÜ—i‘i  Z°(Ñ%:ÀÔGˆŒÜ—i‘i ¨(Ñ 2°C¸eÔDˆŒä#Ø�M‰MØ× Ñ  Ñ.ØØ× Ñ Ø×(Ñ(ó
ˆ�	r)   ÚxÚ
attn_scaleÚmaskÚcacheÚreturnc                 ó0  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }
}	}|j	                  ||| j
                  d«      j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	|
j	                  ||| j                  d«      j                  dddd«      }
d}|�H|j                  }| j                  ||¬«      }| j                  |	|¬«      }	|j                  |	|
«      \  }	}
n"| j                  |«      }| j                  |	«      }	||z  }t        ||	|
|| j                  |¬«      }|j                  dddd«      j	                  ||d«      }| j                  |«      S )Néÿÿÿÿr   r5   r
   é   )r>   )rY   rL   rX   )ÚshaperO   rP   rQ   ÚreshaperJ   Ú	transposerK   r>   rS   Úupdate_and_fetchr   rL   rR   )r&   rV   rW   rX   rY   ÚBÚLÚDÚqueriesÚkeysÚvaluesr>   Úoutputs                r'   Ú__call__zAttention.__call__P   sy  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆð —/‘/ ! Q¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A˜q $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàˆØÐØ—\‘\ˆFØ—i‘i °�iÓ7ˆGØ—9‘9˜T¨&�9Ó1ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDØ˜JÑ&ˆÜ-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r)   ©NN)r*   r+   r,   r   rI   r7   r8   r   r   ri   Ú__classcell__©rU   s   @r'   rB   rB   7   se   ø„ ð
˜Yõ 
ð8 $(Ø#ñ#à�8‰8ð#ð —H‘Hð#ð �r—x‘xÑ ð	#ð
 ˜‰}ð#ð 
�‰÷#r)   rB   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚMLPrC   c                 ó   •— t         ‰| �  «        |j                  }|j                  }t	        j
                  ||d¬«      | _        t	        j
                  ||d¬«      | _        t	        j
                  ||d¬«      | _        y ©NFrE   )	rH   rI   r   r   rM   rN   Ú	gate_projÚ	down_projÚup_proj)r&   rC   rT   Ú
hidden_dimrU   s       €r'   rI   zMLP.__init__s   s`   ø€ Ü‰ÑÔà×ÑˆØ×+Ñ+ˆ
ÜŸ™ 3¨
¸Ô?ˆŒÜŸ™ :¨s¸Ô?ˆŒÜ—y‘y  j°uÔ=ˆ�r)   rZ   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)rr   r   rq   rs   )r&   rV   s     r'   ri   zMLP.__call__|   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr)   )	r*   r+   r,   r   rI   r7   r8   ri   rk   rl   s   @r'   rn   rn   r   s!   ø„ ð>˜Yõ >ðJ˜RŸX™X÷ Jr)   rn   c                   ó¨   ‡ — e Zd Zd
dedefˆ fd„Z	 	 ddej                  dej                  deej                     dee	   dej                  f
d	„Z
ˆ xZS )ÚTransformerBlockrC   Úuse_slidingc                 ó‚  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        |«      | _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        || _        y )N©Úeps)rH   rI   r   r   ry   rB   Ú	self_attnrn   ÚmlprM   ÚRMSNormr   Úinput_layernormÚpost_attention_layernormrC   )r&   rC   ry   rU   s      €r'   rI   zTransformerBlock.__init__�   s’   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø×+Ñ+ˆÔØ&ˆÔÜ" 4›ˆŒÜ�t“9ˆŒÜ!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð ˆ�	r)   rV   rW   rX   rY   rZ   c                 ó    — | j                  | j                  |«      |||«      }||z   }| j                  | j                  |«      «      }||z   }|S rv   )r}   r€   r~   r�   )r&   rV   rW   rX   rY   ÚrÚhÚouts           r'   ri   zTransformerBlock.__call__Ž   sS   € ð �N‰N˜4×/Ñ/°Ó2°JÀÀeÓLˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�!‰eˆØˆ
r)   )Frj   )r*   r+   r,   r   r1   rI   r7   r8   r   r   ri   rk   rl   s   @r'   rx   rx   €   sl   ø„ ñ˜Yð °Tõ ð" $(Ø#ñà�8‰8ðð —H‘Hðð �r—x‘xÑ ð	ð
 ˜‰}ðð 
�‰÷r)   rx   c                   ór   ‡ — e Zd Zdefˆ fd„Zˆ fd„Z	 	 ddej                  deej                     fd„Z	ˆ xZ
S )ÚLanguageModelrC   c                 óŒ  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        t        j                  |j                  |j                  «      | _
        | j
                  D �cg c]  }t        ||dk(  ¬«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        | j
                  j!                  d«      | _        d | _        t'        | j                  «      D ]  \  }}|j(                  sŒ|| _         y  y c c}w )NÚsliding_attention)rC   ry   r{   r$   )rH   rI   rC   r   r   r!   r"   rM   Ú	Embeddingr   Úembed_tokensrx   Úlayersr   r   ÚnormÚindexÚfa_idxÚswa_idxÚ	enumeratery   )r&   rC   Ú
layer_typeÚeÚlrU   s        €r'   rI   zLanguageModel.__init__�   s  ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ×+Ñ+ˆÔØ"×1Ñ1ˆÔÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔð #×.Ò.ó
á.�
ô  $°JÐBUÑ4UÖVØ.ñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	Ø×&Ñ&×,Ñ,Ð-=Ó>ˆŒØˆŒÜ˜dŸk™kÖ*‰DˆAˆqØ�}‹}Ø �”Ùñ +ùò
s   ÂEc                 ó.  •— t         ‰| �  |«       d | _        d | _        t	        | j
                  «      D ]_  \  }}| j                  €|j                  r|| _        n| j                  €|j                  s|| _        | j                  €ŒR| j                  €Œ_ y  y rv   )rH   Úpipeliner�   r�   r‘   Úpipeline_layersry   )r&   Úgroupr“   r”   rU   s       €r'   r–   zLanguageModel.pipeline±   s{   ø€ Ü‰Ñ˜ÔØˆŒØˆŒÜ˜d×2Ñ2Ö3‰DˆAˆqØ�|‰|Ð#¨¯ªØ �•Ø—‘Ð$¨Q¯]ª]Ø�”Ø�{‰{Ñ&¨4¯<©<Ñ+CÙñ 4r)   ÚinputsÚinput_embeddingsc                 ó:  — |�|}n| j                  |«      }| j                  }| j                  }|€d gt        | j                  «      z  }d}n|d   j
                  }d x}}	| j                  �t        ||| j                     «      }	| j                  �%t        ||| j                     | j                  ¬«      }t        |j                  d   || j                  j                  d   | j                  j                  d   «      j                  |j                  «      }
||dz
  k  r#t         j"                  j%                  ||dz   «      }t'        | j                  |«      D ]!  \  }}|j(                  r|n|	} |||
||¬«      }Œ# |dk7  rVt         j"                  j+                  ||dz
  |z  «      }|d   �+t!        j,                  |d   j.                  |«      |d   _        |dkD  r/t         j"                  j1                  |«      d |j                  d    }| j3                  |«      S )Nr   )Úwindow_sizer
   Úllama_4_scaling_betaÚ original_max_position_embeddings)rY   r\   )r‹   Úpipeline_rankÚpipeline_sizeÚlenr—   r>   r�   r   r�   r"   r@   r^   rC   r   ÚastypeÚdtyper7   ÚdistributedÚ	recv_likeÚzipry   ÚsendÚdependsrf   Ú
all_gatherr�   )r&   r™   rY   rš   r„   rŸ   r    r>   Úswa_maskÚfa_maskrW   r”   ÚcrX   s                 r'   ri   zLanguageModel.__call__½   s÷  € ð Ð'Ø ‰Aà×!Ñ! &Ó)ˆAà×*Ñ*ˆØ×*Ñ*ˆàˆ=Ø�FœS ×!5Ñ!5Ó6Ñ6ˆEØ‰Fà˜1‘X—_‘_ˆFà!Ð!ˆ�7Ø�;‰;Ð"Ü+¨A¨u°T·[±[Ñ/AÓBˆGØ�<‰<Ð#Ü,Ø�5˜Ÿ™Ñ&°D×4GÑ4GôˆHô -Ø�L‰L˜‰OØØ�I‰I×%Ñ%Ð&<Ñ=Ø�I‰I×%Ñ%Ð&HÑIó	
÷
 ‰&�—‘‹/ð 	ð ˜=¨1Ñ,Ò,Ü—‘×(Ñ(¨¨]¸QÑ->Ó@ˆAä˜×,Ñ,¨eÖ4‰DˆAˆqØ Ÿ}š}‘8°'ˆDÙ�!�Z ¨QÔ/‰Að 5ð
 ˜AÒÜ—‘×#Ñ# A¨¸Ñ(9¸]Ñ'JÓKˆAØ�R‰yÐ$Ü!#§¡¨E°"©I¯N©N¸AÓ!>��b‘	”ð ˜1ÒÜ—‘×)Ñ)¨!Ó,¨\¨q¯w©w°q©zÐ:ˆAà�y‰y˜‹|Ðr)   rj   )r*   r+   r,   r   rI   r–   r7   r8   r   ri   rk   rl   s   @r'   r‡   r‡   œ   s@   ø„ ð˜Yõ ô(
ð Ø/3ñ	5à—‘ð5ð # 2§8¡8Ñ,÷	5r)   r‡   c                   óÀ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     fd„Zd„ Z	ddeej                  j                     fd„Zed	„ «       Zd
„ Zˆ xZS )ÚModelrC   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y rp   )rH   rI   rC   r   r‡   Úmodelr    rM   rN   r   r   Úlm_head)r&   rC   rU   s     €r'   rI   zModel.__init__ö   sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ" 4Ó(ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r)   r™   rš   c                 óÈ   — | j                  |||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S rv   )r°   rC   r    r‹   Ú	as_linearr±   )r&   r™   rY   rš   r…   s        r'   ri   zModel.__call__þ   sZ   € ð �j‰j˜ Ð(8Ó9ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r)   c                 óT  — |j                  «       D ��ci c]  \  }}d|vsŒ||“Œ }}}| j                  j                  r|j                  dd «       i }|j                  «       D ]:  \  }}d|v r"|}|j	                  dd«      }||   }||z  ||<   Œ,d|v rŒ1||vsŒ6|||<   Œ< |}|S c c}}w )Nzself_attn.rotary_emb.inv_freqzlm_head.weightÚweight_scale_invÚ
_scale_invÚ Úactivation_scale)ÚitemsrC   r    ÚpopÚreplace)r&   ÚweightsÚkÚvÚnew_weightsÚ	scale_invÚwkÚweights           r'   ÚsanitizezModel.sanitize  sÏ   € ð %Ÿ]™]œ_ô
Ù,‘T�Q˜Ð0OÐWXÒ0XˆAˆq‰D˜_ð 	ñ 
ð �9‰9×(Ò(Ø�K‰KÐ(¨$Ô/àˆØ—M‘M–O‰DˆAˆqØ! QÑ&Ø�	Ø—Y‘Y˜|¨RÓ0�Ø  ™�Ø"(¨9Ñ"4�˜B’Ø# qÑ(ØØ˜+Ò%Ø!"�˜A’ð $ð ˆàˆùó'
s
   ”B$¡B$r˜   c                 óÌ  — |xs t         j                  j                  «       }|j                  «       }| j                  j
                  D �]˜  }t        |j                  j                  d|¬«      |j                  _        t        |j                  j                  d|¬«      |j                  _	        t        |j                  j                  d|¬«      |j                  _
        t        |j                  j                  d|¬«      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        t        |j                  j                  d|¬«      |j                  _        t        |j                  j                   d|¬«      |j                  _        t        |j                  j"                  d|¬«      |j                  _        �Œ› y )Nzall-to-sharded)r˜   zsharded-to-all)r7   r¤   Úinitr=   r°   rŒ   r	   r}   rO   rP   rQ   rR   rJ   rK   r~   rq   rr   rs   )r&   r˜   ÚNÚlayers       r'   ÚshardzModel.shard"  se  € ØÒ.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ—Z‘Z×&Õ&ˆEä%1Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ð �O‰O×#Ò#¨Ñ)Õ#Ø�O‰O×&Ò&¨1Ñ,Õ&ô #/Ø—	‘	×#Ñ#Ð%5¸Uô#ˆE�I‰IÔô #/Ø—	‘	×#Ñ#Ð%5¸Uô#ˆE�I‰IÔô !-Ø—	‘	×!Ñ!Ð#3¸5ô!ˆE�I‰IÖñ1 'r)   c                 ó.   — | j                   j                  S rv   )r°   r—   r%   s    r'   rŒ   zModel.layersA  s   € à�z‰z×)Ñ)Ð)r)   c                 ó¨   — | j                   D �cg c]8  }|j                  r t        | j                  j                  ¬«      n	t        «       ‘Œ: c}S c c}w )N)Úmax_size)rŒ   ry   r   r°   r"   r   )r&   rÇ   s     r'   Ú
make_cachezModel.make_cacheE  sU   € ð Ÿšó
ñ %�ð ×$Ò$ô  ¨¯©×)BÑ)BÕCä“Yñð %ñ
ð 	
ùò 
s   �=Arj   rv   )r*   r+   r,   r   rI   r7   r8   r   ri   rÃ   r¤   ÚGrouprÈ   ÚpropertyrŒ   rÌ   rk   rl   s   @r'   r®   r®   õ   sv   ø„ ðT˜Yõ Tð Ø/3ñ	à—‘ðð # 2§8¡8Ñ,ó	òñ.˜8 B§N¡N×$8Ñ$8Ñ9ó ð> ñ*ó ð*ö
r)   r®   )&Údataclassesr   Útypingr   r   r   r   r   Úmlx.coreÚcorer7   Úmlx.nnrM   Úmlx.nn.layers.distributedr	   Úactivationsr   Úbaser   r   r   rY   r   r   r–   r   Ú
rope_utilsr   r   r0   r/   r@   ÚModulerB   rn   rx   r‡   r®   r2   r)   r'   Ú<module>rÙ      s·   ðõ "ß 3Õ 3å Ý Ý 2å ß TÑ Tß +Ý #Ý 'ð ôK�ó Kó ðKð0
 °ð 
 ÐPSó 
 ô8#�—	‘	ô 8#ôvJˆ"�)‰)ô Jô�r—y‘yô ô8V�M 2§9¡9ô VôrX
ˆB�I‰Iõ X
r)   