Ë
    îÿæiÝ  ã                   ó  — d dl Z d dlmZ d dlmZ d dlmZ ddlm	Z	 ddl
mZ ddlmZ e G d„ de«      «       Z G d	„ d
ej                  «      Z G d„ dej                  «      Z G d„ dej                  «      Z G d„ dej                  «      Zy)é    N)Ú	dataclassé   )Úswiglu)ÚBaseModelArgs)ÚArraysCachec                   ó¢   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   dZeed<   dZeed<   dZ	e
ed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚ
state_sizeÚnum_hidden_layersÚconv_kernelÚuse_biasÚuse_conv_biasÚtime_step_rankTÚtie_word_embeddingsFÚuse_bcdt_rmsg�íµ ÷Æ°>Úmixer_rms_epsc                 ó.  — t        | d«      st        | d«      r| j                  | _        t        | d«      st        | d«      r| j                  | _        t        | d«      st        | d«      r| j
                  | _        t        | d«      st        | d«      r| j                  | _        t        | d«      st        | d	«      r| j                  | _        t        | d
«      st        | d«      r| j                  | _        t        | d«      st        | d«      r| j                  | _        t        | d«      st        | d«      r| j                  | _        | j                   dk(  r't#        j$                  | j                  dz  «      | _        | j&                  dk(  rd| _        y y )Nr   Úd_modelr   Úd_innerr   Úd_stater   Ún_layerÚn_layersr   Úd_convr   Úbiasr   Ú	conv_biasÚautoé   Úfalcon_mambaT)Úhasattrr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   ÚmathÚceilr
   r   ©Úselfs    úh/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/mamba.pyÚ__post_init__zModelArgs.__post_init__   s>  € Ü�t˜]Ô+´¸¸iÔ0HØ#Ÿ|™|ˆDÔÜ�tÐ0Ô1´g¸dÀIÔ6NØ%)§\¡\ˆDÔ"Ü�t˜\Ô*¬w°t¸YÔ/GØ"Ÿl™lˆDŒOÜ�tÐ0Ô1´g¸dÀIÔ6NØ%)§\¡\ˆDÔ"Ü�tÐ0Ô1´g¸dÀJÔ6OØ%)§]¡]ˆDÔ"Ü�t˜]Ô+´¸¸hÔ0GØ#Ÿ{™{ˆDÔÜ�t˜ZÔ(¬W°T¸6Ô-BØ ŸI™IˆDŒMÜ�t˜_Ô-´'¸$ÀÔ2LØ!%§¡ˆDÔà×Ñ &Ò(Ü"&§)¡)¨D×,<Ñ,<¸rÑ,AÓ"BˆDÔØ�?‰?˜nÒ,Ø $ˆDÕð -ó    N)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚboolr   r   r   Úfloatr)   © r*   r(   r	   r	      s^   … àƒOØƒOØÓØÓØƒOØÓØÓØƒNØÓØÓØ $Ð˜Ó$Ø€L�$ÓØ€M�5Óó%r*   r	   c                   ó8   ‡ — e Zd Zdefˆ fd„Zdd„Zd„ Zd„ Zˆ xZS )Ú
MambaBlockÚargsc                 ó  •‡— t         ‰| �  «        ‰| _        ‰j                  | _        ‰j                  | _        ‰j                  | _        ‰j                  | _        t        ‰j                  «      | _
        ‰j                  | _        ‰j                  | _        | j                  r
ˆfd„| _        t        j                  | j                  | j                  dz  ‰j                   ¬«      | _        t        j$                  | j                  | j                  | j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d| j
                  z  z   d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t-        j.                  t-        j0                  d| j
                  dz   «      j3                  d	| j
                  g«      | j                  d¬
«      }t-        j4                  |«      | _        t-        j8                  | j                  g«      | _        t        j                  | j                  | j                  ‰j                   ¬«      | _        y )Nc                 ó²   •— t         j                  j                  | t        j                  | j                  d   | j
                  «      ‰j                  ¬«      S )Néÿÿÿÿ)Úeps)ÚmxÚfastÚrms_normÚonesÚshapeÚdtyper   )Úxr6   s    €r(   Ú<lambda>z%MambaBlock.__init__.<locals>.<lambda>C   s=   ø€ ¬¯©×(8Ñ(8Ø”2—7‘7˜1Ÿ7™7 2™;¨¯©Ó0°d×6HÑ6Hð )9ô )r*   é   ©r   r   )Úin_channelsÚout_channelsÚkernel_sizeÚgroupsr   ÚpaddingFTg      ð?r   )ÚrepeatsÚaxis)ÚsuperÚ__init__r6   r   r   Ússm_state_sizer   Úconv_kernel_sizer   r0   r   r   r   Ú
mixer_normÚnnÚLinearr   Úin_projÚConv1dÚconv1dÚx_projÚdt_projr;   ÚrepeatÚarangeÚreshapeÚlogÚA_logr>   ÚDÚout_proj)r'   r6   ÚAÚ	__class__s    ` €r(   rM   zMambaBlock.__init__7   sð  ù€ Ü‰ÑÔØˆŒ	à×+Ñ+ˆÔØ"Ÿo™oˆÔØ $× 0Ñ 0ˆÔØ!%×!7Ñ!7ˆÔÜ! $×"5Ñ"5Ó6ˆÔØ!×/Ñ/ˆÔØ ×-Ñ-ˆÔØ×ÒóˆDŒOô —y‘yØ×Ñ˜d×4Ñ4°qÑ8¸t¿}¹}ô
ˆŒô —i‘iØ×.Ñ.Ø×/Ñ/Ø×-Ñ-Ø×)Ñ)Ø×#Ñ#Øô
ˆŒô —i‘iØ×"Ñ"Ø×Ñ ! d×&9Ñ&9Ñ"9Ñ9Øô
ˆŒô
 —y‘y ×!4Ñ!4°d×6LÑ6LÐSWÔXˆŒä�I‰IÜ�I‰I�c˜4×.Ñ.°Ñ4Ó5×=Ñ=¸qÀ$×BUÑBUÐ>VÓWØ×*Ñ*Øô
ˆô
 —V‘V˜A“YˆŒ
Ü—‘˜$×0Ñ0Ð1Ó2ˆŒäŸ	™	Ø×"Ñ" D×$4Ñ$4¸4¿=¹=ô
ˆ�r*   c           	      óÊ  — | j                   }| j                  |«      }t        | j                  r| j                  nd„ t        j                  || j                  | j                  | j                  z   gd¬«      «      \  }}}| j                  rt        | j                  |||f«      \  }}}t        j                  | j                  |«      «      }t        j                  ||z  d«      t        j                  |d«      z  }	|�2|	|t        j                  t        j                  |d«      |z  «      z  z  }	|	t        j                  |d«      z  j                  d«      }
|
||z  z   }
|
|	fS )Nc                 ó   — | S ©Nr3   )rA   s    r(   rB   z%MambaBlock.ssm_step.<locals>.<lambda>k   s   € Ár*   r9   ©rK   r   rC   )r]   rV   Úmapr   rP   r;   Úsplitr   rN   rQ   ÚsoftplusrW   Úexpand_dimsÚexpÚsqueeze)r'   rA   r_   Ústater]   ÚdeltaBCÚdeltaÚBÚCÚ	new_stateÚys              r(   Ússm_stepzMambaBlock.ssm_stepg   s1  € Ø�F‰FˆØ—+‘+˜a“.ˆÜØ#×0Ò0ˆD�OŠO±kÜ�H‰HØØ×$Ñ$ d×&9Ñ&9¸D×<OÑ<OÑ&OÐPØôó
‰ˆˆq�!ð ×ÒÜ˜dŸo™o°°q¸!¨}Ó=‰KˆE�1�aÜ—‘˜DŸL™L¨Ó/Ó0ˆÜ—N‘N 5¨1¡9¨bÓ1´B·N±NÀ1ÀaÓ4HÑHˆ	ØÐØ˜¤§¡¬¯©°u¸bÓ(AÀAÑ(EÓ!FÑFÑFˆIØœŸ™¨¨2Ó.Ñ.×7Ñ7¸Ó:ˆØ��A‘‰IˆØ�)ˆ|Ðr*   c                 ó   — |j                   \  }}}| j                  |«      }|j                  dd¬«      \  }}| j                  }	|�t	        j
                  ||gd¬«      }
nt	        j                  |d|	dz
  dfdg«      }
| j                  |
«      }|
d d …|	dz
   d …d d …f   }t        j                  |«      }t	        j                  | j                  «       }|}g }t        |«      D ]0  }| j                  |d d …|f   ||«      \  }}|j                  |«       Œ2 t	        j                  |d¬«      }| j!                  t#        ||«      «      }|||ffS )NrC   r9   )Úindices_or_sectionsrK   r   rd   )r   r   r   )r?   rS   rf   rO   r;   ÚconcatenateÚpadrU   rQ   Úsiluri   r\   Úrangerr   ÚappendÚstackr^   r   )r'   rA   Ú
conv_cacheÚstate_cachern   ÚTr]   ÚxzÚzÚKÚx_fullÚconv_outÚnew_conv_cacher_   Úcurrent_staterq   ÚtÚy_ts                     r(   Ú_process_sequencezMambaBlock._process_sequence|   s:  € Ø—'‘'‰ˆˆ1ˆaØ�\‰\˜!‹_ˆØ�x‰x¨A°BˆxÓ7‰ˆˆ1Ø×!Ñ!ˆØÐ!Ü—^‘^ Z° O¸!Ô<‰Fä—V‘V˜A ¨¨Q©°¨
°FÐ;Ó<ˆFØ—;‘;˜vÓ&ˆØ¢ Q¨¡U 8¡:ªqÐ 0Ñ1ˆÜ�G‰G�HÓˆÜ�V‰V�D—J‘JÓÐˆØ#ˆØˆÜ�q–ˆAØ!%§¡¨q²°A°©w¸¸=Ó!IÑˆC�Ø�H‰H�S�Mð ô �H‰H�Q˜QÔˆØ�M‰Mœ&  A›,Ó'ˆØ�> =Ð1Ð1Ð1r*   c                 ó�   — |€d\  }}n
|d   |d   }}| j                  |||«      \  }\  }}t        |t        «      r
||d<   ||d<   |S )N)NNr   r   )r‡   Ú
isinstancer   )r'   rA   Úcacher{   r|   Úoutputrƒ   Únew_state_caches           r(   Ú__call__zMambaBlock.__call__’   sf   € Øˆ=Ø&0Ñ#ˆJ™à&+¨A¡h°°a±˜ˆJà48×4JÑ4JØˆz˜;ó5
Ñ1ˆÑ1� ô �eœ[Ô)Ø%ˆE�!‰HØ&ˆE�!‰Hàˆr*   rc   )	r+   r,   r-   r	   rM   rr   r‡   r�   Ú__classcell__©r`   s   @r(   r5   r5   6   s   ø„ ð.
˜Yõ .
ó`ò*2ö,r*   r5   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚResidualBlockr6   c                 óŠ   •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  «      | _        y rc   )rL   rM   r5   ÚmixerrQ   ÚRMSNormr   Únorm©r'   r6   r`   s     €r(   rM   zResidualBlock.__init__¤   s0   ø€ Ü‰ÑÔÜ Ó%ˆŒ
Ü—J‘J˜t×/Ñ/Ó0ˆ�	r*   rA   c                 óJ   — | j                  | j                  |«      |«      |z   S rc   )r“   r•   )r'   rA   rŠ   s      r(   r�   zResidualBlock.__call__©   s    € Ø�z‰z˜$Ÿ)™) A›,¨Ó.°Ñ2Ð2r*   ©	r+   r,   r-   r	   rM   r;   Úarrayr�   rŽ   r�   s   @r(   r‘   r‘   £   s   ø„ ð1˜Yõ 1ð
3˜"Ÿ(™(÷ 3r*   r‘   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚMambar6   c                 ó4  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        |«      ‘Œ c}| _
        t        j                  |j
                  «      | _        y c c}w rc   )rL   rM   rQ   Ú	Embeddingr   r   Ú
embeddingsrx   r   r‘   Úlayersr”   Únorm_f)r'   r6   Ú_r`   s      €r(   rM   zMamba.__init__®   sn   ø€ Ü‰ÑÔÜŸ,™, t§¡¸×8HÑ8HÓIˆŒÜ49¸$×:PÑ:PÔ4QÓRÑ4Q¨q”} TÕ*Ð4QÑRˆŒÜ—j‘j ×!1Ñ!1Ó2ˆ�ùò Ss   ÁBrA   c                 óÊ   — | j                  |«      }|€d gt        | j                  «      z  }t        | j                  |«      D ]  \  }} |||«      }Œ | j	                  |«      S rc   )rž   ÚlenrŸ   Úzipr    )r'   rA   rŠ   ÚlayerÚcs        r(   r�   zMamba.__call__´   s\   € Ø�O‰O˜AÓˆØˆ=Ø�FœS §¡Ó-Ñ-ˆEÜ˜DŸK™K¨Ö/‰HˆE�1Ù�a˜“‰Að 0à�{‰{˜1‹~Ðr*   r˜   r�   s   @r(   r›   r›   ­   s   ø„ ð3˜Yõ 3ð˜"Ÿ(™(÷ r*   r›   c                   ób   ‡ — e Zd Zdefˆ fd„Zddej                  fd„Zd„ Ze	d„ «       Z
d„ Zˆ xZS )	ÚModelr6   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y )NFrD   )rL   rM   r6   r
   r›   Úbackboner   rQ   rR   r   r   Úlm_headr–   s     €r(   rM   zModel.__init__¾   sW   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ˜d›ˆŒØ×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r*   Úinputsc                 óä   — |j                   \  }}| j                  ||«      }| j                  j                  r'| j                  j                  j                  |«      }|S | j                  |«      }|S rc   )r?   rª   r6   r   rž   Ú	as_linearr«   )r'   r¬   rŠ   rn   r}   rA   Úlogitss          r(   r�   zModel.__call__Æ   sa   € Ø�|‰|‰ˆˆ1à�M‰M˜& %Ó(ˆà�9‰9×(Ò(Ø—]‘]×-Ñ-×7Ñ7¸Ó:ˆFð ˆð —\‘\ !“_ˆFàˆr*   c                 óx   — t        t        | j                  «      «      D �cg c]  }t        d¬«      ‘Œ c}S c c}w )NrC   )Úsize)rx   r£   rŸ   r   )r'   r¡   s     r(   Ú
make_cachezModel.make_cacheÒ   s/   € Ü-2´3°t·{±{Ó3CÔ-DÓEÑ-D¨” Ö#Ð-DÑEÐEùÒEs   ¡7c                 ó.   — | j                   j                  S rc   )rª   rŸ   r&   s    r(   rŸ   zModel.layersÕ   s   € à�}‰}×#Ñ#Ð#r*   c                 ó�   — |j                  «       D ]2  \  }}d|v sŒ|j                  d   dk7  sŒ|j                  dd«      ||<   Œ4 |S )Nzconv1d.weightr9   r   rC   )Úitemsr?   Úmoveaxis)r'   ÚweightsÚkÚvs       r(   ÚsanitizezModel.sanitizeÙ   sH   € Ø—M‘M–O‰DˆAˆqØ !Ò#¨¯©°©°qÓ(8ØŸZ™Z¨¨1Ó-�˜’
ð $ð ˆr*   rc   )r+   r,   r-   r	   rM   r;   r™   r�   r²   ÚpropertyrŸ   rº   rŽ   r�   s   @r(   r¨   r¨   ½   s@   ø„ ðT˜Yõ Tñ
˜rŸx™xó 
òFð ñ$ó ð$ör*   r¨   )r$   Údataclassesr   Úmlx.coreÚcorer;   Úmlx.nnrQ   Úactivationsr   Úbaser   rŠ   r   r	   ÚModuler5   r‘   r›   r¨   r3   r*   r(   Ú<module>rÃ      s{   ðó Ý !å Ý å Ý Ý ð ô$%�ó $%ó ð$%ôNj�—‘ô jôZ3�B—I‘Iô 3ôˆB�I‰Iô ô  ˆB�I‰Iõ  r*   