Ë
    îÿæiw!  ã                   ój  — d dl Z d dlmZ d dlmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZ ddlmZ ddlmZ e G d	„ d
e«      «       Z G d„ dej,                  «      Z G d„ dej,                  «      Z G d„ dej,                  «      Z G d„ dej,                  «      Z G d„ dej,                  «      Zy)é    N)Ú	dataclass)ÚOptionalÚTupleÚUnioné   )Úswiglu)ÚBaseModelArgsÚcreate_ssm_mask)ÚArraysCache)Ú
ssm_updatec                   óê   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eeef   ed<   e	eef   ed<   dZ
ee   ed<   dZeed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚ	num_headsÚhead_dimÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚ
state_sizeÚnum_hidden_layersÚlayer_norm_epsilonÚconv_kernelÚn_groupsÚuse_biasÚuse_conv_biasÚtie_word_embeddingsÚtime_step_limitÚtime_step_rankNÚssm_state_sizei  Úmax_position_embeddingsc                 ó¬   — | j                   dk(  r't        j                  | j                  dz  «      | _         | j                  €| j
                  | _        y y )NÚautoé   )r   ÚmathÚceilr   r   r   ©Úselfs    úi/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/mamba2.pyÚ__post_init__zModelArgs.__post_init__%   sI   € Ø×Ñ &Ò(Ü"&§)¡)¨D×,<Ñ,<¸rÑ,AÓ"BˆDÔØ×ÑÐ&Ø"&§/¡/ˆDÕð 'ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatÚboolr   r   r   r   r    r)   © r*   r(   r   r      sŠ   … àƒOØƒNØƒMØƒOØÓØÓØƒOØÓØÓØÓØƒMØƒNØÓØÓØ˜5 %˜<Ñ(Ó(Ø˜#˜s˜(‘OÓ#Ø$(€N�H˜S‘MÓ(Ø#'Ð˜SÓ'ó2r*   r   c                   ó|   ‡ — e Zd Zddedefˆ fd„Zd	dej                  dej                  dej                  fd„Zˆ xZ	S )
ÚMambaRMSNormGatedr   Úepsc                 ód   •— t         ‰| �  «        || _        t        j                  |«      | _        y ©N)ÚsuperÚ__init__r6   ÚmxÚonesÚweight)r'   r   r6   Ú	__class__s      €r(   r:   zMambaRMSNormGated.__init__-   s%   ø€ Ü‰ÑÔØˆŒÜ—g‘g˜kÓ*ˆ�r*   Úhidden_statesÚgateÚreturnc                 óˆ   — |�t        ||«      }t        j                  j                  || j                  | j
                  «      S r8   )r   r;   ÚfastÚrms_normr=   r6   )r'   r?   r@   s      r(   Ú__call__zMambaRMSNormGated.__call__2   s5   € ØÐÜ" 4¨Ó7ˆMÜ�w‰w×Ñ ¨t¯{©{¸D¿H¹HÓEÐEr*   )g�íµ ÷Æ°>r8   )
r+   r,   r-   r0   r1   r:   r;   ÚarrayrE   Ú__classcell__©r>   s   @r(   r5   r5   ,   s@   ø„ ñ+ Cð +¨eõ +ñ
F b§h¡hð F°b·h±hð FÈ"Ï(É(÷ Fr*   r5   c                   ó”  ‡ — e Zd Zdedefˆ fd„Zdej                  dee	   deej                     dej                  fd„Z
d	ej                  d
ej                  dej                  dej                  dee	   deej                     dej                  fd„Z	 dd	ej                  deej                     dee	   dej                  fd„Zˆ xZS )ÚMamba2BlockÚargsÚ	layer_idxc                 óp  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  |j                  z  | _	        |j                  | _
        |j                  | _        |j                  | _        |j                  | _        | j                  | j                  z  | _        |j                  | _        | j                  d| j                  z  | j
                  z  z   | _        t!        j"                  | j                  | j                  |j                  d| j                  |j                  ¬«      | _        | j                  | j                  z   | j                  z   }t!        j&                  | j                  ||j                  ¬«      | _        t+        j,                  | j                  «      | _        t+        j0                  t+        j2                  d| j                  dz   t*        j4                  ¬«      «      | _        t+        j,                  | j                  «      | _        t;        | j                  |j<                  ¬«      | _        t!        j&                  | j                  | j                  |j                  ¬«      | _         y )Né   r   )Úin_channelsÚout_channelsÚkernel_sizeÚpaddingÚgroupsÚbias©rT   r   ©Údtype©r6   )!r9   r:   rL   r   r   r   r   Úconv_kernel_sizer   r   r   r   r   Úheads_per_groupr   Úconv_dimÚnnÚConv1dÚconv1dÚLinearÚin_projr;   r<   Údt_biasÚlogÚarangeÚfloat32ÚA_logÚDr5   r   ÚnormÚout_proj)r'   rK   rL   Úprojection_sizer>   s       €r(   r:   zMamba2Block.__init__9   sæ  ø€ Ü‰ÑÔØ"ˆŒØŸ™ˆŒØ×+Ñ+ˆÔØ"×1Ñ1ˆÔØ $× 0Ñ 0ˆÔØ!%§¡°$·-±-Ñ!?ˆÔØ!×/Ñ/ˆÔØŸ™ˆŒØŸ™ˆŒØ#×3Ñ3ˆÔØ#Ÿ~™~°·±Ñ>ˆÔØŸ™ˆŒà×.Ñ.°°T·]±]Ñ1BÀT×EXÑEXÑ1XÑXˆŒä—i‘iØŸ™ØŸ™Ø×(Ñ(ØØ—=‘=Ø×#Ñ#ô
ˆŒð ×0Ñ0°4·=±=Ñ@À4Ç>Á>ÑQˆÜ—y‘y ×!1Ñ!1°?ÈÏÉÔWˆŒä—w‘w˜tŸ~™~Ó.ˆŒÜ—V‘VœBŸI™I a¨¯©¸!Ñ);Ä2Ç:Á:ÔNÓOˆŒ
Ü—‘˜Ÿ™Ó(ˆŒä%Ø×"Ñ"¨×(?Ñ(?ô
ˆŒ	ô Ÿ	™	Ø×"Ñ" D×$4Ñ$4¸4¿=¹=ô
ˆ�r*   Ú
conv_inputÚcacheÚmaskrA   c                 óò  — |�t        j                  |d   |d«      }|��|d   €It        j                  |j                  d   | j                  dz
  | j
                  f|j                  ¬«      }n|d   }t        j                  ||gd¬«      }| j                  dz
  }|j                  �q|j                  d   }t        j                  |j                  d||z
  «      }|d d …d f   t        j                  |«      z   d   }	t        j                  ||	d¬«      |d<   n;|d d …| d …d d …f   |d<   n(t        j                  |d| j                  dz
  dfdg«      }| j                  |«      }
t        j                  |
«      S )N).Nr   r   rV   ©Úaxis)r   r   )r;   ÚwhereÚzerosÚshaperY   r[   rW   ÚconcatenateÚlengthsÚcliprc   Útake_along_axisÚpadr^   r\   Úsilu)r'   rj   rk   rl   Ú
conv_stateÚpadded_inputÚn_keepÚtÚendsÚ	positionsÚconv_outputs              r(   Ú_convzMamba2Block._conva   sh  € ð ÐÜŸ™ $ y¡/°:¸qÓAˆJàÑØ�Q‰xÐÜŸX™XØ×%Ñ% aÑ(¨$×*?Ñ*?À!Ñ*CÀTÇ]Á]ÐSØ$×*Ñ*ô‘
ð
 # 1™X�
ÜŸ>™>¨:°zÐ*BÈÔKˆLØ×*Ñ*¨QÑ.ˆFØ�}‰}Ð(Ø ×&Ñ& qÑ)�Ü—w‘w˜uŸ}™}¨a°°V±Ó<�Ø!¢! T '™]¬R¯Y©Y°vÓ->Ñ>À	ÑJ�	Ü×-Ñ-¨l¸IÈAÔN��a’à'ª¨F¨7©8²Q¨Ñ7��a’äŸ6™6Ø˜V d×&;Ñ&;¸aÑ&?ÀÐ%CÀVÐLóˆLð —k‘k ,Ó/ˆÜ�w‰w�{Ó#Ð#r*   r?   ÚBÚCÚdtc                 ó  — |j                   \  }}}	|j                  ||| j                  | j                  «      }|j                  ||| j                  | j
                  «      }|j                  ||| j                  | j
                  «      }|r|d   }
|j                  }nd\  }
}t        || j                  ||| j                  || j                  |
| j                  ||«      \  }}
|r|
|d<   |j                  ||| j                  «      S )Nr   )NN)rr   Úreshaper   r   r   r   rt   r   re   rf   ra   r   r   )r'   r?   r�   r‚   rƒ   rk   rl   Ú
batch_sizeÚseq_lenÚ_Ústatert   Úys                r(   Ú_ssmzMamba2Block._ssmƒ   sû   € ð "/×!4Ñ!4Ñˆ
�G˜QØ%×-Ñ-Ø˜ §¡°·±ó
ˆð �I‰I�j '¨4¯=©=¸$×:MÑ:MÓNˆØ�I‰I�j '¨4¯=©=¸$×:MÑ:MÓNˆÙØ˜!‘HˆEØ—m‘m‰Gà'‰NˆE�7ÜØØ�J‰JØØØ�F‰FØØ�L‰LØØ× Ñ ØØó
‰ˆˆ5ñ ØˆE�!‰HØ�y‰y˜ W¨d×.DÑ.DÓEÐEr*   c                 ó  — | j                  |«      }t        j                  || j                  | j                  | j                  z   gd¬«      \  }}}| j                  |||«      }t        j                  || j                  | j                  | j                  | j                  z  z   gd¬«      \  }}	}
| j                  ||	|
|||¬«      }|r|j                  |j                  d   «       | j                  ||«      }| j                  |«      S )Néÿÿÿÿrn   )rl   r   )r`   r;   Úsplitr   r[   r€   r   r   r‹   Úadvancerr   rg   rh   )r'   r?   rl   rk   Ú	projectedr@   rj   rƒ   r   r�   r‚   rŠ   s               r(   rE   zMamba2Block.__call__¨   sø   € ð —L‘L Ó/ˆ	Ü!Ÿx™xØØ×#Ñ# T×%;Ñ%;¸d¿m¹mÑ%KÐLØô 
Ñˆˆj˜"ð
 —j‘j ¨U°DÓ9ˆÜ Ÿh™hØà×&Ñ&Ø×&Ñ&¨¯©¸×9LÑ9LÑ)LÑLðð ô
Ñˆ�q˜!ð �I‰I�m Q¨¨2¨u¸4ˆIÓ@ˆÙØ�M‰M˜!Ÿ'™' !™*Ô%Ø�I‰I�a˜ÓˆØ�}‰}˜QÓÐr*   r8   )r+   r,   r-   r   r0   r:   r;   rF   r   r   r€   r‹   rE   rG   rH   s   @r(   rJ   rJ   8   s  ø„ ð&
˜Yð &
°3õ &
ðP $à—H‘Hð $ð ˜Ñ$ð $ð �r—x‘xÑ ð	 $ð
 
�‰ó $ðD#Fà—x‘xð#Fð �8‰8ð#Fð �8‰8ð	#Fð
 �H‰Hð#Fð ˜Ñ$ð#Fð �r—x‘xÑ ð#Fð 
�‰ó#FðR (,ñ	 à—x‘xð ð �r—x‘xÑ ð ð ˜Ñ$ð	 ð
 
�‰÷ r*   rJ   c            	       óŒ   ‡ — e Zd Zdedefˆ fd„Z	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚResidualBlockrK   rL   c                 óŒ   •— t         ‰| �  «        t        ||«      | _        t	        j
                  |j                  «      | _        y r8   )r9   r:   rJ   Úmixerr\   ÚRMSNormr   rg   )r'   rK   rL   r>   s      €r(   r:   zResidualBlock.__init__Å   s2   ø€ Ü‰ÑÔÜ   yÓ1ˆŒ
Ü—J‘J˜t×/Ñ/Ó0ˆ�	r*   Úxrl   rk   rA   c                 óP   — | j                  | j                  |«      ||«      }||z   S r8   )r”   rg   )r'   r–   rl   rk   Úoutputs        r(   rE   zResidualBlock.__call__Ê   s(   € ð —‘˜DŸI™I a›L¨$°Ó6ˆØ˜‰zÐr*   r8   )r+   r,   r-   r   r0   r:   r;   rF   r   r   rE   rG   rH   s   @r(   r’   r’   Ä   sX   ø„ ð1˜Yð 1°3õ 1ð UYñØ—‘ðØ!)¨"¯(©(Ñ!3ðØ<DÀ[Ñ<Qðà	�‰÷r*   r’   c                   óp   ‡ — e Zd Zdefˆ fd„Z	 ddej                  deee	      dej                  fd„Z
ˆ xZS )ÚMamba2rK   c                 ó\  •— t         ‰| �  «        || _        t        j                  |j
                  |j                  «      | _        t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        y c c}w )NrX   )r9   r:   rK   r\   Ú	Embeddingr   r   Ú
embeddingsÚranger   r’   Úlayersr•   r   Únorm_f)r'   rK   Úir>   s      €r(   r:   zMamba2.__init__Ò   s   ø€ Ü‰ÑÔØˆŒ	ÜŸ,™, t§¡¸×8HÑ8HÓIˆŒÜ7<¸T×=SÑ=SÔ7TÓUÑ7T°!”} T¨1Õ-Ð7TÑUˆŒÜ—j‘j ×!1Ñ!1°t×7NÑ7NÔOˆ�ùò Vs   ÁB)r–   rk   rA   c                 óê   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||«      }Œ | j                  |«      S )Nr   )r�   ÚlenrŸ   r
   Úzipr    )r'   r–   rk   Úhiddenrl   ÚlayerÚcs          r(   rE   zMamba2.__call__Ù   st   € ð —‘ Ó#ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä˜v u¨Q¡xÓ0ˆÜ˜DŸK™K¨Ö/‰HˆE�1Ù˜6 4¨Ó+‰Fð 0ð �{‰{˜6Ó"Ð"r*   r8   )r+   r,   r-   r   r:   r;   rF   r   Úlistr   rE   rG   rH   s   @r(   rš   rš   Ñ   sG   ø„ ðP˜Yõ Pð AEñ#Ø—‘ð#Ø"*¨4°Ñ+<Ñ"=ð#à	�‰÷#r*   rš   c                   óž   ‡ — e Zd Zdefˆ fd„Z	 ddej                  deee	      dej                  fd„Z
ddedee	   fd„Zed	„ «       Zd
„ Zˆ xZS )ÚModelrK   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y )NFrU   )r9   r:   rK   r   rš   Úbackboner   r\   r_   r   r   Úlm_head)r'   rK   r>   s     €r(   r:   zModel.__init__é   sW   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ˜t›ˆŒà×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r*   Úinputsrk   rA   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S r8   )r¬   rK   r   r�   Ú	as_linearr­   )r'   r®   rk   r¥   Úlogitss        r(   rE   zModel.__call__ò   sW   € ð —‘˜v uÓ-ˆà�9‰9×(Ò(Ø—]‘]×-Ñ-×7Ñ7¸Ó?ˆFð ˆð —\‘\ &Ó)ˆFØˆr*   r†   c                 óz   — t        | j                  j                  «      D �cg c]  }t        d¬«      ‘Œ c}S c c}w )NrN   )Úsize)rž   rK   r   r   )r'   r†   rˆ   s      r(   Ú
make_cachezModel.make_cacheý   s0   € Ü-2°4·9±9×3NÑ3NÔ-OÓPÑ-O¨” Ö#Ð-OÑPÐPùÒPs   ¢8c                 ó.   — | j                   j                  S r8   )r¬   rŸ   r&   s    r(   rŸ   zModel.layers   s   € à�}‰}×#Ñ#Ð#r*   c                 ó�   — |j                  «       D ]2  \  }}d|v sŒ|j                  d   dk7  sŒ|j                  dd«      ||<   Œ4 |S )Nzconv1d.weightr�   r   rN   )Úitemsrr   Úmoveaxis)r'   ÚweightsÚkÚvs       r(   ÚsanitizezModel.sanitize  sH   € Ø—M‘M–O‰DˆAˆqØ !Ò#¨¯©°©°qÓ(8ØŸZ™Z¨¨1Ó-�˜’
ð $ð ˆr*   r8   )r   )r+   r,   r-   r   r:   r;   rF   r   r¨   r   rE   r0   r´   ÚpropertyrŸ   r¼   rG   rH   s   @r(   rª   rª   è   s{   ø„ ðT˜Yõ Tð FJñ	Ø—h‘hð	Ø'/°°[Ñ0AÑ'Bð	à	�‰ó	ñQ Sð Q°°kÑ1Bó Qð ñ$ó ð$ör*   rª   )r$   Údataclassesr   Útypingr   r   r   Úmlx.coreÚcorer;   Úmlx.nnr\   Úactivationsr   Úbaser	   r
   rk   r   Ússmr   r   ÚModuler5   rJ   r’   rš   rª   r3   r*   r(   Ú<module>rÇ      s•   ðó Ý !ß )Ñ )å Ý å ß 0Ý Ý ð ô2�ó 2ó ð2ô6	F˜Ÿ	™	ô 	FôI �"—)‘)ô I ôX
�B—I‘Iô 
ô#ˆR�Y‰Yô #ô. ˆB�I‰Iõ  r*   