Ë
    îÿæiI&  ã                   óŒ  — d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZmZ ddlmZmZ e G d„ d	e«      «       Z G d
„ de
j,                  «      Z G d„ de
j,                  «      Z G d„ de
j,                  «      Z G d„ de
j,                  «      Z G d„ de
j,                  «      Z G d„ de
j,                  «      Zy)é    )Ú	dataclass)ÚAnyÚListÚOptionalNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_maskÚscaled_dot_product_attention)ÚArraysCacheÚKVCachec                   ó   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   dZeed<   dZ	e
e   ed<   dZe
ee      ed<   dZe
ee      ed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚ
vocab_sizeÚhidden_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚmax_position_embeddingsÚnorm_epsÚ	conv_biasÚconv_L_cacheÚ	block_dimÚblock_ff_dimÚblock_multiple_ofÚblock_ffn_dim_multiplierÚblock_auto_adjust_ff_dimg    €„.AÚ
rope_thetaNÚrope_parametersÚfull_attn_idxsÚlayer_typesc                 ó$  — | j                   �"d| j                   v r| j                   d   | _        | j                  €| j                  | _        | j                  €3t        | j                  «      D ��cg c]  \  }}|dk(  r|‘Œ c}}| _        y y c c}}w )Nr    Úfull_attention)r!   r    r   r   r"   Ú	enumerater#   )ÚselfÚiÚ
layer_types      úg/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/lfm2.pyÚ__post_init__zModelArgs.__post_init__(   s—   € Ø×ÑÐ+°À×@TÑ@TÑ0TØ"×2Ñ2°<Ñ@ˆDŒOØ×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$Ø×ÑÐ&ô &/¨t×/?Ñ/?Ô%@ô#á%@‘M�A�zØÐ!1Ò1ò Ø%@ò#ˆDÕð 'ùó#s   Á0B)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatÚboolr    r!   r   Údictr"   r   r#   r+   © ó    r*   r   r      sš   … àƒOØƒOØÓØÓØÓØÓØ Ó ØƒOØƒOØÓØƒNØÓØÓØ#Ó#Ø"Ó"Ø!€J�Ó!Ø&*€O�X˜d‘^Ó*Ø*.€N�H˜T #™YÑ'Ó.Ø'+€K�˜$˜s™)Ñ$Ó+ó
r6   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	Ú	AttentionÚargsc                 óÀ  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  x| _        }|dz  | _        t        j                  ||j                  ¬«      | _        t        j                  ||j                  ¬«      | _        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  ||z  |d¬«      | _        t        j&                  | j                  |j(                  d¬«      | _        y )Ng      à¿©ÚepsF©Úbias)ÚbaseÚtraditional)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsÚhead_dimÚscaleÚnnÚRMSNormr   Úq_layernormÚk_layernormÚLinearÚq_projÚk_projÚv_projÚout_projÚRoPEr    Úrope)r'   r9   ÚdimrC   rD   rE   Ú	__class__s         €r*   rB   zAttention.__init__6   s  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à#'×#3Ñ#3°wÑ#>Ð>ˆŒ˜à˜t‘^ˆŒ
äŸ:™: h°D·M±MÔBˆÔÜŸ:™: h°D·M±MÔBˆÔä—i‘i  W¨xÑ%7¸eÔDˆŒÜ—i‘i  Z°(Ñ%:ÀÔGˆŒÜ—i‘i  Z°(Ñ%:ÀÔGˆŒÜŸ	™	 '¨HÑ"4°cÀÔFˆŒä—G‘GØ�M‰MØ—‘Øô
ˆ�	r6   ÚxÚmaskÚcacheÚreturnc                 ón  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}| j	                  |j                  ||| j                  d«      «      j                  dddd«      }| j                  |j                  ||| j                  d«      «      j                  dddd«      }|	j                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	||| j                  ¬«      }
|
j                  dddd«      j                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   é   r   é   )Úoffset)rV   rU   rF   )ÚshaperL   rM   rN   rI   ÚreshaperC   Ú	transposerJ   rD   rQ   r\   Úupdate_and_fetchr   rF   rO   )r'   rT   rU   rV   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r*   Ú__call__zAttention.__call__O   s‡  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆà×"Ñ" 7§?¡?°1°a¸¿¹ÀrÓ#JÓK×UÑUØˆq�!�Qó
ˆð ×Ñ §¡¨Q°°4·?±?ÀBÓ GÓH×RÑRØˆq�!�Qó
ˆð —‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°TÀÇÁô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�}‰}˜VÓ$Ð$r6   ©NN)r,   r-   r.   r   rB   ÚmxÚarrayr   r   rh   Ú__classcell__©rS   s   @r*   r8   r8   5   sW   ø„ ð
˜Yõ 
ð8 $(Ø#ñ	%à�8‰8ð%ð �r—x‘xÑ ð%ð ˜‰}ð	%ð
 
�‰÷%r6   r8   c                   óv   ‡ — e Zd Zdedefˆ fd„Z	 	 ddej                  deej                     dee	   fd„Z
ˆ xZS )	Ú	ShortConvr9   Ú	layer_idxc                 ó  •— t         ‰| �  «        || _        || _        |j                  | _        |j                  | _        t        j                  |j                  |j                  | j
                  |j                  | j                  ¬«      | _        t        j                  |j                  d|j                  z  | j                  ¬«      | _        t        j                  |j                  |j                  | j                  ¬«      | _        y )N)Úin_channelsÚout_channelsÚkernel_sizeÚgroupsr>   r[   r=   )rA   rB   r9   rp   r   ÚL_cacher   r>   rG   ÚConv1dr   ÚconvrK   Úin_projrO   ©r'   r9   rp   rS   s      €r*   rB   zShortConv.__init__q   s¾   ø€ ô
 	‰ÑÔØˆŒ	Ø"ˆŒØ×(Ñ(ˆŒØ—N‘NˆŒ	ä—I‘IØ×(Ñ(Ø×)Ñ)ØŸ™Ø×#Ñ#Ø—‘ô
ˆŒ	ô —y‘y ×!1Ñ!1°1°t×7GÑ7GÑ3GÈdÏiÉiÔXˆŒÜŸ	™	 $×"2Ñ"2°D×4DÑ4DÈ4Ï9É9ÔUˆ�r6   rT   rU   rV   c                 óˆ  — | j                  |«      }t        j                  |dd¬«      \  }}}||z  }|�t        j                  |d   |d«      }|��#|d   €St        j                  |j
                  d   | j                  dz
  | j                  j                  f|j                  ¬«      }n|d   }t        j                  ||gd¬«      }| j                  dz
  }	|j
                  d   }
|j                  �_t        j                  |j                  d|
«      }|d d …d f   t        j                  |	«      z   d   }t        j                  ||d¬«      |d<   n|d d …|	 d …d d …f   |d<   |j                  |
«       n(t        j                   |d| j                  dz
  dfdg«      }| j#                  |«      }||z  }| j%                  |«      S )	Nr[   rY   )Úaxis).Nr   r   )Údtype)r   r   )ry   rj   ÚsplitÚwhereÚzerosr]   rv   r9   r   r}   ÚconcatenateÚlengthsÚclipÚarangeÚtake_along_axisÚadvanceÚpadrx   rO   )r'   rT   rU   rV   ÚBCxra   ÚCÚBxÚstateÚn_keepÚtÚendsÚ	positionsÚconv_outÚys                  r*   rh   zShortConv.__call__†   s˜  € ð �l‰l˜1‹oˆÜ—(‘(˜3 ¨Ô+‰ˆˆ1ˆaØ�‰UˆØÐÜ—‘˜$˜y™/¨2¨qÓ1ˆBàÑØ�Q‰xÐÜŸ™Ø—X‘X˜a‘[ $§,¡,°Ñ"2°D·I±I×4IÑ4IÐJØŸ(™(ô‘ð
 ˜a™�Ü—‘ ¨ °!Ô4ˆBØ—\‘\ AÑ%ˆFØ—‘˜‘
ˆAØ�}‰}Ð(Ü—w‘w˜uŸ}™}¨a°Ó3�Ø!¢! T '™]¬R¯Y©Y°vÓ->Ñ>À	ÑJ�	Ü×-Ñ-¨b°)À!ÔD��a’àša & ¡ª1˜nÑ-��a‘Ø�M‰M˜!Õä—‘˜˜V d§l¡l°QÑ&6¸Ð%:¸FÐCÓDˆBà—9‘9˜R“=ˆà�‰LˆØ�}‰}˜QÓÐr6   ri   ©r,   r-   r.   r   r1   rB   rj   rk   r   r   rh   rl   rm   s   @r*   ro   ro   p   sW   ø„ ðVàðVð õVð0 $(Ø#ñ	$ à�8‰8ð$ ð �r—x‘xÑ ð$ ð ˜‰}÷	$ r6   ro   c                   óZ   ‡ — e Zd Zdededededee   f
ˆ fd„Zdej                  fd„Z
ˆ xZS )	ÚMLPrR   Úff_dimÚmultiple_ofÚauto_adjust_ff_dimÚffn_dim_multiplierc                 ó2  •— t         ‰| �  «        |r/t        d|z  dz  «      }|�t        ||z  «      }|||z   dz
  |z  z  }t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        y )NrZ   r[   r   Fr=   )rA   rB   r1   rG   rK   Úw1Úw3Úw2)r'   rR   r•   r–   r—   r˜   rS   s         €r*   rB   zMLP.__init__®   s�   ø€ ô 	‰ÑÔÙÜ˜˜V™ a™Ó(ˆFØ!Ð-ÜÐ/°&Ñ8Ó9�Ø  V¨kÑ%9¸AÑ%=À+Ñ$MÑNˆFä—)‘)˜C ¨eÔ4ˆŒÜ—)‘)˜C ¨eÔ4ˆŒÜ—)‘)˜F C¨eÔ4ˆ�r6   rW   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)rœ   r   rš   r›   )r'   rT   s     r*   rh   zMLP.__call__Á   s(   € Ø�w‰w”v˜dŸg™g a›j¨$¯'©'°!«*Ó5Ó6Ð6r6   )r,   r-   r.   r1   r3   r   r2   rB   rj   rk   rh   rl   rm   s   @r*   r”   r”   ­   sL   ø„ ð5àð5ð ð5ð ð	5ð
 !ð5ð % U™Oõ5ð&7˜RŸX™X÷ 7r6   r”   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚLfm2DecoderLayerr9   rp   c                 óò  •— t         ‰| �  «        ||j                  v | _        | j                  rt	        |«      | _        nt        ||«      | _        t        |j                  |j                  |j                  |j                  |j                  ¬«      | _        t        j                   |j"                  |j$                  ¬«      | _        t        j                   |j"                  |j$                  ¬«      | _        y )N)rR   r•   r–   r—   r˜   r;   )rA   rB   r"   Úis_attention_layerr8   Ú	self_attnro   rx   r”   r   r   r   r   r   Úfeed_forwardrG   rH   r   r   Úoperator_normÚffn_normrz   s      €r*   rB   zLfm2DecoderLayer.__init__Æ   s¶   ø€ Ü‰ÑÔØ"+¨t×/BÑ/BÐ"BˆÔà×"Ò"Ü& t›_ˆD�Nä! $¨	Ó2ˆDŒIÜØ—‘Ø×$Ñ$Ø×.Ñ.Ø#×<Ñ<Ø#×<Ñ<ô
ˆÔô  ŸZ™Z¨×(8Ñ(8¸d¿m¹mÔLˆÔÜŸ
™
 4×#3Ñ#3¸¿¹ÔGˆ�r6   rT   rU   rV   rW   c                 óü   — | j                   r$| j                  | j                  |«      ||¬«      }n#| j                  | j                  |«      ||¬«      }||z   }|| j	                  | j                  |«      «      z   }|S )N)rU   rV   )r¢   r£   r¥   rx   r¤   r¦   )r'   rT   rU   rV   ÚrÚhÚouts          r*   rh   zLfm2DecoderLayer.__call__Ù   s�   € ð ×"Ò"Ø—‘˜t×1Ñ1°!Ó4¸4Àu�ÓM‰Aà—	‘	Ø×"Ñ" 1Ó%ØØð ó ˆAð
 �‰EˆØ�$×#Ñ# D§M¡M°!Ó$4Ó5Ñ5ˆØˆ
r6   ri   r’   rm   s   @r*   r    r    Å   sa   ø„ ðH˜Yð H°3õ Hð, $(Ø#ñ	à�8‰8ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷r6   r    c                   óh   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     fd„Zˆ xZ	S )Ú	Lfm2Modelr9   c                 óV  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  |j                  |j                  «      | _        t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        |j                  d   | _        d| _        t        |j                  «      D ]'  }||j                  v r| xj"                  dz  c_        Œ' y  y c c}w )N)rp   r;   r   r   )rA   rB   r9   r   r   rG   Ú	Embeddingr   Úembed_tokensÚranger    ÚlayersrH   r   Úembedding_normr"   Úfa_idxÚconv_idx)r'   r9   r(   rS   s      €r*   rB   zLfm2Model.__init__î   së   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔä9>¸t×?UÑ?UÔ9Vó
Ù9V°AÔ˜T¨QÖ/Ð9Vñ
ˆŒô !Ÿj™j¨×)9Ñ)9¸t¿}¹}ÔMˆÔà×)Ñ)¨!Ñ,ˆŒØˆŒÜ�t×-Ñ-Ö.ˆAØ�D×'Ñ'Ñ'Ø—’ Ñ"–áñ	 /ùò
s   Á?D&ÚinputsÚinput_embeddingsc                 ó\  — |�|}n| j                  |«      }|€d gt        | j                  «      z  }t        ||| j                     «      }t        ||| j                     «      }t        | j                  |«      D ]   \  }}|j                  r|n|}	 |||	|¬«      }Œ" | j                  |«      S )N)rV   )
r¯   Úlenr±   r
   r³   r   r´   Úzipr¢   r²   )
r'   rµ   rV   r¶   r©   Ú	attn_maskÚ	conv_maskÚlayerÚcrU   s
             r*   rh   zLfm2Model.__call__  s¬   € ð Ð'Ø ‰Aà×!Ñ! &Ó)ˆAàˆ=Ø�FœS §¡Ó-Ñ-ˆEä)¨!¨U°4·;±;Ñ-?Ó@ˆ	Ü# A u¨T¯]©]Ñ';Ó<ˆ	ä˜DŸK™K¨Ö/‰HˆE�1Ø %× 8Ò 8‘9¸iˆDÙ�a˜ QÔ'‰Að 0ð ×"Ñ" 1Ó%Ð%r6   ri   )
r,   r-   r.   r   rB   rj   rk   r   rh   rl   rm   s   @r*   r¬   r¬   í   s;   ø„ ð˜Yõ ð. Ø/3ñ	&à—‘ð&ð # 2§8¡8Ñ,÷	&r6   r¬   c                   ó„   ‡ — e Zd Zdefˆ fd„Z	 	 d	dej                  deej                     fd„Zd„ Z	e
d„ «       Zd„ Zˆ xZS )
ÚModelr9   c                 ór   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        y rž   )rA   rB   r9   r   r¬   Úmodel)r'   r9   rS   s     €r*   rB   zModel.__init__  s+   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ˜t“_ˆ�
r6   rµ   r¶   c                 ór   — | j                  |||«      }| j                   j                  j                  |«      S rž   )rÁ   r¯   Ú	as_linear)r'   rµ   rV   r¶   rª   s        r*   rh   zModel.__call__!  s2   € ð �j‰j˜ Ð(8Ó9ˆØ�z‰z×&Ñ&×0Ñ0°Ó5Ð5r6   c                 ó°   — i }|j                  «       D ]@  \  }}d|v r2|j                  d   |j                  d   kD  r|j                  ddd«      }|||<   ŒB |S )Nzconv.weightrY   r   r   rZ   )Úitemsr]   r_   )r'   ÚweightsÚsanitized_weightsÚnameÚparams        r*   ÚsanitizezModel.sanitize*  sa   € ØÐØ"Ÿ=™=ž?‰KˆD�%Ø Ñ$Ø—;‘;˜r‘? U§[¡[°¡^Ò3Ø!ŸO™O¨A¨q°!Ó4�Eà&+Ð˜dÒ#ð +ð !Ð r6   c                 ó.   — | j                   j                  S rž   )rÁ   r±   )r'   s    r*   r±   zModel.layers4  s   € à�z‰z× Ñ Ð r6   c                 ó€   — | j                   D �cg c]$  }|j                  r
t        «       nt        d¬«      ‘Œ& c}S c c}w )Nr   )Úsize)r±   r¢   r   r   )r'   Úls     r*   Ú
make_cachezModel.make_cache8  sA   € ð —[’[ó
á �ð ×-Ò-ŒGŒI´;ÀAÔ3FÑFØ ñ
ð 	
ùò 
s   �);ri   )r,   r-   r.   r   rB   rj   rk   r   rh   rÊ   Úpropertyr±   rÏ   rl   rm   s   @r*   r¿   r¿     sY   ø„ ð%˜Yõ %ð Ø/3ñ	6à—‘ð6ð # 2§8¡8Ñ,ó	6ò!ð ñ!ó ð!ö
r6   r¿   )Údataclassesr   Útypingr   r   r   Úmlx.coreÚcorerj   Úmlx.nnrG   Úactivationsr   r?   r	   r
   r   r   rV   r   r   r   ÚModuler8   ro   r”   r    r¬   r¿   r5   r6   r*   Ú<module>rØ      s¦   ðå !ß &Ñ &å Ý å ÷ó ÷ (ð ô�ó ó ðôD8%�—	‘	ô 8%ôv: �—	‘	ô : ôz7ˆ"�)‰)ô 7ô0%�r—y‘yô %ôP*&�—	‘	ô *&ôZ"
ˆB�I‰Iõ "
r6   