Ë
    îÿæi�  ã                   óž  — d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ ddlmZ ddlmZmZmZ e G d„ de«      «       Z	 	 dd	eeef   d
ee   deeeef      deeef   fd„Z G d„ dej.                  «      Z G d„ dej.                  «      Z G d„ dej.                  «      Z G d„ dej.                  «      Z G d„ dej.                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚListÚOptionalÚUnionNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attentionc                   ó®   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   d
Z	eed<   d
Z
eed<   dZeed<   dZeed<   y)Ú	ModelArgsÚ
model_typeÚhead_dimÚnum_transformer_layersÚ	model_dimÚ
vocab_sizeÚffn_dim_divisorÚnum_query_headsÚnum_kv_headsÚffn_multipliersTÚffn_with_gluÚnormalize_qk_projectionsÚshare_input_output_layersg�íµ ÷Æ°>Úrms_norm_epsi'  Úrope_freq_constantN)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__Úintr   r   Úboolr   r   r   Úfloatr   © ó    új/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/openelm.pyr   r      si   … àƒOØƒMØÓØƒNØƒOØÓØÓØÓØÓØ€L�$ÓØ%)Ð˜dÓ)Ø&*Ð˜tÓ*Ø€L�%ÓØ %Ð˜Ô%r'   r   ÚvÚdivisorÚ	min_valueÚreturnc                 ój   — |€|}t        |t        | |dz  z   «      |z  |z  «      }|d| z  k  r||z  }|S )aÅ  
    This function is taken from the original tf repo.
    It ensures that all layers have a channel number that is divisible by the divisor
    It can be seen at:
    https://github.com/tensorflow/models/blob/2cfc99eff5e5eb729c6793d2f3d03aa1c9be2b15/research/slim/nets/mobilenet/mobilenet.py#L62
    Args:
        v: input value
        divisor: default to 8
        min_value: minimum divisor value
    Returns:
        new_v: new divisible value
    é   gÍÌÌÌÌÌì?)Úmaxr#   )r)   r*   r+   Únew_vs       r(   Úmake_divisibler1      sL   € ð" ÐØˆ	Ü�	œ3˜q 7¨Q¡;™Ó/°7Ñ:¸WÑDÓE€Eàˆs�Q‰w‚Ø�ÑˆØ€Lr'   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
Ú	AttentionÚargsÚlayer_idc                 ó˜  •— t         ‰| �  «        |j                  x| _        }|| _        |j                  x| _        }|j
                  |   x| _        }|j                  |   x| _        }|dz  | _	        ||dz  z   |z  }t        j                  ||d¬«      | _        t        j                  ||z  |d¬«      | _        |j                  | _        | j                  rLt        j                  ||j                   ¬«      | _        t        j                  ||j                   ¬«      | _        t        j&                  |d|j(                  ¬«      | _        y )Ng      à¿r.   F©Úbias©Úeps)ÚtraditionalÚbase)ÚsuperÚ__init__r   r5   r   r   Ún_headsr   Ú
n_kv_headsÚscaleÚnnÚLinearÚqkv_projÚout_projr   ÚRMSNormr   Úq_normÚk_normÚRoPEr   Úrope)	Úselfr4   r5   r   r   r?   r@   Úop_sizeÚ	__class__s	           €r(   r>   zAttention.__init__:   s  ø€ Ü‰ÑÔØ#'§=¡=Ð0ˆŒ˜Ø ˆŒØ%)§^¡^Ð3ˆŒ˜à!%×!5Ñ!5°hÑ!?Ð?ˆŒ�wØ'+×'8Ñ'8¸Ñ'BÐBˆŒ˜*Ø˜t‘^ˆŒ
à˜j¨1™nÑ-°Ñ9ˆÜŸ	™	 )¨W¸5ÔAˆŒÜŸ	™	 '¨HÑ"4°iÀeÔLˆŒà(,×(EÑ(EˆÔ%à×(Ò(ÜŸ*™* X°4×3DÑ3DÔEˆDŒKÜŸ*™* X°4×3DÑ3DÔEˆDŒKä—G‘G˜H°%¸d×>UÑ>UÔVˆ�	r'   ÚxÚmaskÚcacher,   c                 ó<  — |j                   \  }}}| j                  |«      }|j                  ||| j                  | j                  dz  z   | j
                  «      j                  dddd«      }t        j                  || j                  | j                  | j                  z   gd¬«      \  }}	}
| j                  r"| j                  |«      }| j                  |	«      }	|�P| j                  ||j                  ¬«      }| j                  |	|j                  ¬«      }	|j                  |	|
«      \  }	}
n"| j                  |«      }| j                  |	«      }	t        ||	|
|| j                   |¬«      }|j                  dddd«      j                  ||d«      }| j#                  |«      S )	Nr.   r   r	   é   ©Úaxis)Úoffset)rP   rA   rO   éÿÿÿÿ)ÚshaperD   Úreshaper?   r@   r   Ú	transposeÚmxÚsplitr   rG   rH   rJ   rU   Úupdate_and_fetchr   rA   rE   )rK   rN   rO   rP   ÚBÚLÚDÚqkvÚqueriesÚkeysÚvaluesÚoutputs               r(   Ú__call__zAttention.__call__P   sp  € ð —'‘'‰ˆˆ1ˆaà�m‰m˜AÓˆà�k‰kØˆq�$—,‘, $§/¡/°AÑ"5Ñ6¸¿¹ó
ç
‰)�A�q˜!˜QÓ
ð 	ô !#§¡Ø�$—,‘, §¡¨t¯©Ñ >Ð?Àaô!
Ñˆ��vð
 ×(Ò(Ø—k‘k 'Ó*ˆGØ—;‘;˜tÓ$ˆDàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆà�}‰}˜VÓ$Ð$r'   ©NN©r   r   r    r   r#   r>   rZ   Úarrayr   r   re   Ú__classcell__©rM   s   @r(   r3   r3   9   sa   ø„ ðW˜Yð W°#õ Wð2 $(Ø#ñ	%%à�8‰8ð%%ð �r—x‘xÑ ð%%ð ˜‰}ð	%%ð
 
�‰÷%%r'   r3   c                   óH   ‡ — e Zd Zdedefˆ fd„Zdej                  fd„Zˆ xZ	S )ÚMLPr4   r5   c                 ó:  •— t         ‰| �  «        || _        |j                  }|j                  |   }t        t        ||j                  z  |j                  ¬«      «      }t        j                  |d|z  d¬«      | _
        t        j                  ||d¬«      | _        y )N)r*   r.   Fr7   )r=   r>   r4   r   r   r#   r1   r   rB   rC   Úproj_1Úproj_2)rK   r4   r5   ÚdimÚffn_multiplierÚintermediate_dimrM   s         €r(   r>   zMLP.__init__y   s‡   ø€ Ü‰ÑÔØˆŒ	Ø�n‰nˆØ×-Ñ-¨hÑ7ˆäÜØ §¡Ñ/Ø×,Ñ,ôó
Ðô —i‘i  QÐ)9Ñ%9ÀÔFˆŒÜ—i‘iÐ 0°#¸EÔBˆ�r'   r,   c                 ó�   — | j                  |«      }t        j                  |dd¬«      \  }}| j                  t	        ||«      «      S )Nr.   rV   rS   )rn   rZ   r[   ro   r
   )rK   rN   Úgates      r(   re   zMLP.__call__‰   s:   € Ø�K‰K˜‹NˆÜ—(‘(˜1˜a bÔ)‰ˆˆaØ�{‰{œ6 $¨›?Ó+Ð+r'   )
r   r   r    r   r#   r>   rZ   rh   re   ri   rj   s   @r(   rl   rl   x   s)   ø„ ðC˜Yð C°#õ Cð ,˜RŸX™X÷ ,r'   rl   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚTransformerBlockr4   r5   c                 ó  •— t         ‰| �  «        |j                  }t        ||¬«      | _        t        ||¬«      | _        t        j                  ||j                  ¬«      | _
        t        j                  ||j                  ¬«      | _        y )N©r5   r9   )r=   r>   r   r3   Úattnrl   ÚffnrB   rF   r   Úffn_normÚ	attn_norm)rK   r4   r5   rp   rM   s       €r(   r>   zTransformerBlock.__init__�   sc   ø€ Ü‰ÑÔØ�n‰nˆÜ˜d¨XÔ6ˆŒ	Ü�t hÔ/ˆŒÜŸ
™
 3¨D×,=Ñ,=Ô>ˆŒÜŸ™ C¨T×->Ñ->Ô?ˆ�r'   rN   rO   rP   r,   c                 óž   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   }|S ©N)ry   r|   rz   r{   )rK   rN   rO   rP   ÚrÚhÚouts          r(   re   zTransformerBlock.__call__˜   sM   € ð �I‰I�d—n‘n QÓ'¨¨uÓ5ˆØ�‰EˆØ�H‰H�T—]‘] 1Ó%Ó&ˆØ�!‰eˆØˆ
r'   rf   rg   rj   s   @r(   rv   rv   �   sa   ø„ ð@˜Yð @°#õ @ð $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
r'   rv   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )ÚOpenELMModelr4   c                 óÄ  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        | j                  dkD  sJ ‚t        j                  |j                  |j                  «      | _        t        | j                  «      D �cg c]  }t        ||¬«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        y c c}w )Nr   rx   r9   )r=   r>   r4   r   r   rB   Ú	Embeddingr   Útoken_embeddingsÚrangerv   ÚlayersrF   r   Únorm)rK   r4   r5   rM   s      €r(   r>   zOpenELMModel.__init__¦   s±   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒØ&*×&AÑ&AˆÔ#Ø�‰ Ò"Ð"Ð"Ü "§¡¨T¯_©_¸d¿n¹nÓ MˆÔô " $×"=Ñ"=Ô>ó
á>�ô ˜T¨HÖ5Ø>ñ
ˆŒô —J‘J˜tŸ~™~°4×3DÑ3DÔEˆ�	ùò	
s   ÂCÚinputsc                 óì   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||¬«      }Œ | j                  |«      S )Nr   )rP   )r†   Úlenrˆ   r   Úzipr‰   )rK   rŠ   rP   r€   rO   ÚlayerÚcs          r(   re   zOpenELMModel.__call__³   su   € ð
 ×!Ñ! &Ó)ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä$ Q¨¨a©Ó1ˆÜ˜DŸK™K¨Ö/‰HˆE�1Ù�a˜ QÔ'‰Að 0ð �y‰y˜‹|Ðr'   r~   )	r   r   r    r   r>   rZ   rh   re   ri   rj   s   @r(   rƒ   rƒ   ¥   s'   ø„ ðF˜Yõ Fð  ñà—‘÷r'   rƒ   c                   óX   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zed„ «       Z	ˆ xZ
S )ÚModelr4   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y )NFr7   )r=   r>   r4   r   rƒ   Útransformerr   rB   rC   r   r   Úlm_head)rK   r4   rM   s     €r(   r>   zModel.__init__Å   sW   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ'¨Ó-ˆÔØ×-Ò-ÜŸ9™9 T§^¡^°T·_±_È5ÔQˆD�Lð .r'   rŠ   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S r~   )r“   r4   r   r†   Ú	as_linearr”   )rK   rŠ   rP   r�   s       r(   re   zModel.__call__Í   s[   € ð
 ×Ñ˜v uÓ-ˆØ�9‰9×.Ò.Ø×"Ñ"×3Ñ3×=Ñ=¸cÓBˆCð ˆ
ð —,‘,˜sÓ#ˆCàˆ
r'   c                 ó.   — | j                   j                  S r~   )r“   rˆ   )rK   s    r(   rˆ   zModel.layersÚ   s   € à×Ñ×&Ñ&Ð&r'   r~   )r   r   r    r   r>   rZ   rh   re   Úpropertyrˆ   ri   rj   s   @r(   r‘   r‘   Ä   s;   ø„ ðR˜Yõ Rð ñà—‘óð ñ'ó ô'r'   r‘   )é   N)Údataclassesr   Útypingr   r   r   r   r   Úmlx.coreÚcorerZ   Úmlx.nnrB   Úactivationsr
   r<   r   r   r   r   r%   r#   r1   ÚModuler3   rl   rv   rƒ   r‘   r&   r'   r(   Ú<module>r¡      sà   ðõ "ß 3Õ 3å Ý å ß TÑ Tð ô&�ó &ó ð&ð& Ø-1ñØˆU�CˆZÑðà�c‰]ðð ˜˜e S˜jÑ)Ñ*ðð ˆ5�#ˆ:Ñó	ô4<%�—	‘	ô <%ô~,ˆ"�)‰)ô ,ô.�r—y‘yô ô,�2—9‘9ô ô>'ˆB�I‰Iõ 'r'   