Ë
    îÿæi[  ã                   óZ  — d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ ddlmZmZ e G d„ d	e«      «       Z G d
„ de	j(                  «      Z G d„ de	j(                  «      Z G d„ de	j(                  «      Z G d„ de	j(                  «      Z G d„ de	j(                  «      Zy)é    )Ú	dataclass)ÚOptionalÚTupleNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCachec                   óÜ   — e Zd ZU eed<   dZeed<   dZeed<   dZeed<   dZ	eed	<   dZ
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   y)Ú	ModelArgsÚ
model_typei   Úhidden_sizeé€   Úhead_dimé    Únum_hidden_layersi 8  Úintermediate_sizeÚnum_attention_headsé   Únum_key_value_headsg     jè@Ú
rope_thetai è Ú
vocab_sizegñhãˆµøä>Úlayer_norm_epsg      °?Úlogit_scaleFÚattention_biasÚlayer_norm_biasÚsliding_windowé   Úsliding_window_patternN)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__r   Úintr   r   r   r   r   r   Úfloatr   r   r   r   Úboolr   r   r!   © ó    új/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/cohere2.pyr   r      s˜   … àƒOØ€K�ÓØ€HˆcÓØÐ�sÓØ"Ð�sÓ"Ø!Ð˜Ó!Ø Ð˜Ó Ø€J�ÓØ€J�ÓØ!€N�EÓ!Ø€K�ÓØ €N�DÓ Ø!€O�TÓ!Ø€N�CÓØ"#Ð˜CÔ#r+   r   c            
       óÀ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	ej                  ej                  f      dej                  fd„Z
ˆ xZS )
Ú	AttentionÚargsÚ	layer_idxc                 óª  •— t         ‰| �  «        || _        || _        |j                  }|j
                  x| _        }|j                  x| _        }|j                  x| _	        }||z  |k7  rt        d|› d|› d�«      ‚|dz  | _        |j                  }t        j                  |||z  |¬«      | _        t        j                  |||z  |¬«      | _        t        j                  |||z  |¬«      | _        t        j                  ||z  ||¬«      | _        t        j&                  |d|j(                  ¬«      | _        |dz   |j,                  z  d	k7  | _        y )
Nz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).g      à¿©ÚbiasT)ÚtraditionalÚbaser   r   )ÚsuperÚ__init__r/   r0   r   r   Ún_headsr   Ú
n_kv_headsr   Ú
ValueErrorÚscaler   ÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚRoPEr   Úroper!   Úuse_sliding_window)	Úselfr/   r0   Údimr8   r9   r   Úattetion_biasÚ	__class__s	           €r,   r7   zAttention.__init__"   s=  ø€ Ü‰ÑÔØˆŒ	Ø"ˆŒà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*Ø#'§=¡=Ð0ˆŒ˜Ø�wÑ 3Ò&ÜØQÐRUÐQVØ$ W I¨Rð1óð ð ˜t‘^ˆŒ
à×+Ñ+ˆä—i‘i  W¨xÑ%7¸mÔLˆŒÜ—i‘i  Z°(Ñ%:ÀÔOˆŒÜ—i‘i  Z°(Ñ%:ÀÔOˆŒÜ—i‘i ¨(Ñ 2°C¸mÔLˆŒä—G‘G˜H°$¸T¿_¹_ÔMˆŒ	à#,¨q¡=°D×4OÑ4OÑ"OÐSTÑ"TˆÕr+   ÚxÚmaskÚcacheÚreturnc                 ó  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      j                  dddd«      }|j	                  ||| j                  d«      j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	| j                  r_|€#| j                  |«      }| j                  |«      }n:| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|�|j                  ||	«      \  }}	|j                  t        j                  k(  rt        j                  n|j                  }
t!        |j#                  |
«      ||	|| j$                  |¬«      j#                  |j                  «      }|j                  dddd«      j	                  ||d«      }| j'                  |«      S )Néÿÿÿÿr   é   r   é   )Úoffset)rK   r;   rJ   )Úshaper>   r?   r@   Úreshaper8   Ú	transposer9   rD   rC   rQ   Úupdate_and_fetchÚdtypeÚmxÚfloat16Úfloat32r
   Úastyper;   rA   )rE   rI   rJ   rK   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚ	sdpa_typeÚoutputs               r,   Ú__call__zAttention.__call__=   s¸  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆà—/‘/ ! Q¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A˜q $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆð ×"Ò"Øˆ}ØŸ)™) GÓ,�Ø—y‘y “‘àŸ)™) G°E·L±L˜)ÓA�Ø—y‘y ¨e¯l©l�yÓ;�àÐØ ×1Ñ1°$¸Ó?‰LˆD�&ð #*§-¡-´2·:±:Ò"=”B—J’JÀ7Ç=Á=ˆ	Ü-Ø�N‰N˜9Ó%ØØØØ—*‘*Øô
÷ ‰&�—‘Ó
ð 	ð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r+   ©NN©r"   r#   r$   r   r'   r7   rW   Úarrayr   r   rc   Ú__classcell__©rH   s   @r,   r.   r.   !   st   ø„ ðU˜Yð U°3õ Uð< $(Ø59ñ	&#à�8‰8ð&#ð �r—x‘xÑ ð&#ð ˜˜bŸh™h¨¯©Ð0Ñ1Ñ2ð	&#ð
 
�‰÷&#r+   r.   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚMLPc                 óÐ   •— t         ‰| �  «        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        y )NFr2   )r6   r7   r<   r=   Ú	gate_projÚup_projÚ	down_proj)rE   rF   Ú
hidden_dimrH   s      €r,   r7   zMLP.__init__g   sJ   ø€ Ü‰ÑÔÜŸ™ 3¨
¸Ô?ˆŒÜ—y‘y  j°uÔ=ˆŒÜŸ™ :¨s¸Ô?ˆ�r+   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)rn   r   rl   rm   )rE   rI   s     r,   rc   zMLP.__call__m   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr+   )r"   r#   r$   r7   rc   rg   rh   s   @r,   rj   rj   f   s   ø„ ô@öJr+   rj   c            
       óÀ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	ej                  ej                  f      dej                  fd„Z
ˆ xZS )
ÚTransformerBlockr/   r0   c                 óV  •— t         ‰| �  «        |j                  | _        |j                  | _        t        ||«      | _        t        |j                  |j                  «      | _	        t        j                  |j                  |j                  |j                  ¬«      | _        || _        y )N©Úepsr3   )r6   r7   r   r   r8   r.   Ú	self_attnrj   r   Úmlpr<   Ú	LayerNormr   r   Úinput_layernormr/   )rE   r/   r0   rH   s      €r,   r7   zTransformerBlock.__init__r   s„   ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ×/Ñ/ˆŒä" 4¨Ó3ˆŒÜ�t×'Ñ'¨×)?Ñ)?Ó@ˆŒÜ!Ÿ|™|Ø×Ñ $×"5Ñ"5¸D×<PÑ<Pô 
ˆÔð ˆ�	r+   rI   rJ   rK   rL   c                 ó|   — | j                  |«      }| j                  |||«      }| j                  |«      }||z   |z   S rq   )rz   rw   rx   )rE   rI   rJ   rK   ÚhÚattn_hÚff_hs          r,   rc   zTransformerBlock.__call__~   sB   € ð × Ñ  Ó#ˆØ—‘  4¨Ó/ˆØ�x‰x˜‹{ˆà˜‰}˜qÑ Ð r+   rd   re   rh   s   @r,   rs   rs   q   sq   ø„ ð
˜Yð 
°3õ 
ð $(Ø59ñ	!à�8‰8ð!ð �r—x‘xÑ ð!ð ˜˜bŸh™h¨¯©Ð0Ñ1Ñ2ð	!ð
 
�‰÷!r+   rs   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )ÚCohereModelr/   c                 óü  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        | j                  dkD  sJ ‚|j
                  | _        t        j                  |j                  |j                  «      | _
        t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}| _        t        j                  |j                  |j                  |j                   ¬«      | _        y c c}w )Nr   )r/   r0   ru   )r6   r7   r/   r   r   r   Úwindow_sizer<   Ú	Embeddingr   Úembed_tokensÚrangers   Úlayersry   r   r   Únorm)rE   r/   ÚirH   s      €r,   r7   zCohereModel.__init__�   sÎ   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ�‰ Ò"Ð"Ð"Ø×.Ñ.ˆÔÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔô ˜4×1Ñ1Ô2ó
á2�ô  $°!Ö4Ø2ñ
ˆŒô —L‘LØ×Ñ $×"5Ñ"5¸D×<PÑ<Pô
ˆ�	ùò	
s   Â!C9Úinputsc                 óÜ  — | j                  |«      }|€d gt        | j                  «      z  }| j                  j                  }t        |||dz
     «      }t        ||d   | j                  ¬«      }t        t        | j                  |«      «      D ]K  \  }\  }}	|| j                  j                  z  | j                  j                  dz
  k(  }
|
r|n|} ||||	«      }ŒM | j                  |«      S )Nr   r   )r‚   )
r„   Úlenr†   r/   r!   r	   r‚   Ú	enumerateÚzipr‡   )rE   r‰   rK   r|   ÚjÚ	full_maskÚswa_maskrˆ   ÚlayerÚcÚ	is_globalrJ   s               r,   rc   zCohereModel.__call__�   sã   € ð
 ×Ñ˜fÓ%ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEà�I‰I×,Ñ,ˆÜ)¨!¨U°1°q±5©\Ó:ˆ	Ü(¨¨E°!©HÀ$×BRÑBRÔSˆä&¤s¨4¯;©;¸Ó'>Ö?‰MˆA‰z��qà�D—I‘I×4Ñ4Ñ4Ø—9‘9×3Ñ3°aÑ7ñ8ð ñ
 !*‘9¨xˆDá�a˜˜qÓ!‰Að @ð �y‰y˜‹|Ðr+   rq   )	r"   r#   r$   r   r7   rW   rf   rc   rg   rh   s   @r,   r€   r€   Œ   s%   ø„ ð
˜Yõ 
ð& ñà—‘÷r+   r€   c                   ó^   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zd„ Ze	d„ «       Z
ˆ xZS )ÚModelr/   c                 ór   •— t         ‰| �  «        |j                  | _        t        |«      | _        || _        y rq   )r6   r7   r   r€   Úmodelr/   )rE   r/   rH   s     €r,   r7   zModel.__init__¹   s,   ø€ Ü‰ÑÔØŸ/™/ˆŒÜ  Ó&ˆŒ
Øˆ�	r+   r‰   c                 óº   — | j                  ||«      }| j                   j                  j                  |«      }|| j                   j                  j                  z  }|S rq   )r—   r„   Ú	as_linearr/   r   )rE   r‰   rK   Úouts       r,   rc   zModel.__call__¿   sK   € ð
 �j‰j˜ Ó'ˆØ�j‰j×%Ñ%×/Ñ/°Ó4ˆØ�D—J‘J—O‘O×/Ñ/Ñ/ˆØˆ
r+   c                 óL  — g }t        | j                  j                  «      D ]  }|| j                  j                  z  | j                  j                  dz
  k(  r|j	                  t        «       «       ŒP|j	                  t        | j                  j                  d¬«      «       Œ� |S )Nr   r   )Úmax_sizeÚkeep)r…   r/   r   r!   Úappendr   r   r   )rE   Úcachesrˆ   s      r,   Ú
make_cachezModel.make_cacheÉ   s€   € ØˆÜ�t—y‘y×2Ñ2Ö3ˆAà�D—I‘I×4Ñ4Ñ4Ø—9‘9×3Ñ3°aÑ7ò8ð —‘œg›iÕ(à—‘Ü#¨T¯Y©Y×-EÑ-EÈAÔNõð 4ð ˆr+   c                 ó.   — | j                   j                  S rq   )r—   r†   )rE   s    r,   r†   zModel.layers×   s   € à�z‰z× Ñ Ð r+   rq   )r"   r#   r$   r   r7   rW   rf   rc   r    Úpropertyr†   rg   rh   s   @r,   r•   r•   ¸   s>   ø„ ð˜Yõ ð ñà—‘óòð ñ!ó ô!r+   r•   )Údataclassesr   Útypingr   r   Úmlx.coreÚcorerW   Úmlx.nnr<   Úactivationsr   r5   r   r	   r
   rK   r   r   r   ÚModuler.   rj   rs   r€   r•   r*   r+   r,   Ú<module>rª      s�   ðõ "ß "å Ý å ß TÑ Tß +ð ô$�ó $ó ð$ô$B#�—	‘	ô B#ôJJˆ"�)‰)ô Jô!�r—y‘yô !ô6)�"—)‘)ô )ôX!!ˆB�I‰Iõ !!r+   