Ë
    îÿæi"  ã                   óf  — d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZ ddlmZmZmZmZ e G d„ d	e«      «       Z G d
„ de
j.                  «      Z G d„ de
j.                  «      Z G d„ de
j.                  «      Z G d„ de
j.                  «      Z G d„ de
j.                  «      Zy)é    )Ú	dataclass)ÚAnyÚListÚOptionalNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚArraysCacheÚ	CacheListÚKVCacheÚRotatingKVCachec                   óÆ   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   ee   ed	<   eed
<   eed<   dZeed<   dZ	e
e   ed<   dZe
e   ed<   dZeed<   y)Ú	ModelArgsÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚ
rope_thetaÚsliding_windowÚsliding_window_layersÚconv_windowÚrms_norm_epsÚbaichuan_m1Ú
model_typeNÚnum_swa_attention_headsÚnum_swa_key_value_headsFÚtie_word_embeddings)Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__Úfloatr   r   Ústrr   r   r    r!   Úbool© ó    ún/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/baichuan_m1.pyr   r      sz   … àƒOØÓØÓØÓØÓØÓØÓØÓØ ™9Ó$ØÓØÓØ#€J�Ó#Ø-1Ð˜X c™]Ó1Ø-1Ð˜X c™]Ó1Ø %Ð˜Ô%r+   r   c            	       ó�   ‡ — e Zd Zd
dedee   fˆ fd„Zd
d„Z	 ddej                  dej                  de
dej                  fd	„Zˆ xZS )Ú	AttentionÚconfigÚ	layer_idxc                 ó²  •— t         ‰| �  «        || _        || _        |€t	        d«      ‚||j
                  v | _        | j                  r|j                  r|j                  n|j                  | _	        | j                  r|j                  r|j                  n|j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k(  sJ ‚| j                  dz  | _        t!        j"                  |j                  | j                  d| j                  z  | j                  z  z   d¬«      | _        t!        j"                  | j                  | j                  z  |j                  d¬«      | _        t!        j(                  | j                  d|j*                  ¬«      | _        |j.                  | _        | j.                  dk(  sJ ‚t1        j2                  dd| j                  d| j.                  f«      | _        t1        j2                  dd| j                  d| j.                  f«      | _        y )Nz1Layer index must be provided to Attention module.g      à¿é   F©Úbias)ÚtraditionalÚbaser   )ÚsuperÚ__init__r/   r0   Ú
ValueErrorr   Úis_swar   r   Ú	num_headsr    r   Únum_kv_headsr   Úhead_dimÚscaleÚnnÚLinearÚW_packÚo_projÚRoPEr   Úroper   ÚmxÚzerosÚconv_kÚconv_v©Úselfr/   r0   Ú	__class__s      €r,   r8   zAttention.__init__"   sê  ø€ Ü‰ÑÔØˆŒØ"ˆŒØÐÜÐPÓQÐQà 6×#?Ñ#?Ð?ˆŒð �{Š{˜v×=Ò=ð ×*Ò*à×+Ñ+ð 	Œð �{Š{˜v×=Ò=ð ×*Ò*à×+Ñ+ð 	Ôð "×-Ñ-ˆÔØ×(Ñ(¨D¯N©NÑ:ˆŒØ�}‰}˜tŸ~™~Ñ-°×1AÑ1AÒAÐAÐAà—]‘] DÑ(ˆŒ
ä—i‘iØ×ÑØ×Ñ˜q 4×#4Ñ#4Ñ4°t·}±}ÑDÑDØô
ˆŒô
 —i‘iØ�N‰N˜TŸ]™]Ñ*¨F×,>Ñ,>ÀUô
ˆŒô —G‘G˜DŸM™M°uÀ6×CTÑCTÔUˆŒ	à!×-Ñ-ˆÔØ×Ñ 1Ò$Ð$Ð$Ü—h‘h  1 d×&7Ñ&7¸¸D×<LÑ<LÐMÓNˆŒÜ—h‘h  1 d×&7Ñ&7¸¸D×<LÑ<LÐMÓNˆ�r+   c                 óP  — |j                   \  }}}}|j                  d|| j                  ddf«      }|d d …d d …df   }|d d …d d …df   }	|€$t        j                  ||d|f|j
                  «      }|dkD  r&t        j                  ||d d …d d …d d…f   gd¬«      }
n|}
|
|z  ||	z  z   S )Nr   r   éÿÿÿÿr2   ©Úaxis)ÚshapeÚreshaper   rE   rF   ÚdtypeÚconcatenate)rJ   ÚuÚweightsÚstateÚBÚHÚLÚDÚw0Úw1Úu_prevs              r,   Ú_custom_convolutionzAttention._custom_convolutionK   sµ   € Ø—W‘W‰
ˆˆ1ˆa�Ø—/‘/ 1 a¨×)9Ñ)9¸1¸aÐ"@ÓAˆØ’Qš˜1�WÑˆØ’Qš˜1�WÑˆØˆ=Ü—H‘H˜a  A q˜\¨1¯7©7Ó3ˆEØˆqŠ5Ü—^‘^ U¨Aªa²°C°R°C¨i©LÐ$9ÀÔB‰FàˆFØ˜‰{˜Q ™VÑ#Ð#r+   ÚxÚmaskÚcacheÚreturnc                 ó„  — |j                   \  }}}| j                  |«      }t        j                  |||| j                  | j
                  z  z   fd¬«      \  }}	}
|j                  ||| j                  | j
                  «      j                  dddd«      }|	j                  ||| j                  | j
                  «      j                  dddd«      }	|
j                  ||| j                  | j
                  «      j                  dddd«      }
|€d}|d   � |d   j                  }|d   d   |d   d   }}nd}d\  }}|	}|
}| j                  |	| j                  |¬«      }	| j                  |
| j                  |¬«      }
| j                  ||¬	«      }| j                  |	|¬	«      }	|d   �K|d   j                  |	|
«      \  }	}
|dkD  r.|d d …d d …dd …d d …f   |d   d<   |d d …d d …dd …d d …f   |d   d<   t        ||	|
|d   | j                   |¬
«      }|j                  dddd«      j                  ||d«      }| j#                  |«      S )NrM   rN   r   r2   r   é   ©NN)rV   )Úoffset)ra   r>   r`   )rP   rA   rE   Úsplitr<   r=   rQ   r;   Ú	transposerf   r^   rG   rH   rD   Úupdate_and_fetchr   r>   rB   )rJ   r_   r`   ra   rW   rY   rZ   ÚprojÚqÚkÚvrf   Úlast_kÚlast_vÚk_initÚv_initÚouts                    r,   Ú__call__zAttention.__call__X   sA  € ð —'‘'‰ˆˆ1ˆaà�{‰{˜1‹~ˆÜ—(‘(˜4 ! Q¨×):Ñ):¸T¿]¹]Ñ)JÑ%JÐ!KÐRTÔU‰ˆˆ1ˆaà�I‰I�a˜˜DŸN™N¨D¯M©MÓ:×DÑDÀQÈÈ1ÈaÓPˆØ�I‰I�a˜˜D×-Ñ-¨t¯}©}Ó=×GÑGÈÈ1ÈaÐQRÓSˆØ�I‰I�a˜˜D×-Ñ-¨t¯}©}Ó=×GÑGÈÈ1ÈaÐQRÓSˆàˆ=Ø ˆEà�‰8ÐØ˜1‘X—_‘_ˆFØ" 1™X a™[¨%°©(°1©+�F‰FàˆFØ'‰NˆF�FàˆØˆØ×$Ñ$ Q¨¯©¸6Ð$ÓBˆØ×$Ñ$ Q¨¯©¸6Ð$ÓBˆØ�I‰I�a ˆIÓ'ˆØ�I‰I�a ˆIÓ'ˆà�‰8ÐØ˜‘8×,Ñ,¨Q°Ó2‰DˆAˆqØ�1ŠuØ$¢Qª¨2©3² \Ñ2��a‘˜‘Ø$¢Qª¨2©3² \Ñ2��a‘˜‘ä*Øˆq�!˜5 ™8¨4¯:©:¸Dô
ˆð �m‰m˜A˜q ! QÓ'×/Ñ/°°1°bÓ9ˆØ�{‰{˜3ÓÐr+   ©Nre   )r"   r#   r$   r   r   r%   r8   r^   rE   Úarrayr   rs   Ú__classcell__©rK   s   @r,   r.   r.   !   s[   ø„ ñ'O˜yð 'O°X¸c±]õ 'OóR$ð @Dñ' Ø—‘ð' Ø!#§¡ð' Ø9<ð' à	�‰÷' r+   r.   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚMLPr/   c                 óH  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        y ©NFr3   )	r7   r8   r?   r@   r   r   Ú	gate_projÚup_projÚ	down_proj©rJ   r/   rK   s     €r,   r8   zMLP.__init__ƒ   s{   ø€ Ü‰ÑÔÜŸ™Ø×Ñ × 8Ñ 8¸uô
ˆŒô —y‘yØ×Ñ × 8Ñ 8¸uô
ˆŒô Ÿ™Ø×$Ñ$ f×&8Ñ&8¸uô
ˆ�r+   r_   rb   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S rt   )r~   r   r|   r}   )rJ   r_   s     r,   rs   zMLP.__call__�   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr+   )	r"   r#   r$   r   r8   rE   ru   rs   rv   rw   s   @r,   ry   ry   ‚   s-   ø„ ð

˜yõ 

ðJ˜"Ÿ(™(ð J r§x¡x÷ Jr+   ry   c            	       ó€   ‡ — e Zd Zdedefˆ fd„Z	 d	dej                  dej                  dedej                  fd„Z	ˆ xZ
S )
ÚDecoderLayerr/   r0   c                 ó$  •— t         ‰| �  «        t        ||«      | _        t	        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  ¬«      | _        y )N©Úeps)r7   r8   r.   Ú	self_attnry   Úmlpr?   ÚRMSNormr   r   Úinput_layernormÚpost_attention_layernormrI   s      €r,   r8   zDecoderLayer.__init__”   sh   ø€ Ü‰ÑÔÜ" 6¨9Ó5ˆŒÜ�v“;ˆŒÜ!Ÿz™z¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
©
Ø×Ñ F×$7Ñ$7ô)
ˆÕ%r+   r_   r`   ra   rb   c                 óš   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   S rt   )r†   r‰   r‡   rŠ   )rJ   r_   r`   ra   Úrs        r,   rs   zDecoderLayer.__call__�   sL   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�1‰uˆr+   re   )r"   r#   r$   r   r%   r8   rE   ru   r   rs   rv   rw   s   @r,   r‚   r‚   “   sN   ø„ ð
˜yð 
°Sõ 
ð @DñØ—‘ðØ!#§¡ðØ9<ðà	�‰÷r+   r‚   c                   ób   ‡ — e Zd Zdefˆ fd„Zddej                  dedej                  fd„Zˆ xZ	S )ÚBaichuanModelr/   c                 óp  •— t         ‰| �  «        || _        t        j                  |j
                  |j                  «      | _        t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        |j                  | _        d | _        |j"                  r|j"                  d   | _        d | _        t'        |j"                  «      | _        t        |j                  «      D ]  }|| j(                  v rŒ|| _         y  y c c}w )Nr„   r   )r7   r8   Úargsr?   Ú	Embeddingr   r   Úembed_tokensÚranger   r‚   Úlayersrˆ   r   Únormr   Úfirst_swa_idxr   Úfirst_global_idxÚsetÚ
swa_layers)rJ   r/   ÚirK   s      €r,   r8   zBaichuanModel.__init__§   sý   ø€ Ü‰ÑÔØˆŒ	ÜŸL™L¨×):Ñ):¸F×<NÑ<NÓOˆÔÜ8=¸f×>VÑ>VÔ8WÓXÑ8W°1”| F¨AÕ.Ð8WÑXˆŒÜ—J‘J˜v×1Ñ1°v×7JÑ7JÔKˆŒ	à$×3Ñ3ˆÔØ!ˆÔØ×'Ò'Ø!'×!=Ñ!=¸aÑ!@ˆDÔà $ˆÔÜ˜f×:Ñ:Ó;ˆŒÜ�v×/Ñ/Ö0ˆAØ�D—O‘OÑ#ØØ$%ˆDÔ!Ùñ	 1ùò Ys   ÁD3Úinputsra   rb   c                 óÔ  — | j                  |«      }|€dgt        | j                  «      z  }| j                  €d }n|| j                     d   }| j                  €d }n|| j                     d   }t        ||«      }t        ||| j                  ¬«      }t        t        | j                  |«      «      D ]$  \  }\  }	}
|| j                  v r|n|} |	|||
«      }Œ& | j                  |«      S )Nre   r   )Úwindow_size)r’   Úlenr”   r—   r–   r
   r   Ú	enumerateÚzipr™   r•   )rJ   r›   ra   r_   Úc_globalÚc_swaÚglobal_maskÚswa_maskÚlÚlayerÚcr`   s               r,   rs   zBaichuanModel.__call__»   së   € Ø×Ñ˜fÓ%ˆàˆ=Ø!�N¤S¨¯©Ó%5Ñ5ˆEà× Ñ Ð(Ø‰Hà˜T×2Ñ2Ñ3°AÑ6ˆHà×ÑÐ%Ø‰Eà˜$×,Ñ,Ñ-¨aÑ0ˆEä+¨A¨xÓ8ˆÜ(¨¨E¸t×?RÑ?RÔSˆä&¤s¨4¯;©;¸Ó'>Ö?‰MˆA‰z��qØ  D§O¡OÑ3‘8¸ˆDÙ�a˜˜qÓ!‰Að @ð �y‰y˜‹|Ðr+   rt   )
r"   r#   r$   r   r8   rE   ru   r   rs   rv   rw   s   @r,   rŽ   rŽ   ¦   s1   ø„ ð˜yõ ñ(˜rŸx™xð °ð ¸r¿x¹x÷ r+   rŽ   c                   ó´   ‡ — e Zd Zdefˆ fd„Zdee   fd„Zdedefd„Z	dde
j                  dede
j                  fd	„Zedeej                     fd
„«       Zˆ xZS )ÚModelr/   c                 ó  •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y r{   )r7   r8   r/   r   rŽ   Úmodelr!   r?   r@   r   r   Úlm_headr   s     €r,   r8   zModel.__init__Õ   sk   ø€ Ü‰ÑÔØˆŒØ ×+Ñ+ˆŒÜ" 6Ó*ˆŒ
Ø#)×#=Ñ#=ˆÔ Ø×)Ò)ÜŸ9™9 V×%7Ñ%7¸×9JÑ9JÐQVÔWˆD�Lð *r+   rb   c                 ó0  — g }t        | j                  j                  «      D ]q  \  }}|| j                  j                  v }t        d¬«      }|r!t        | j                  j                  ¬«      }n
t        «       }|j                  t        ||«      «       Œs |S )Nr2   )Úsize)Úmax_size)rŸ   r«   r”   r/   r   r   r   r   r   Úappendr   )rJ   Úcachesrš   r¦   r:   Ú
conv_cacheÚkv_caches          r,   Ú
make_cachezModel.make_cacheÞ   sz   € ØˆÜ! $§*¡*×"3Ñ"3Ö4‰HˆAˆuØ˜$Ÿ+™+×;Ñ;Ð;ˆFÜ$¨!Ô,ˆJÙÜ*°D·K±K×4NÑ4NÔO‘ä"›9�Ø�M‰Mœ) J°Ó9Õ:ð 5ð ˆr+   rU   c                 óö   — d|v }|srd|v rn|d   }|j                   }|j                  t        j                  «      }t        j                  j                  |dd¬«      }||dz   z  j                  |«      }||d<   |S )Nzlm_head.scaleszlm_head.weightrM   T)rO   ÚkeepdimsgH¯¼šò×z>)rR   ÚastyperE   Úfloat32Úlinalgr•   )rJ   rU   Úis_quantizedÚwrR   r•   s         r,   ÚsanitizezModel.sanitizeê   s~   € Ø'¨7Ð2ˆÙÐ 0°GÑ ;ØÐ(Ñ)ˆAØ—G‘GˆEØ—‘œŸ™Ó$ˆAÜ—9‘9—>‘> !¨"°t�>Ó<ˆDØ�d˜T‘kÑ"×*Ñ*¨5Ó1ˆAØ()ˆGÐ$Ñ%Øˆr+   r›   ra   c                 óH   — | j                  ||«      }| j                  |«      S rt   )r«   r¬   )rJ   r›   ra   Úoutputss       r,   rs   zModel.__call__õ   s!   € Ø—*‘*˜V UÓ+ˆØ�|‰|˜GÓ$Ð$r+   c                 ó.   — | j                   j                  S rt   )r«   r”   )rJ   s    r,   r”   zModel.layersù   s   € à�z‰z× Ñ Ð r+   rt   )r"   r#   r$   r   r8   r   r   r´   Údictr¼   rE   ru   rs   Úpropertyr?   ÚModuler”   rv   rw   s   @r,   r©   r©   Ô   sy   ø„ ðX˜yõ Xð
˜D ™Ió 
ð	 ð 	¨ó 	ñ%˜rŸx™xð %°ð %¸r¿x¹xó %ð ð!˜˜RŸY™Y™ò !ó ô!r+   r©   )Údataclassesr   Útypingr   r   r   Úmlx.coreÚcorerE   Úmlx.nnr?   Úactivationsr   r6   r	   r
   r   ra   r   r   r   r   r   rÂ   r.   ry   r‚   rŽ   r©   r*   r+   r,   Ú<module>rÉ      s–   ðõ "ß &Ñ &å Ý å ß TÑ Tß CÓ Cð ô&�ó &ó ð&ô$^ �—	‘	ô ^ ôBJˆ"�)‰)ô Jô"�2—9‘9ô ô&+�B—I‘Iô +ô\'!ˆB�I‰Iõ '!r+   