Ë
    îÿæi
3  ã                   ó  — d dl Z d dlmZ d dlmZmZ d dlmZ d dl	m
Z
 ddlmZmZmZ ddlmZmZ e G d„ de«      «       Z G d	„ d
e
j&                  «      Zd„ Z G d„ de
j&                  «      Z G d„ de
j&                  «      Z G d„ de
j&                  «      Z G d„ de
j&                  «      Z G d„ de
j&                  «      Z G d„ de
j&                  «      Z G d„ de
j&                  «      Z G d„ de
j&                  «      Zy)é    N)Ú	dataclass)ÚListÚOptionalé   )ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚArraysCacheÚRotatingKVCachec                   óØ   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   dZeed<   dZ	e
ee      ed<   dZe
ee      ed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚattention_biasÚconv1d_widthÚhidden_sizeÚintermediate_sizeÚlogits_soft_capÚnum_attention_headsÚnum_hidden_layersÚnum_key_value_headsÚrms_norm_epsÚ
rope_thetaÚattention_window_sizeÚ
vocab_sizeTÚembeddings_scale_by_sqrt_dimNÚblock_typesÚ_block_typesc                 ó@   — | j                   €| j                  | _         y y ©N)r   r   ©Úselfs    úr/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/recurrent_gemma.pyÚ__post_init__zModelArgs.__post_init__!   s!   € à×ÑÐ#Ø#×0Ñ0ˆDÕð $ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚboolÚintÚfloatr   r   r   r   r   r#   © r$   r"   r   r      s„   … àƒOØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØƒOØ)-Ð  $Ó-Ø'+€K�˜$˜s™)Ñ$Ó+Ø(,€L�(˜4 ™9Ñ%Ó,ó1r$   r   c                   ó0   ‡ — e Zd Zddedefˆ fd„Zd„ Zˆ xZS )ÚRMSNormÚdimsÚepsc                 óf   •— t         ‰| �  «        t        j                  |f«      | _        || _        y r   )ÚsuperÚ__init__ÚmxÚonesÚweightr1   )r!   r0   r1   Ú	__class__s      €r"   r4   zRMSNorm.__init__(   s'   ø€ Ü‰ÑÔÜ—g‘g˜t˜gÓ&ˆŒØˆ�r$   c                 ór   — t         j                  j                  |d| j                  z   | j                  «      S )Nç      ð?)r5   ÚfastÚrms_normr7   r1   )r!   Úxs     r"   Ú__call__zRMSNorm.__call__-   s(   € Ü�w‰w×Ñ  3¨¯©Ñ#4°d·h±hÓ?Ð?r$   )gñhãˆµøä>)r%   r&   r'   r+   r,   r4   r>   Ú__classcell__©r8   s   @r"   r/   r/   '   s   ø„ ñ˜Sð  uõ ö
@r$   r/   c                 ó8  — | j                   dk(  sJ ‚|j                  | j                  |j                    d  k(  sJ ‚|j                  | j                  k(  sJ ‚| j                  d   dk(  r'|€| | d d …df   fS ||d d …d f   z  | z   }||d d …df   fS |�|}n3| j                  \  }}}t        j                  ||f| j                  ¬«      }t        j
                  | «      }t        | j                  d   «      D ]!  }|d d …|f   |z  | d d …|f   z   }||d d …|f<   Œ# ||fS )Né   r   r   éÿÿÿÿ)Údtype)ÚndimÚshaperD   r5   ÚzerosÚ
zeros_likeÚrange)	r=   ÚaÚh0ÚyÚh_tÚBÚ_ÚDÚts	            r"   Úrnn_scanrR   1   s)  € Ø�6‰6�QŠ;Ðˆ;Ø�7‰7�a—g‘g˜qŸv™v˜g˜iÐ(Ò(Ð(Ð(Ø�7‰7�a—g‘gÒÐÐà‡w�wˆq�z�Q‚àˆ:Ø�aš˜1˜‘g�:Ðð �B’q˜$�w‘K‘ !Ñ#ˆAØ�aš˜2˜‘h�;Ðð ˆ>Ø‰Cà—g‘g‰GˆAˆq�!Ü—(‘(˜A˜q˜6¨¯©Ô1ˆCä�M‰M˜!ÓˆÜ�q—w‘w˜q‘zÖ"ˆAØ’A�q�D‘'˜C‘- !¢A q D¡'Ñ)ˆCØˆAŠa�ˆdŠGð #ð ˆcˆ6€Mr$   c                   ó0   ‡ — e Zd Zdedefˆ fd„Zdd„Zˆ xZS )ÚConv1dÚchannelsÚkernel_sizec                 ó’   •— t         ‰| �  «        t        j                  ||df«      | _        t        j                  |f«      | _        y )Nr   )r3   r4   r5   rG   r7   Úbias)r!   rU   rV   r8   s      €r"   r4   zConv1d.__init__P   s9   ø€ ô
 	‰ÑÔÜ—h‘h ¨+°qÐ9Ó:ˆŒÜ—H‘H˜h˜[Ó)ˆ�	r$   c                 óT  — |j                   \  }}}| j                  j                   \  }}}|�t        j                  ||gd¬«      }nt        j                  |d|dz
  dfdg«      }t        j
                  || j                  |¬«      }	|	| j                  z   }	|	|d d …| dz   d …d d …f   fS )Nr   )Úaxis)r   r   r   )Úgroups)rF   r7   r5   ÚconcatenateÚpadÚconv_generalrX   )
r!   r=   ÚcacherN   ÚLÚCr[   ÚKrO   rL   s
             r"   r>   zConv1d.__call__Y   s    € Ø—'‘'‰ˆˆ1ˆaØ—{‘{×(Ñ(‰ˆ��1àÐÜ—‘  q˜z°Ô2‰Aä—‘�q˜6 A¨¡E¨1 :¨vÐ6Ó7ˆAä�O‰O˜A˜tŸ{™{°6Ô:ˆØ�—	‘	‰Mˆà�!’A˜�r˜A‘v‘x¢�NÑ#Ð#Ð#r$   r   )r%   r&   r'   r+   r4   r>   r?   r@   s   @r"   rT   rT   O   s   ø„ ð*àð*ð õ*÷$r$   rT   c                   óP   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej                  fd„Zˆ xZ	S )ÚRGLRUz2A Real-Gated Linear Recurrent Unit (RG-LRU) layer.ÚwidthÚ	num_headsc                 óp  •— t         ‰| �  «        || _        || _        | j                  | j                  z  | _        t        j                  | j                  f«      | _        t        j                  | j                  | j                  | j                  f«      | _        t        j                  | j                  | j                  f«      | _	        t        j                  | j                  | j                  | j                  f«      | _
        t        j                  | j                  | j                  f«      | _        y r   )r3   r4   re   rf   Úhead_dimr5   rG   Úrecurrent_paramÚinput_gate_weightÚinput_gate_biasÚrecurrent_gate_weightÚrecurrent_gate_bias)r!   re   rf   r8   s      €r"   r4   zRGLRU.__init__k   s×   ø€ ô
 	‰ÑÔØˆŒ
Ø"ˆŒØŸ
™
 d§n¡nÑ4ˆŒä!Ÿx™x¨¯©¨Ó6ˆÔä!#§¡Ø�^‰^˜TŸ]™]¨D¯M©MÐ:ó"
ˆÔô  "Ÿx™x¨¯©¸¿¹Ð(GÓHˆÔä%'§X¡XØ�^‰^˜TŸ]™]¨D¯M©MÐ:ó&
ˆÔ"ô $&§8¡8¨T¯^©^¸T¿]¹]Ð,KÓ#LˆÕ r$   r=   c                 ó  ‡ ‡‡— |j                   \  ŠŠ}ˆˆˆ fd„} ||‰ j                  ‰ j                  «      } ||‰ j                  ‰ j                  «      }d|z  t        j                  ‰ j                  «      z  }t        j                  |«      }t        j                  d|z  «      }	||z  }
t        j                  d|	z
  «      }|€d|d d …dd d …f<   |
|j                  |j                  «      z  }t        |||¬«      \  }}||fS )Nc                 óð   •— | j                  ‰‰‰j                  ‰j                  f«      } | j                  dd«      |z  j                  dd«      |z   } t	        j
                  | j                  dd«      «      S )Nr   é   rB   )Úreshaperf   rh   Úswapaxesr5   ÚsigmoidÚflatten)ÚhÚwÚbrN   r`   r!   s      €€€r"   Úapply_block_linearz*RGLRU.__call__.<locals>.apply_block_linearˆ   sb   ø€ Ø—	‘	˜1˜a §¡°·±Ð?Ó@ˆAØ—‘˜A˜qÓ! AÑ%×/Ñ/°°1Ó5¸Ñ9ˆAÜ—:‘:˜aŸi™i¨¨1›oÓ.Ð.r$   g       Àrp   r   r:   r   )r=   rJ   rK   )rF   rj   rk   rl   rm   ÚnnÚsoftplusri   r5   ÚexpÚsqrtÚastyperD   rR   )r!   r=   r_   rO   rx   Úgate_xÚgate_aÚlog_arJ   Úa_squareÚgated_xÚ
multiplierÚnormalized_xrL   Úlast_hrN   r`   s   `              @@r"   r>   zRGLRU.__call__�   s   ú€ ð
 —'‘'‰ˆˆ1ˆaö	/ñ $ A t×'=Ñ'=¸t×?SÑ?SÓTˆÙ#Øˆt×)Ñ)¨4×+CÑ+Có
ˆð
 �v‘¤§¡¨D×,@Ñ,@Ó AÑAˆÜ�F‰F�5‹MˆÜ—6‘6˜!˜e™)Ó$ˆð �f‘*ˆô —W‘W˜Q ™\Ó*ˆ
Øˆ=Ø"%ˆJ’q˜!šQ�wÑØ ×!2Ñ!2°1·7±7Ó!;Ñ;ˆäØØØô
‰	ˆˆ6ð �&ˆyÐr$   r   )
r%   r&   r'   Ú__doc__r+   r4   r5   Úarrayr>   r?   r@   s   @r"   rd   rd   h   s6   ø„ Ù<ðMàðMð õMð2 ñ&à�8‰8÷&r$   rd   c            	       ó\   ‡ — e Zd Z	 	 ddedededefˆ fd„Z	 	 d	dej                  fd„Zˆ xZS )
ÚRecurrentBlockre   rf   Ú	lru_widthÚconv1d_temporal_widthc                 óØ  •— t         ‰| �  «        || _        || _        |xs || _        || _        t        j                  || j                  «      | _        t        j                  || j                  «      | _	        t        j                  | j                  |«      | _
        t        | j                  | j
                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )N)rU   rV   )re   rf   )r3   r4   re   rf   rŠ   r‹   ry   ÚLinearÚlinear_yÚlinear_xÚ
linear_outrT   Úconv_1drd   Úrg_lru)r!   re   rf   rŠ   r‹   r8   s        €r"   r4   zRecurrentBlock.__init__¬   s¬   ø€ ô 	‰ÑÔØˆŒ
Ø"ˆŒØ"Ò+ eˆŒØ%:ˆÔ"äŸ	™	 %¨¯©Ó8ˆŒÜŸ	™	 %¨¯©Ó8ˆŒÜŸ)™) D§N¡N°EÓ:ˆŒÜØ—^‘^Ø×2Ñ2ô
ˆŒô Ø—.‘.Ø—n‘nô
ˆ�r$   r=   c                 ó  — | j                  |«      }t        j                  |«      }| j                  |«      }|€d d g}| j	                  ||d   ¬«      \  }|d<   | j                  ||d   ¬«      \  }|d<   ||z  }| j                  |«      }|S )Nr   )r=   r_   r   )rŽ   ry   Úgelu_approxr�   r‘   r’   r�   )r!   r=   r_   ÚmaskrL   s        r"   r>   zRecurrentBlock.__call__Å   s–   € ð �M‰M˜!ÓˆÜ�N‰N˜1Óˆð �M‰M˜!ÓˆØˆ=Ø˜4�LˆEØ—l‘l Q¨e°A©h�lÓ7‰ˆˆ5�‰8Ø—k‘k A¨U°1©X�kÓ6‰ˆˆ5�‰8à�‰EˆØ�O‰O˜AÓˆàˆr$   ©Né   ©NN©	r%   r&   r'   r+   r4   r5   r‡   r>   r?   r@   s   @r"   r‰   r‰   ª   sO   ø„ ð Ø%&ñ
àð
ð ð
ð ð	
ð
  #õ
ð8 Øñ	à�8‰8÷r$   r‰   c                   óR   ‡ — e Zd Zdededefˆ fd„Z	 	 ddej                  fd„Zˆ xZS )ÚLocalAttentionBlockre   rf   Úwindow_sizec                 ó|  •— t         ‰| �  «        || _        || _        || _        ||z  dz  | _        | j                  | j                  z  | _        t        j                  | j                  | j                  d¬«      | _	        t        j                  | j                  | j                  d¬«      | _
        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  dz  d¬«      | _        y )Ng      à¿F©rX   Trp   )Útraditional)r3   r4   re   rf   rœ   Úscalerh   ry   r�   Úq_projÚk_projÚv_projÚo_projÚRoPEÚrope)r!   re   rf   rœ   r8   s       €r"   r4   zLocalAttentionBlock.__init__Þ   sÚ   ø€ ô 	‰ÑÔØˆŒ
Ø"ˆŒØ&ˆÔØ˜yÑ(¨dÑ3ˆŒ
àŸ
™
 d§n¡nÑ4ˆŒÜ—i‘i §
¡
¨D¯J©J¸UÔCˆŒÜ—i‘i §
¡
¨D¯M©MÀÔFˆŒÜ—i‘i §
¡
¨D¯M©MÀÔFˆŒÜ—i‘i §
¡
¨D¯J©J¸TÔBˆŒÜ—G‘GØ�M‰M˜QÑØô
ˆ�	r$   r=   c                 ó
  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      j                  dddd«      }|j	                  ||dd«      j                  dddd«      }|	j	                  ||dd«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j	                  ||d«      }
| j                  |
«      S )NrC   r   rp   r   rB   )Úoffset)r_   r    r•   )rF   r¡   r¢   r£   rq   rf   Ú	transposer¦   r¨   Úupdate_and_fetchr	   r    r¤   )r!   r=   r_   r•   rN   r`   rP   ÚqueriesÚkeysÚvaluesÚoutputs              r"   r>   zLocalAttentionBlock.__call__ô   s_  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆà—/‘/ ! Q¨¯©¸Ó;×EÑEÀaÈÈAÈqÓQˆØ�|‰|˜A˜q ! RÓ(×2Ñ2°1°a¸¸AÓ>ˆØ—‘  1 a¨Ó,×6Ñ6°q¸!¸QÀÓBˆàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r$   r˜   r™   r@   s   @r"   r›   r›   Ü   s=   ø„ ð
àð
ð ð
ð õ	
ð2 Øñ	#à�8‰8÷#r$   r›   c                   óH   ‡ — e Zd Zdedefˆ fd„Zdej                  fd„Zˆ xZS )ÚMLPBlockre   Úexpanded_widthc                 óÖ   •— t         ‰| �  «        t        j                  ||dz  «      | _        t        j                  ||dz  «      | _        t        j                  |dz  |«      | _        y )Nrp   )r3   r4   ry   r�   Úup_projÚ	gate_projÚ	down_proj)r!   re   r±   r8   s      €r"   r4   zMLPBlock.__init__  sS   ø€ Ü‰ÑÔÜ—y‘y ¨¸!Ñ(;Ó<ˆŒÜŸ™ 5¨.¸AÑ*=Ó>ˆŒÜŸ™ >°QÑ#6¸Ó>ˆ�r$   r=   c                 ó”   — | j                  |«      }| j                  |«      }| j                  t        j                  |«      |z  «      S r   )r´   r³   rµ   ry   r”   )r!   r=   Úgates      r"   r>   zMLPBlock.__call__  s:   € Ø�~‰~˜aÓ ˆØ�L‰L˜‹OˆØ�~‰~œbŸn™n¨TÓ2°QÑ6Ó7Ð7r$   r™   r@   s   @r"   r°   r°     s&   ø„ ð?˜cð ?°3õ ?ð8˜"Ÿ(™(÷ 8r$   r°   c                   ón   ‡ — e Zd Z	 	 ddedededededee   defˆ fd„Z	 	 dd	ej                  fd
„Z	ˆ xZ
S )ÚResidualBlockre   Úmlp_expanded_widthrf   r   Útemporal_block_typerŠ   r‹   c                 ó
  •— t         ‰| �  «        || _        || _        || _        || _        || _        || _        || _        t        |«      | _
        | j                  dk(  r=t        | j                  | j                  | j                  | j                  ¬«      | _        n1t        | j                  | j                  | j
                  ¬«      | _        t        |«      | _        t        | j                  | j                  ¬«      | _        y)aV  Initializes the residual block.

        Args:
          width: The width of the block.
          mlp_expanded_width: The width of the expansion inside the MLP block.
          num_heads: The number of heads for the Attention or the RG-LRU.
          attention_window_size: The window size for the local attention block.
          temporal_block_type: Either "recurrent" or "attention", specifying the
            type of recurrent block to use.
          lru_width: The width of the RG-LRU if different from `width`.
          conv1d_temporal_width: The width of the temporal convolution.
        Ú	recurrent)re   rf   rŠ   r‹   )re   rf   rœ   )re   r±   N)r3   r4   re   rº   rf   r   r»   rŠ   r‹   r/   Útemporal_pre_normr‰   Útemporal_blockr›   Úchannel_pre_normr°   Ú	mlp_block)	r!   re   rº   rf   r   r»   rŠ   r‹   r8   s	           €r"   r4   zResidualBlock.__init__!  sÚ   ø€ ô, 	‰ÑÔØˆŒ
Ø"4ˆÔØ"ˆŒØ%:ˆÔ"Ø#6ˆÔ Ø"ˆŒØ%:ˆÔ"ä!(¨£ˆÔØ×#Ñ# {Ò2Ü"0Ø—j‘jØŸ.™.ØŸ.™.Ø&*×&@Ñ&@ô	#ˆDÕô #6Ø—j‘jØŸ.™.Ø ×6Ñ6ô#ˆDÔô !(¨£ˆÔÜ!Ø—*‘*Ø×2Ñ2ô
ˆ�r$   r=   c                 ó¬   — |}| j                  |«      }| j                  |||¬«      }||z   }| j                  |«      }| j                  |«      }||z   }|S )N)r_   r•   )r¾   r¿   rÀ   rÁ   )r!   r=   r_   r•   Úraw_xÚinputs_normalizedÚresiduals          r"   r>   zResidualBlock.__call__V  sg   € ð ˆà ×2Ñ2°5Ó9Ðà×ÑÐ 1¸ÀTÐÓJˆØ�u‘9ˆà×!Ñ! (Ó+ˆØ�N‰N˜1Óˆà�‰Lˆàˆr$   r–   r˜   )r%   r&   r'   r+   r(   r   r4   r5   r‡   r>   r?   r@   s   @r"   r¹   r¹     sr   ø„ ð $(Ø%&ñ3
àð3
ð  ð3
ð ð	3
ð
  #ð3
ð !ð3
ð ˜C‘=ð3
ð  #õ3
ðp Øñ	à�8‰8÷r$   r¹   c                   ó(   ‡ — e Zd Zˆ fd„Z	 dd„Zˆ xZS )ÚGriffinc                 óH  •— t         ‰| �  «        || _        t        j                  |j
                  |j                  «      | _        |j                  | _	        |j                  }t        |j                  «      D �cg c]J  }t        |j                  |j                  |j                  |j                   ||t#        |«      z     d ¬«      ‘ŒL c}| _        t'        |j                  |j(                  ¬«      | _        |j                   | _        |j/                  d«      | _        y c c}w )N)re   rº   rf   r   r»   rŠ   )r1   Ú	attention)r3   r4   Úconfigry   Ú	Embeddingr   r   Úembed_tokensr   Úscale_by_sqrt_dimr   rI   r   r¹   r   r   r   ÚlenÚlayersr/   r   Ú
final_normrœ   ÚindexÚswa_idx)r!   rÊ   r   Úir8   s       €r"   r4   zGriffin.__init__l  s  ø€ Ü‰ÑÔàˆŒÜŸL™LØ×ÑØ×Ñó
ˆÔð
 "(×!DÑ!DˆÔØ×(Ñ(ˆô ˜6×3Ñ3Ô4ó

ñ 5�ô Ø×(Ñ(Ø#)×#;Ñ#;Ø ×4Ñ4Ø&,×&BÑ&BØ$/°´C¸Ó4DÑ0DÑ$EØöð 5ñ

ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒØ!×7Ñ7ˆÔØ"×(Ñ(¨Ó5ˆ�ùò

s   Á:ADc                 ó~  — | j                  |«      }| j                  r%|t        j                  |j                  d   «      z  }|€d gt        | j                  «      z  }t        ||| j                     | j                  ¬«      }t        | j                  «      D ]  \  }} |||||   ¬«      }Œ | j                  |«      S )NrC   )rœ   )r•   r_   )rÌ   rÍ   Úmathr|   rF   rÎ   rÏ   r   rÒ   rœ   Ú	enumeraterÐ   )r!   Útokensr_   r=   r•   rÓ   Úblocks          r"   r>   zGriffin.__call__‡  s®   € ð
 ×Ñ˜fÓ%ˆØ×!Ò!Ø”D—I‘I˜aŸg™g b™kÓ*Ñ*ˆAàˆ=Ø�FœS §¡Ó-Ñ-ˆEä$Øˆu�T—\‘\Ñ"°×0@Ñ0@ô
ˆô " $§+¡+Ö.‰HˆAˆuÙ�a˜d¨%°©(Ô3‰Að /ð �‰˜qÓ!Ð!r$   r   )r%   r&   r'   r4   r>   r?   r@   s   @r"   rÇ   rÇ   k  s   ø„ ô6ð< ÷"r$   rÇ   c                   óh   — e Zd Zd„ Zd	dej
                  dej
                  fd„Zed„ «       Zd„ Z	d„ Z
y)
ÚModelc                 ó¶   — || _         t        |«      | _        |j                  | _        t	        j
                  |j                  |j                  d¬«      | _        y )NFrž   )	ÚargsrÇ   Úmodelr   ry   r�   r   r   Úlm_head)r!   rÊ   s     r"   r4   zModel.__init__Ÿ  sA   € ØˆŒ	Ü˜V“_ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆ�r$   Nr×   Úreturnc                 ó  — | j                  ||¬«      }d| v r| j                  |«      }n%| j                   j                  j                  |«      }| j                  j
                  }|rt        j                  ||z  «      |z  }|S )N)r_   rÞ   )rÝ   rÞ   rÌ   Ú	as_linearrÜ   r   r5   Útanh)r!   r×   r_   ÚlogitsÚcs        r"   r>   zModel.__call__¥  sr   € Ø—‘˜F¨%�Ó0ˆØ˜ÑØ—\‘\ &Ó)‰Fà—Z‘Z×,Ñ,×6Ñ6°vÓ>ˆFà�I‰I×%Ñ%ˆÙÜ—W‘W˜V a™ZÓ(¨1Ñ,ˆFØˆr$   c                 ó.   — | j                   j                  S r   )rÝ   rÏ   r    s    r"   rÏ   zModel.layers±  s   € à�z‰z× Ñ Ð r$   c                 óº   — |j                  «       D ]2  \  }}d|v sŒ|j                  d   dk7  sŒ|j                  dd«      ||<   Œ4 d|vr| j                  d«       |S )Nzconv_1d.weightrC   r   rp   zlm_head.weightrÞ   )ÚitemsrF   ÚmoveaxisÚpop)r!   ÚweightsÚkÚvs       r"   ÚsanitizezModel.sanitizeµ  s\   € Ø—M‘M–O‰DˆAˆqØ 1Ò$¨¯©°©¸Ó)9ØŸZ™Z¨¨1Ó-�˜’
ð $ð  7Ñ*Ø�H‰H�YÔØˆr$   c                 óà   — g }| j                   D ]\  }|j                  dk(  r|j                  t        d¬«      «       Œ.|j                  t	        | j
                  j                  ¬«      «       Œ^ |S )Nr½   rp   )Úsize)Úmax_size)rÏ   r»   Úappendr
   r   rÜ   r   )r!   r_   Úlayers      r"   Ú
make_cachezModel.make_cache½  sV   € ØˆØ—[”[ˆEØ×(Ñ(¨KÒ7Ø—‘œ[¨aÔ0Õ1à—‘œ_°d·i±i×6UÑ6UÔVÕWð	 !ð
 ˆr$   r   )r%   r&   r'   r4   r5   r‡   r>   ÚpropertyrÏ   rí   ró   r-   r$   r"   rÚ   rÚ   �  sA   „ òTñ
˜rŸx™xð 
¸¿¹ó 
ð ñ!ó ð!òór$   rÚ   )rÕ   Údataclassesr   Útypingr   r   Úmlx.coreÚcorer5   Úmlx.nnry   Úbaser   r   r	   r_   r
   r   r   ÚModuler/   rR   rT   rd   r‰   r›   r°   r¹   rÇ   rÚ   r-   r$   r"   Ú<module>rü      sØ   ðó Ý !ß !å Ý ç TÑ Tß /ð ô1�ó 1ó ð1ô0@ˆb�i‰iô @òô<$ˆR�Y‰Yô $ô2?ˆB�I‰Iô ?ôD/�R—Y‘Yô /ôd2#˜"Ÿ)™)ô 2#ôj8ˆr�y‰yô 8ôI�B—I‘Iô IôX/"ˆb�i‰iô /"ôd'ˆB�I‰Iõ 'r$   