+
    TV-j
3  ã                   ó  € ^ RI t ^ RIHt ^ RIHtHt ^ RIHt ^ RI	H
t
 ^RIHtHtHt ^RIHtHt ] ! R R]4      4       t ! R R	]
P&                  4      tR
 t ! R R]
P&                  4      t ! R R]
P&                  4      t ! R R]
P&                  4      t ! R R]
P&                  4      t ! R R]
P&                  4      t ! R R]
P&                  4      t ! R R]
P&                  4      t ! R R]
P&                  4      tR# )é    N)Ú	dataclass)ÚListÚOptional)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚArraysCacheÚRotatingKVCachec                   ó>   a € ] tR t^t o RtRtRtR tV 3R ltRt	V t
R# )Ú	ModelArgsTNc                óH   € V P                   f   V P                  V n         R # R # ©N)Úblock_typesÚ_block_types©Úselfs   &Ún/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/recurrent_gemma.pyÚ__post_init__ÚModelArgs.__post_init__!   s!   € à×ÑÒ#Ø#×0Ñ0ˆDÖñ $ó    c                ó  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[S[ ,          ,          ;R&   S[S[S[ ,          ,          ;R&   # )é   Ú
model_typeÚattention_biasÚconv1d_widthÚhidden_sizeÚintermediate_sizeÚlogits_soft_capÚnum_attention_headsÚnum_hidden_layersÚnum_key_value_headsÚrms_norm_epsÚ
rope_thetaÚattention_window_sizeÚ
vocab_sizeÚembeddings_scale_by_sqrt_dimr   r   )ÚstrÚboolÚintÚfloatr   r   )ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚModelArgs.__annotate__   sã   ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ �Oñ ñ #'Ñ-ñ ñ  ™$™s�)Õ$Ñ+ñ! ñ" ™4¡�9Õ%Ñ,ò# r   )r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r&   r   r   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r,   s   @r   r   r      s#   ø‡ € ð *.Ð Ø'+€KØ(,€Lò1÷' ƒ r   r   c                   óH   a a€ ] tR t^'t oRV3R lV 3R llltR tRtVtV ;t# )ÚRMSNormc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   ÚdimsÚeps)r)   r*   )r+   r,   s   "€r   r-   ÚRMSNorm.__annotate__(   s   ø€ ÷ ñ ™Sð ¡uñ r   c                óh   <€ \         SV `  4        \        P                  ! V34      V n        W n        R # r   )ÚsuperÚ__init__ÚmxÚonesÚweightr;   )r   r:   r;   Ú	__class__s   &&&€r   r?   ÚRMSNorm.__init__(   s%   ø€ Ü‰ÑÔÜ—g’g˜t˜gÓ&ˆŒØŽr   c                óz   € \         P                  P                  VR V P                  ,           V P                  4      # )ç      ð?)r@   ÚfastÚrms_normrB   r;   )r   Úxs   &&r   Ú__call__ÚRMSNorm.__call__-   s(   € Ü�w‰w×Ñ  3¨¯©Õ#4°d·h±hÓ?Ð?r   )r;   rB   )gñhãˆµøä>©	r/   r0   r1   r2   r?   rJ   r4   r5   Ú__classcell__©rC   r,   s   @@r   r8   r8   '   s   ù‡ € ÷õ ÷
@ò @r   r8   c                 ó~  € V P                   ^8X  g   Q hVP                  V P                  VP                   ) R 8X  g   Q hVP                  V P                  8X  g   Q hV P                  ^,          ^8X  d/   Vf   W R,          3# WR,          ,          V ,           pW3R,          3# Ve   TpM2V P                  w  rVp\        P                  ! WW3V P                  R7      p\        P
                  ! V 4      p\        V P                  ^,          4       F+  pVRV3,          V,          V RV3,          ,           pWCRV3&   K-  	  W43# )é   NºNNN)Údtype)rQ   r   )rQ   N)rQ   éÿÿÿÿ)ÚndimÚshaperR   r@   ÚzerosÚ
zeros_likeÚrange)	rI   ÚaÚh0ÚyÚh_tÚBÚ_ÚDÚts	   &&&      r   Úrnn_scanra   1   s  € Ø�6‰6�QŒ;Ðˆ;Ø�7‰7�a—g‘g˜qŸv™v˜g˜iÐ(Ô(Ð(Ð(Ø�7‰7�a—g‘gÔÐÐà‡w�wˆq…z�Q„àŠ:Ø˜•g�:Ðð �w•K• !Õ#ˆAØ˜•h�;Ðð Š>Ø‰Cà—g‘g‰GˆA�!Ü—(’(˜A˜6¨¯©Ô1ˆCä�MŠM˜!ÓˆÜ�q—w‘w˜q•zÖ"ˆAØ�A�q�D•'˜C•- ! A q D¥'Õ)ˆCØˆa�ˆd‹Gñ #ð ˆ6€Mr   c                   óH   a a€ ] tR t^Ot oV3R lV 3R lltRR ltRtVtV ;t# )ÚConv1dc                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   ÚchannelsÚkernel_size©r)   )r+   r,   s   "€r   r-   ÚConv1d.__annotate__P   s   ø€ ÷ *ñ *áð*ñ ñ*r   c                ó–   <€ \         SV `  4        \        P                  ! W^34      V n        \        P                  ! V34      V n        R# )é   N)r>   r?   r@   rV   rB   Úbias)r   re   rf   rC   s   &&&€r   r?   ÚConv1d.__init__P   s7   ø€ ô
 	‰ÑÔÜ—h’h °qÐ9Ó:ˆŒÜ—H’H˜h˜[Ó)ˆŽ	r   c                ój  € VP                   w  r4pV P                  P                   w  rgpVe   \        P                  ! W!.^R7      pM#\        P                  ! VRV^,
          ^ 3R.4      p\        P
                  ! WP                  VR7      p	W�P                  ,           p	W‘RV) ^,           R 1R3,          3# )N)Úaxis)ÚgroupsrQ   )r   r   )rU   rB   r@   ÚconcatenateÚpadÚconv_generalrk   )
r   rI   Úcacher]   ÚLÚCro   ÚKr^   r[   s
   &&&       r   rJ   ÚConv1d.__call__Y   s”   € Ø—'‘'‰ˆˆaØ—{‘{×(Ñ(‰ˆ�1àÒÜ—’ ˜z°Ô2‰Aä—’�q˜6 A¨¥E¨1 :¨vÐ6Ó7ˆAä�OŠO˜AŸ{™{°6Ô:ˆØ—	‘	�Mˆà�A˜�r˜A•v‘x �NÕ#Ð#Ð#r   )rk   rB   r   rL   rN   s   @@r   rc   rc   O   s   ù‡ € ÷*ó *÷$ô $r   rc   c                   óX   a a€ ] tR t^ht oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚRGLRUz2A Real-Gated Linear Recurrent Unit (RG-LRU) layer.c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   ÚwidthÚ	num_headsrg   )r+   r,   s   "€r   r-   ÚRGLRU.__annotate__k   s#   ø€ ÷ Mñ MáðMñ ñMr   c                ó€  <€ \         SV `  4        Wn        W n        V P                  V P                  ,          V n        \
        P                  ! V P                  34      V n        \
        P                  ! V P                  V P                  V P                  34      V n        \
        P                  ! V P                  V P                  34      V n	        \
        P                  ! V P                  V P                  V P                  34      V n
        \
        P                  ! V P                  V P                  34      V n        R # r   )r>   r?   r{   r|   Úhead_dimr@   rV   Úrecurrent_paramÚinput_gate_weightÚinput_gate_biasÚrecurrent_gate_weightÚrecurrent_gate_bias)r   r{   r|   rC   s   &&&€r   r?   ÚRGLRU.__init__k   sÓ   ø€ ô
 	‰ÑÔØŒ
Ø"ŒØŸ
™
 d§n¡nÕ4ˆŒä!Ÿxšx¨¯©¨Ó6ˆÔä!#§¢Ø�^‰^˜TŸ]™]¨D¯M©MÐ:ó"
ˆÔô  "Ÿxšx¨¯©¸¿¹Ð(GÓHˆÔä%'§X¢XØ�^‰^˜TŸ]™]¨D¯M©MÐ:ó&
ˆÔ"ô $&§8¢8¨T¯^©^¸T¿]¹]Ð,KÓ#LˆÖ r   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   rI   ©r@   Úarray)r+   r,   s   "€r   r-   r}   �   s   ø€ ÷ &ñ &á�8‰8ñ&r   c                ó6  a aa€ VP                   w  oopVVV 3R  lpV! VS P                  S P                  4      pV! VS P                  S P                  4      pRV,          \
        P                  ! S P                  4      ,          p\        P                  ! V4      p\        P                  ! ^V,          4      p	W,          p
\        P                  ! ^V	,
          4      pVf   RVR&   W«P                  VP                  4      ,          p\        VVVR7      w  rÞWÞ3# )c                 ó  <€ V P                  SSSP                  SP                  34      p V P                  ^^4      V,          P                  ^^4      V,           p \        P
                  ! V P                  ^^4      4      # )rj   )Úreshaper|   r   Úswapaxesr@   ÚsigmoidÚflatten)ÚhÚwÚbr]   rt   r   s   &&&€€€r   Úapply_block_linearÚ*RGLRU.__call__.<locals>.apply_block_linearˆ   sb   ø€ Ø—	‘	˜1˜a §¡°·±Ð?Ó@ˆAØ—‘˜A˜qÓ! AÕ%×/Ñ/°°1Ó5¸Õ9ˆAÜ—:’:˜aŸi™i¨¨1›oÓ.Ð.r   rF   )rI   rY   rZ   g       À)rQ   r   rQ   )rU   r�   r‚   rƒ   r„   ÚnnÚsoftplusr€   r@   ÚexpÚsqrtÚastyperR   ra   )r   rI   rs   r^   r“   Úgate_xÚgate_aÚlog_arY   Úa_squareÚgated_xÚ
multiplierÚnormalized_xr[   Úlast_hr]   rt   s   f&&            @@r   rJ   ÚRGLRU.__call__�   sò   ú€ ð
 —'‘'‰ˆˆ1ˆa÷	/ñ $ A t×'=Ñ'=¸t×?SÑ?SÓTˆÙ#Øˆt×)Ñ)¨4×+CÑ+Có
ˆð
 �v•¤§¢¨D×,@Ñ,@Ó AÕAˆÜ�FŠF�5‹MˆÜ—6’6˜!˜e�)Ó$ˆð •*ˆô —W’W˜Q �\Ó*ˆ
ØŠ=Ø"%ˆJ�wÑØ×!2Ñ!2°1·7±7Ó!;Õ;ˆäØØØô
‰	ˆð ˆyÐr   )r   r‚   r�   r|   r„   rƒ   r€   r{   r   )
r/   r0   r1   r2   Ú__doc__r?   rJ   r4   r5   rM   rN   s   @@r   ry   ry   h   s#   ù‡ € Ù<÷Mó M÷,&÷ &ò &r   ry   c                   óX   a a€ ] tR t^ªt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚRecurrentBlockc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ /# )r   r{   r|   Ú	lru_widthÚconv1d_temporal_widthrg   )r+   r,   s   "€r   r-   ÚRecurrentBlock.__annotate__¬   s3   ø€ ÷ 
ñ 
áð
ñ ð
ñ ð	
ñ
  #ñ
r   c                óâ  <€ \         SV `  4        Wn        W n        T;'       g    TV n        W@n        \        P                  ! WP                  4      V n        \        P                  ! WP                  4      V n	        \        P                  ! V P                  V4      V n
        \        V P                  V P
                  R 7      V n        \        V P                  V P                  R7      V n        R# ))re   rf   )r{   r|   N)r>   r?   r{   r|   r§   r¨   r•   ÚLinearÚlinear_yÚlinear_xÚ
linear_outrc   Úconv_1dry   Úrg_lru)r   r{   r|   r§   r¨   rC   s   &&&&&€r   r?   ÚRecurrentBlock.__init__¬   s¥   ø€ ô 	‰ÑÔØŒ
Ø"ŒØ"×+Ð+ eˆŒØ%:Ô"äŸ	š	 %¯©Ó8ˆŒÜŸ	š	 %¯©Ó8ˆŒÜŸ)š) D§N¡N°EÓ:ˆŒÜØ—^‘^Ø×2Ñ2ô
ˆŒô Ø—.‘.Ø—n‘nô
ˆŽr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r‡   rˆ   )r+   r,   s   "€r   r-   r©   Å   s   ø€ ÷ ñ á�8‰8ñr   c                ó0  € V P                  V4      p\        P                  ! V4      pV P                  V4      pVf   R R .pV P	                  W^ ,          R7      w  q^ &   V P                  W^,          R7      w  q^&   W,          pV P                  V4      pV# )N)rI   rs   )r¬   r•   Úgelu_approxr­   r¯   r°   r®   )r   rI   rs   Úmaskr[   s   &&&& r   rJ   ÚRecurrentBlock.__call__Å   sŒ   € ð �M‰M˜!ÓˆÜ�NŠN˜1Óˆð �M‰M˜!ÓˆØŠ=Ø˜4�LˆEØ—l‘l Q°A­h�lÓ7‰ˆ�‰8Ø—k‘k A°1­X�kÓ6‰ˆ�‰8à�EˆØ�O‰O˜AÓˆàˆr   )	r¨   r¯   r®   r­   r¬   r§   r|   r°   r{   ©Né   ©NNrL   rN   s   @@r   r¥   r¥   ª   s   ù‡ € ÷
õ 
÷2÷ ò r   r¥   c                   óT   a a€ ] tR t^Üt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLocalAttentionBlockc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r   r{   r|   Úwindow_sizerg   )r+   r,   s   "€r   r-   Ú LocalAttentionBlock.__annotate__Þ   s)   ø€ ÷ 
ñ 
áð
ñ ð
ñ ñ	
r   c                ó   <€ \         SV `  4        Wn        W n        W0n        W,          R,          V n        V P                  V P                  ,          V n        \        P                  ! V P                  V P                  RR7      V n	        \        P                  ! V P                  V P                  RR7      V n
        \        P                  ! V P                  V P                  RR7      V n        \        P                  ! V P                  V P                  RR7      V n        \        P                  ! V P                  ^,          RR7      V n        R# )g      à?F©rk   T)ÚtraditionalNg      à¿)r>   r?   r{   r|   r½   Úscaler   r•   r«   Úq_projÚk_projÚv_projÚo_projÚRoPEÚrope)r   r{   r|   r½   rC   s   &&&&€r   r?   ÚLocalAttentionBlock.__init__Þ   sÒ   ø€ ô 	‰ÑÔØŒ
Ø"ŒØ&ÔØÕ(¨dÕ3ˆŒ
àŸ
™
 d§n¡nÕ4ˆŒÜ—i’i §
¡
¨D¯J©J¸UÔCˆŒÜ—i’i §
¡
¨D¯M©MÀÔFˆŒÜ—i’i §
¡
¨D¯M©MÀÔFˆŒÜ—i’i §
¡
¨D¯J©J¸TÔBˆŒÜ—G’GØ�M‰M˜QÕØô
ˆŽ	r   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r‡   rˆ   )r+   r,   s   "€r   r-   r¾   ô   s   ø€ ÷ #ñ #á�8‰8ñ#r   c           	     óö  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WE^R4      P                  ^ ^^^4      pV	P	                  WE^R4      P                  ^ ^^^4      p	VeM   V P                  WrP                  R7      pV P                  W‚P                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW’V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      # )rj   )Úoffset)rs   rÂ   rµ   rS   )rU   rÃ   rÄ   rÅ   rŒ   r|   Ú	transposerÈ   rÌ   Úupdate_and_fetchr   rÂ   rÆ   )r   rI   rs   rµ   r]   rt   r_   ÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r   rJ   ÚLocalAttentionBlock.__call__ô   sI  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆà—/‘/ !¨¯©¸Ó;×EÑEÀaÈÈAÈqÓQˆØ�|‰|˜A ! RÓ(×2Ñ2°1°a¸¸AÓ>ˆØ—‘  a¨Ó,×6Ñ6°q¸!¸QÀÓBˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r   )
r   rÄ   r|   rÆ   rÃ   rÈ   rÂ   rÅ   r{   r½   r¹   rL   rN   s   @@r   r»   r»   Ü   s   ù‡ € ÷
ó 
÷,#÷ #ò #r   r»   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMLPBlocki  c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   r{   Úexpanded_widthrg   )r+   r,   s   "€r   r-   ÚMLPBlock.__annotate__  s   ø€ ÷ ?ñ ?™cð ?±3ñ ?r   c                óò   <€ \         SV `  4        \        P                  ! W^,          4      V n        \        P                  ! W^,          4      V n        \        P                  ! V^,          V4      V n        R# )r   N)r>   r?   r•   r«   Úup_projÚ	gate_projÚ	down_proj)r   r{   r×   rC   s   &&&€r   r?   ÚMLPBlock.__init__  sO   ø€ Ü‰ÑÔÜ—y’y ¸!Õ(;Ó<ˆŒÜŸš 5¸AÕ*=Ó>ˆŒÜŸš >°QÕ#6¸Ó>ˆŽr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r‡   rˆ   )r+   r,   s   "€r   r-   rØ     s   ø€ ÷ 8ñ 8™"Ÿ(™(ñ 8r   c                óž   € V P                  V4      pV P                  V4      pV P                  \        P                  ! V4      V,          4      # r   )rÛ   rÚ   rÜ   r•   r´   )r   rI   Úgates   && r   rJ   ÚMLPBlock.__call__  s:   € Ø�~‰~˜aÓ ˆØ�L‰L˜‹OˆØ�~‰~œbŸnšn¨TÓ2°QÕ6Ó7Ð7r   )rÜ   rÛ   rÚ   rL   rN   s   @@r   rÕ   rÕ     s   ù‡ € ÷?ó ?÷8÷ 8ð 8r   rÕ   c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚResidualBlocki  c                óT   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[RS[S[ ,          RS[ /# )r   r{   Úmlp_expanded_widthr|   r$   Útemporal_block_typer§   r¨   )r)   r'   r   )r+   r,   s   "€r   r-   ÚResidualBlock.__annotate__!  sU   ø€ ÷ 3
ñ 3
áð3
ñ  ð3
ñ ð	3
ñ
  #ð3
ñ !ð3
ñ ™C•=ð3
ñ  #ñ3
r   c                ó  <€ \         SV `  4        Wn        W n        W0n        W@n        WPn        W`n        Wpn        \        V4      V n
        V P                  R8X  d>   \        V P                  V P                  V P                  V P                  R7      V n        M1\        V P                  V P                  V P
                  R7      V n        \        V4      V n        \        V P                  V P                  R7      V n        R# )a  Initializes the residual block.

Args:
  width: The width of the block.
  mlp_expanded_width: The width of the expansion inside the MLP block.
  num_heads: The number of heads for the Attention or the RG-LRU.
  attention_window_size: The window size for the local attention block.
  temporal_block_type: Either "recurrent" or "attention", specifying the
    type of recurrent block to use.
  lru_width: The width of the RG-LRU if different from `width`.
  conv1d_temporal_width: The width of the temporal convolution.
Ú	recurrent)r{   r|   r§   r¨   )r{   r|   r½   )r{   r×   N)r>   r?   r{   rå   r|   r$   ræ   r§   r¨   r8   Útemporal_pre_normr¥   Útemporal_blockr»   Úchannel_pre_normrÕ   Ú	mlp_block)	r   r{   rå   r|   r$   ræ   r§   r¨   rC   s	   &&&&&&&&€r   r?   ÚResidualBlock.__init__!  sÌ   ø€ ô, 	‰ÑÔØŒ
Ø"4ÔØ"ŒØ%:Ô"Ø#6Ô Ø"ŒØ%:Ô"ä!(¨£ˆÔØ×#Ñ# {Ô2Ü"0Ø—j‘jØŸ.™.ØŸ.™.Ø&*×&@Ñ&@ô	#ˆDÕô #6Ø—j‘jØŸ.™.Ø ×6Ñ6ô#ˆDÔô !(¨£ˆÔÜ!Ø—*‘*Ø×2Ñ2ô
ˆŽr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r‡   rˆ   )r+   r,   s   "€r   r-   rç   V  s   ø€ ÷ ñ á�8‰8ñr   c                ó¶   € TpV P                  V4      pV P                  WRVR 7      pW,           pV P                  V4      pV P                  V4      pW,           pV# ))rs   rµ   )rê   rë   rì   rí   )r   rI   rs   rµ   Úraw_xÚinputs_normalizedÚresiduals   &&&&   r   rJ   ÚResidualBlock.__call__V  sa   € ð ˆà ×2Ñ2°5Ó9Ðà×ÑÐ 1ÀTÐÓJˆØ•9ˆà×!Ñ! (Ó+ˆØ�N‰N˜1Óˆà�Lˆàˆr   )r$   rì   r¨   r§   rí   rå   r|   rë   ræ   rê   r{   r·   r¹   rL   rN   s   @@r   rã   rã     s   ù‡ € ÷3
õ 3
÷j÷ ò r   rã   c                   ó<   a a€ ] tR tRt oV 3R ltRR ltRtVtV ;t# )ÚGriffinik  c                ó\  <€ \         SV `  4        Wn        \        P                  ! VP
                  VP                  4      V n        VP                  V n	        VP                  p\        VP                  4       Uu. uFR  p\        VP                  VP                  VP                  VP                   W#\#        V4      ,          ,          R R7      NKT  	  upV n        \'        VP                  VP(                  R7      V n        VP                   V n        VP/                  R4      V n        R # u upi )N)r{   rå   r|   r$   ræ   r§   )r;   Ú	attention)r>   r?   Úconfigr•   Ú	Embeddingr%   r   Úembed_tokensr&   Úscale_by_sqrt_dimr   rX   r    rã   r   r   r$   ÚlenÚlayersr8   r"   Ú
final_normr½   ÚindexÚswa_idx)r   rù   r   ÚirC   s   &&  €r   r?   ÚGriffin.__init__l  sþ   ø€ Ü‰ÑÔàŒÜŸLšLØ×ÑØ×Ñó
ˆÔð
 "(×!DÑ!DˆÔØ×(Ñ(ˆô ˜6×3Ñ3Ô4ó

ñ 5�ô Ø×(Ñ(Ø#)×#;Ñ#;Ø ×4Ñ4Ø&,×&BÑ&BØ$/´C¸Ó4DÕ0DÕ$EØ÷ñ 5ñ

ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒØ!×7Ñ7ˆÔØ"×(Ñ(¨Ó5ˆŽùò

s   Á:AD)c                ó´  € V P                  V4      pV P                  '       d/   V\        P                  ! VP                  R,          4      ,          pVf   R.\        V P                  4      ,          p\        W2V P                  ,          V P                  R7      p\        V P                  4       F  w  rVV! W4W%,          R7      pK  	  V P                  V4      # )rj   N)r½   )rµ   rs   rS   )rû   rü   Úmathr˜   rU   rý   rþ   r   r  r½   Ú	enumeraterÿ   )r   Útokensrs   rI   rµ   r  Úblocks   &&&    r   rJ   ÚGriffin.__call__‡  s©   € ð
 ×Ñ˜fÓ%ˆØ×!×!Ð!Ø”D—I’I˜aŸg™g b�kÓ*Õ*ˆAàŠ=Ø�FœS §¡Ó-Õ-ˆEä$Ø�T—\‘\Õ"°×0@Ñ0@ô
ˆô " $§+¡+Ö.‰HˆAÙ�a¨%­(Ô3ŠAñ /ð �‰˜qÓ!Ð!r   )rù   rû   rÿ   rþ   rü   r  r½   r   rL   rN   s   @@r   rö   rö   k  s   ù‡ € õ6÷6"ô "r   rö   c                   óX   a € ] tR tRt o R tR
V 3R lR llt]R 4       tR tR t	R	t
V tR# )ÚModeli�  c                ó¸   € Wn         \        V4      V n        VP                  V n        \        P
                  ! VP                  VP                  R R7      V n        R# )FrÀ   N)	Úargsrö   Úmodelr   r•   r«   r   r%   Úlm_head)r   rù   s   &&r   r?   ÚModel.__init__Ÿ  s?   € ØŒ	Ü˜V“_ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŽr   Nc                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r  Úreturnrˆ   )r+   r,   s   "€r   r-   ÚModel.__annotate__¥  s#   ø€ ÷ 
ñ 
™rŸx™xð 
¹¿¹ñ 
r   c                ó(  € V P                  WR 7      pRV 9   d   V P                  V4      pM%V P                   P                  P                  V4      pV P                  P
                  pV'       d$   \        P                  ! W4,          4      V,          pV# ))rs   r  )r  r  rû   Ú	as_linearr  r   r@   Útanh)r   r  rs   ÚlogitsÚcs   &&&  r   rJ   ÚModel.__call__¥  sn   € Ø—‘˜F�Ó0ˆØ˜ÔØ—\‘\ &Ó)‰Fà—Z‘Z×,Ñ,×6Ñ6°vÓ>ˆFà�I‰I×%Ñ%ˆßÜ—W’W˜V�ZÓ(¨1Õ,ˆFØˆr   c                ó.   € V P                   P                  # r   )r  rþ   r   s   &r   rþ   ÚModel.layers±  s   € à�z‰z× Ñ Ð r   c                óÖ   € VP                  4        F<  w  r#R V9   g   K  VP                  R,          ^8w  g   K(  VP                  ^^4      W&   K>  	  RV9  d   V P                  R4       V# )zconv_1d.weightzlm_head.weightr  rS   )ÚitemsrU   ÚmoveaxisÚpop)r   ÚweightsÚkÚvs   &&  r   ÚsanitizeÚModel.sanitizeµ  sX   € Ø—M‘M–O‰DˆAØ 1Ö$¨¯©°­¸Ö)9ØŸZ™Z¨¨1Ó-�“
ñ $ð  7Ô*Ø�H‰H�YÔØˆr   c                óê   € . pV P                    F`  pVP                  R 8X  d   VP                  \        ^R7      4       K1  VP                  \	        V P
                  P                  R7      4       Kb  	  V# )ré   )Úsize)Úmax_size)rþ   ræ   Úappendr	   r
   r  r$   )r   rs   Úlayers   &  r   Ú
make_cacheÚModel.make_cache½  sV   € ØˆØ—[”[ˆEØ×(Ñ(¨KÔ7Ø—‘œ[¨aÔ0Ö1à—‘œ_°d·i±i×6UÑ6UÔVÖWñ	 !ð
 ˆr   )r  r  r  r   r   )r/   r0   r1   r2   r?   rJ   Úpropertyrþ   r#  r*  r4   r5   r6   s   @r   r  r  �  s8   ø‡ € òT÷
ò 
ð ñ!ó ð!ò÷ð r   r  )r  Údataclassesr   Útypingr   r   Úmlx.coreÚcorer@   Úmlx.nnr•   Úbaser   r   r   rs   r	   r
   r   ÚModuler8   ra   rc   ry   r¥   r»   rÕ   rã   rö   r  © r   r   Ú<module>r5     sØ   ðó Ý !ß !å Ý ç TÑ Tß /ð ô1�ó 1ó ð1ô0@ˆb�i‰iô @òô<$ˆR�Y‰Yô $ô2?ˆB�I‰Iô ?ôD/�R—Y‘Yô /ôd2#˜"Ÿ)™)ô 2#ôj8ˆr�y‰yô 8ôI�B—I‘Iô IôX/"ˆb�i‰iô /"ôd'ˆB�I‰Iö 'r   